testdriverai 7.2.38 → 7.2.40

package/agent/lib/sandbox.js

@@ -1,5 +1,4 @@
 const WebSocket = require("ws");
-const marky = require("marky");
 const crypto = require("crypto");
 const { events } = require("../events");
 
@@ -83,10 +82,6 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
           }
         }
 
-        // Start timing for this message
-        const timingKey = `sandbox-${message.type}`;
-        marky.mark(timingKey);
-
         let p = new Promise((resolve, reject) => {
           this.socket.send(JSON.stringify(message));
           emitter.emit(events.sandbox.sent, message);
@@ -99,13 +94,6 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
         // Set up timeout to prevent hanging requests
         const timeoutId = setTimeout(() => {
           if (this.ps[requestId]) {
-            const pendingMessage = this.ps[requestId];
-            // Stop the timing marker to prevent memory leak
-            try {
-              marky.stop(pendingMessage.timingKey);
-            } catch (e) {
-              // Ignore timing errors
-            }
             delete this.ps[requestId];
             rejectPromise(new Error(`Sandbox message '${message.type}' timed out after ${timeout}ms`));
           }
@@ -122,7 +110,6 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
             rejectPromise(error);
           },
           message,
-          timingKey,
           startTime: Date.now(),
         };
 
@@ -252,25 +239,6 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
             this.ps[message.requestId].reject(error);
           } else {
             emitter.emit(events.sandbox.received);
-
-            // Get timing information for this message
-            const pendingMessage = this.ps[message.requestId];
-            if (pendingMessage) {
-              const timing = marky.stop(pendingMessage.timingKey);
-
-              // Track timing for each message type
-              await analytics.track("sandbox", {
-                operation: pendingMessage.message.type,
-                timing,
-                requestId: message.requestId,
-                timestamp: Date.now(),
-                data: {
-                  messageType: pendingMessage.message.type,
-                  ...pendingMessage.message,
-                },
-              });
-            }
-
             this.ps[message.requestId]?.resolve(message);
           }
           delete this.ps[message.requestId];

package/agents.md

@@ -241,6 +241,8 @@ describe("Experiment", () => {
 });
 ```
 
+> ⚠️ **NEVER REMOVE `reconnect: true`**: If a test file already has `reconnect: true`, do NOT remove it. This option is intentional and removing it will break the two-file workflow. Only remove `reconnect: true` when explicitly combining files into a final standalone test.
+
 ### Step 4: Iterate in Experiment File
 
 - Run experiment, see output, fix issues
@@ -260,13 +262,15 @@ vitest run tests/experiment.test.mjs
 
 ### After Experimentation: Combine and Rename
 
-**IMPORTANT:** Once the test is working, combine both files into a single, properly-named test file:
+**IMPORTANT:** Once the test is working AND the user explicitly asks to combine/finalize the test, combine both files into a single, properly-named test file:
 
 1. **Merge the code** - Copy the working steps from `experiment.test.mjs` into `setup.test.mjs`
-2. **Remove reconnect options** - Delete `reconnect: true` since the final test runs standalone
+2. **Remove reconnect options** - Delete `reconnect: true` since the final test runs standalone (ONLY do this when creating the final combined test - never remove it from an existing experiment file!)
 3. **Rename the file** - Give it a meaningful name like `login-flow.test.mjs` or `checkout-process.test.mjs`
 4. **Delete the experiment file** - Clean up `experiment.test.mjs`
 
+> ⚠️ **WARNING FOR AI AGENTS**: Do NOT remove `reconnect: true` from any existing test file unless you are explicitly combining files into a final test. If a test has `reconnect: true`, it is there intentionally.
+
 ```javascript
 // Final combined test: login-flow.test.mjs
 import { describe, expect, it } from "vitest";

package/docs/v7/customizing-devices.mdx

@@ -15,27 +15,6 @@ const testdriver = TestDriver(context, {
 });
 ```
 
-### Headless Mode
-
-Run tests without a visible browser window. Useful for CI/CD pipeline or spawning multiple tests locally:
-
-```javascript
-const testdriver = TestDriver(context, {
-  headless: true,  // No visible browser window
-});
-```
-
-### IP Target
-
-If self-hosting TestDriver, use `ip` to specify the device IP. See [Self-Hosting TestDriver](../self-hosting.md) for details.
-
-```javascript
-const testdriver = TestDriver(context, {
-  newSandbox: true,
-  ip: "203.0.113.42",  // Your allowlisted IP
-});
-```
-
 ### Operating System
 
 Set the `os` property to run tests on a specific operating system. Available options are `linux` (default) and `windows`.
@@ -77,6 +56,27 @@ steps:
   - run: TD_OS=${{ matrix.os }} npx vitest run
 ```
 
+### Headless Mode
+
+Run tests without a visible browser window. Useful for CI/CD pipeline or spawning multiple tests locally:
+
+```javascript
+const testdriver = TestDriver(context, {
+  headless: true,  // No visible browser window
+});
+```
+
+### IP Target
+
+If self-hosting TestDriver, use `ip` to specify the device IP. See [Self-Hosting TestDriver](../self-hosting.md) for details.
+
+```javascript
+const testdriver = TestDriver(context, {
+  newSandbox: true,
+  ip: "203.0.113.42",  // Your allowlisted IP
+});
+```
+
 ## Keepalive
 
 By default, sandboxes terminate immediately when the test finishes. Set this value to keep the sandbox alive for reconnection.

package/interfaces/vitest-plugin.d.ts

@@ -0,0 +1,101 @@
+/**
+ * TypeScript definitions for TestDriver Vitest Plugin
+ * @module testdriverai/vitest/plugin
+ */
+
+import TestDriverSDK, { TestDriverOptions } from '../sdk';
+
+/**
+ * Plugin state object
+ */
+export interface PluginState {
+  dashcamUrls: Map<string, { url: string; platform: string }>;
+  suiteTestRuns: Map<string, any>;
+  testDriverOptions: TestDriverOptions;
+}
+
+/**
+ * Current plugin state
+ */
+export const pluginState: PluginState;
+
+/**
+ * Register a Dashcam URL for a test
+ */
+export function registerDashcamUrl(testId: string, url: string, platform: string): void;
+
+/**
+ * Get Dashcam URL for a test
+ */
+export function getDashcamUrl(testId: string): { url: string; platform: string } | undefined;
+
+/**
+ * Clear all Dashcam URLs
+ */
+export function clearDashcamUrls(): void;
+
+/**
+ * Get suite test run data
+ */
+export function getSuiteTestRun(suiteId: string): any;
+
+/**
+ * Set suite test run data
+ */
+export function setSuiteTestRun(suiteId: string, runData: any): void;
+
+/**
+ * Clear suite test run data
+ */
+export function clearSuiteTestRun(suiteId: string): void;
+
+/**
+ * Get the current plugin state
+ */
+export function getPluginState(): PluginState;
+
+/**
+ * Authenticate with API key
+ */
+export function authenticateWithApiKey(apiKey: string, apiRoot?: string): Promise<string>;
+
+/**
+ * Create a test run directly via API
+ */
+export function createTestRunDirect(token: string, apiRoot: string, testRunData: any): Promise<any>;
+
+/**
+ * Record a test case directly via API
+ */
+export function recordTestCaseDirect(token: string, apiRoot: string, testCaseData: any): Promise<any>;
+
+/**
+ * Create a TestDriver instance
+ */
+export function createTestDriver(options?: TestDriverOptions): Promise<TestDriverSDK>;
+
+/**
+ * Register a test with TestDriver
+ */
+export function registerTest(testdriver: TestDriverSDK, context: any): void;
+
+/**
+ * Cleanup a TestDriver instance
+ */
+export function cleanupTestDriver(testdriver: TestDriverSDK): Promise<void>;
+
+/**
+ * Plugin options
+ */
+export interface TestDriverPluginOptions extends TestDriverOptions {
+  /**
+   * API key (defaults to TD_API_KEY env var)
+   */
+  apiKey?: string;
+}
+
+/**
+ * TestDriver Vitest Plugin
+ * @param options - Plugin configuration options
+ */
+export default function testDriverPlugin(options?: TestDriverPluginOptions): any;

package/jsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "target": "ES2022",
+    "checkJs": true,
+    "strict": false,
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "baseUrl": ".",
+    "typeRoots": ["."]
+  },
+  "include": [
+    "**/*.js",
+    "**/*.mjs",
+    "**/*.d.ts",
+    "sdk.d.ts",
+    "lib/**/*.d.ts",
+    "interfaces/**/*.d.ts"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "build"
+  ]
+}

package/lib/core/Dashcam.js

@@ -130,23 +130,6 @@ class Dashcam {
       const dashcamPath = await this._getDashcamPath();
       this._log('debug', 'Dashcam executable path:', dashcamPath);
       
-      const installedVersion = await this.client.exec(
-        shell,
-        'npm ls dashcam -g',
-        40000,
-        true
-      );
-      this._log('debug', 'Installed dashcam version:', installedVersion);
-      
-      // Test version command
-      const versionTest = await this.client.exec(
-        shell,
-        `& "${dashcamPath}" version`,
-        40000,
-        true
-      );
-      this._log('debug', 'Dashcam version test:', versionTest);
-      
       // Authenticate with TD_API_ROOT
       const authOutput = await this.client.exec(
         shell,

package/lib/vitest/hooks.d.ts

@@ -3,7 +3,7 @@
  * @module testdriverai/vitest/hooks
  */
 
-import { Dashcam, DashcamOptions, TestDriver, TestDriverOptions } from '../core/index';
+import TestDriverSDK, { TestDriverOptions } from '../../sdk';
 
 /**
  * Vitest test context (from test function parameter)
@@ -21,99 +21,49 @@ export interface VitestContext {
 }
 
 /**
- * Options for useTestDriver hook
+ * Options for TestDriver hook
  */
-export interface UseTestDriverOptions extends TestDriverOptions {
+export interface TestDriverHookOptions extends TestDriverOptions {
   /**
-   * Automatically connect to sandbox (default: true)
+   * Force creation of a new sandbox (default: true)
    */
-  autoConnect?: boolean;
+  newSandbox?: boolean;
   
   /**
-   * Create new sandbox (default: true)
+   * Reconnect to the last used sandbox
    */
-  new?: boolean;
-}
-
-/**
- * Options for useDashcam hook
- */
-export interface UseDashcamOptions extends DashcamOptions {
-  /**
-   * Automatically authenticate (default: true)
-   */
-  autoAuth?: boolean;
+  reconnect?: boolean;
   
   /**
-   * Automatically start recording (default: false)
+   * Direct IP address to connect to a running sandbox instance
    */
-  autoStart?: boolean;
-  
-  /**
-   * Automatically stop recording at test end (default: false)
-   */
-  autoStop?: boolean;
+  ip?: string;
 }
 
 /**
- * Use TestDriver client in a test
- * Creates and manages TestDriver instance for the current test
+ * Create a TestDriver client for use in Vitest tests
+ * Manages lifecycle automatically (connects on first use, disconnects after test)
  * 
  * @param context - Vitest test context (from async (context) => {})
  * @param options - TestDriver options
- * @returns TestDriver client instance
+ * @returns TestDriver SDK instance
  * 
  * @example
- * test('my test', async (context) => {
- *   const client = useTestDriver(context, { os: 'linux' });
- *   await client.find('Login button').click();
- * });
- */
-export function useTestDriver(context: VitestContext, options?: UseTestDriverOptions): TestDriver;
-
-/**
- * Use Dashcam in a test
- * Creates and manages Dashcam instance for the current test
- * 
- * @param context - Vitest test context
- * @param client - TestDriver client instance (from useTestDriver)
- * @param options - Dashcam options
- * @returns Dashcam instance
+ * import { describe, expect, it } from "vitest";
+ * import { TestDriver } from "testdriverai/lib/vitest/hooks.mjs";
  * 
- * @example
- * test('my test', async (context) => {
- *   const client = useTestDriver(context);
- *   const dashcam = useDashcam(context, client, {
- *     autoStart: true,
- *     autoStop: true
- *   });
- *   
- *   await client.find('button').click();
- * });
- */
-export function useDashcam(context: VitestContext, client: TestDriver, options?: UseDashcamOptions): Dashcam;
-
-/**
- * Use TestDriver with Dashcam in one call
- * Combined hook for the simplest usage pattern
- * 
- * @param context - Vitest test context
- * @param options - Combined options for TestDriver and Dashcam
- * @returns Object with client and dashcam instances
- * 
- * @example
- * test('my test', async (context) => {
- *   const { client, dashcam } = useTestDriverWithDashcam(context, {
- *     os: 'linux'
+ * describe("My Test Suite", () => {
+ *   it("should do something", async (context) => {
+ *     const testdriver = TestDriver(context, { newSandbox: true, headless: false });
+ *     
+ *     await testdriver.provision.chrome({ url: 'https://example.com' });
+ *     
+ *     const button = await testdriver.find("Sign In button");
+ *     await button.click();
+ *     
+ *     const result = await testdriver.assert("the dashboard is visible");
+ *     expect(result).toBeTruthy();
  *   });
- *   
- *   await client.find('button').click();
  * });
  */
-export function useTestDriverWithDashcam(
-  context: VitestContext,
-  options?: UseTestDriverOptions & UseDashcamOptions
-): {
-  client: TestDriver;
-  dashcam: Dashcam;
-};
+export function TestDriver(context: VitestContext, options?: TestDriverHookOptions): TestDriverSDK;

package/package.json

@@ -1,15 +1,31 @@
 {
   "name": "testdriverai",
-  "version": "7.2.38",
+  "version": "7.2.40",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
+  "types": "sdk.d.ts",
   "exports": {
-    ".": "./sdk.js",
-    "./core": "./lib/core/index.js",
-    "./vitest": "./interfaces/vitest-plugin.mjs",
-    "./vitest/plugin": "./interfaces/vitest-plugin.mjs",
+    ".": {
+      "types": "./sdk.d.ts",
+      "default": "./sdk.js"
+    },
+    "./core": {
+      "types": "./lib/core/index.d.ts",
+      "default": "./lib/core/index.js"
+    },
+    "./vitest": {
+      "types": "./interfaces/vitest-plugin.d.ts",
+      "default": "./interfaces/vitest-plugin.mjs"
+    },
+    "./vitest/plugin": {
+      "types": "./interfaces/vitest-plugin.d.ts",
+      "default": "./interfaces/vitest-plugin.mjs"
+    },
     "./vitest/setup": "./lib/vitest/setup.mjs",
-    "./vitest/hooks": "./lib/vitest/hooks.mjs",
+    "./vitest/hooks": {
+      "types": "./lib/vitest/hooks.d.ts",
+      "default": "./lib/vitest/hooks.mjs"
+    },
     "./presets": "./lib/presets/index.mjs"
   },
   "bin": {
@@ -47,12 +63,12 @@
   "license": "ISC",
   "dependencies": {
     "@npmcli/redact": "^3.2.2",
-    "@octokit/rest": "^20.1.1",
     "@oclif/core": "^4.5.0",
     "@oclif/plugin-commands": "^4.1.28",
     "@oclif/plugin-help": "^6.2.30",
     "@oclif/plugin-not-found": "^3.2.59",
     "@oclif/plugin-warn-if-update-available": "^3.1.43",
+    "@octokit/rest": "^20.1.1",
     "@sentry/node": "^9.47.1",
     "@stoplight/yaml-ast-parser": "^0.0.50",
     "ajv": "^8.17.1",
@@ -107,7 +123,6 @@
     "mocha": "^10.8.2",
     "node-addon-api": "^8.0.0",
     "prettier": "3.3.3",
-    "testdriverai": "^7.2.3",
     "vitest": "^4.0.16"
   },
   "optionalDependencies": {

package/sdk.d.ts

@@ -242,7 +242,7 @@ export interface TestDriverOptions {
   sandboxInstance?: string;
   /** Cache key for element finding operations. If provided, enables caching tied to this key */
   cacheKey?: string;
-  /** Reconnect to the last used sandbox (throws error if no last sandbox exists) */
+  /** Reconnect to the last used sandbox instead of creating a new one. When true, provision methods (chrome, vscode, installer, etc.) will be skipped since the application is already running. Throws error if no previous sandbox exists. */
   reconnect?: boolean;
   /** Redraw configuration for screen change detection */
   redraw?: boolean | {
@@ -266,7 +266,7 @@ export interface ConnectOptions {
   sandboxId?: string;
   /** Force creation of a new sandbox */
   newSandbox?: boolean;
-  /** Reconnect to the last used sandbox (throws error if no last sandbox exists) */
+  /** Reconnect to the last used sandbox instead of creating a new one. When true, provision methods (chrome, vscode, installer, etc.) will be skipped since the application is already running. Throws error if no previous sandbox exists. */
   reconnect?: boolean;
   /** Direct IP address to connect to a running sandbox instance */
   ip?: string;
@@ -683,9 +683,139 @@ export class Element {
   readonly label: string | null;
 }
 
+// ====================================
+// Provision API Interfaces
+// ====================================
+
+/** Options for provision.chrome */
+export interface ProvisionChromeOptions {
+  /** URL to navigate to (default: 'http://testdriver-sandbox.vercel.app/') */
+  url?: string;
+  /** Start maximized (default: true) */
+  maximized?: boolean;
+  /** Use guest mode (default: false) */
+  guest?: boolean;
+}
+
+/** Options for provision.chromeExtension */
+export interface ProvisionChromeExtensionOptions {
+  /** Local filesystem path to the unpacked extension directory */
+  extensionPath?: string;
+  /** Chrome Web Store extension ID */
+  extensionId?: string;
+  /** Start maximized (default: true) */
+  maximized?: boolean;
+}
+
+/** Options for provision.vscode */
+export interface ProvisionVSCodeOptions {
+  /** Path to workspace or folder to open */
+  workspace?: string;
+  /** Array of extension IDs to install */
+  extensions?: string[];
+}
+
+/** Options for provision.installer */
+export interface ProvisionInstallerOptions {
+  /** URL to download the installer from */
+  url: string;
+  /** Filename to save as (auto-detected from URL if not provided) */
+  filename?: string;
+  /** Application name to focus after install */
+  appName?: string;
+  /** Whether to launch the app after installation (default: true) */
+  launch?: boolean;
+}
+
+/** Options for provision.electron */
+export interface ProvisionElectronOptions {
+  /** Path to Electron app (required) */
+  appPath: string;
+  /** Additional electron args */
+  args?: string[];
+}
+
+/** Provision API for launching applications */
+export interface ProvisionAPI {
+  /**
+   * Launch Chrome browser
+   * @param options - Chrome launch options
+   */
+  chrome(options?: ProvisionChromeOptions): Promise<void>;
+
+  /**
+   * Launch Chrome browser with a custom extension loaded
+   * @param options - Chrome extension launch options
+   */
+  chromeExtension(options?: ProvisionChromeExtensionOptions): Promise<void>;
+
+  /**
+   * Launch VS Code
+   * @param options - VS Code launch options
+   */
+  vscode(options?: ProvisionVSCodeOptions): Promise<void>;
+
+  /**
+   * Download and install an application
+   * @param options - Installer options
+   * @returns Path to the downloaded file
+   */
+  installer(options: ProvisionInstallerOptions): Promise<string>;
+
+  /**
+   * Launch Electron app
+   * @param options - Electron launch options
+   */
+  electron(options: ProvisionElectronOptions): Promise<void>;
+}
+
+/** Dashcam API for screen recording */
+export interface DashcamAPI {
+  /**
+   * Start recording
+   */
+  start(): Promise<void>;
+
+  /**
+   * Stop recording and get replay URL
+   */
+  stop(): Promise<string | null>;
+
+  /**
+   * Check if currently recording
+   */
+  isRecording(): boolean;
+}
+
 export default class TestDriverSDK {
   constructor(apiKey: string, options?: TestDriverOptions);
 
+  /**
+   * Whether the SDK is currently connected to a sandbox
+   */
+  readonly connected: boolean;
+
+  /**
+   * The operating system of the sandbox
+   */
+  readonly os: 'windows' | 'linux';
+
+  /**
+   * Provision API for launching applications
+   */
+  readonly provision: ProvisionAPI;
+
+  /**
+   * Dashcam API for screen recording
+   */
+  readonly dashcam: DashcamAPI;
+
+  /**
+   * Wait for the sandbox to be ready
+   * Called automatically by provision methods
+   */
+  ready(): Promise<void>;
+
   /**
    * Authenticate with TestDriver API
    */

package/sdk.js

@@ -1362,10 +1362,13 @@ class TestDriverSDK {
 
   /**
    * Create the provision API with methods for launching applications
+   * Automatically skips provisioning when reconnect mode is enabled
    * @private
    */
   _createProvisionAPI() {
-    return {
+    const self = this;
+    
+    const provisionMethods = {
       /**
        * Launch Chrome browser
        * @param {Object} options - Chrome launch options
@@ -2073,6 +2076,24 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
         await this.focusApplication('Electron');
       },
     };
+
+    // Wrap all provision methods with reconnect check using Proxy
+    return new Proxy(provisionMethods, {
+      get(target, prop) {
+        const method = target[prop];
+        if (typeof method === 'function') {
+          return async (...args) => {
+            // Skip provisioning if reconnecting to existing sandbox
+            if (self.reconnect) {
+              console.log(`[provision.${prop}] Skipping provisioning (reconnect mode)`);
+              return;
+            }
+            return method(...args);
+          };
+        }
+        return method;
+      }
+    });
   }
 
   /**

package/setup/aws/cloudformation.yaml

@@ -168,8 +168,8 @@ Resources:
             IpProtocol: tcp,
             FromPort: 8765,
             ToPort: 8765,
-            CidrIp: 0.0.0.0/0,
-            Description: "pyautogui-cli WebSockets - Open for testing",
+            CidrIp: !Ref AllowedIngressCidr,
+            Description: "pyautogui-cli WebSockets"
           }
         - {
             IpProtocol: tcp,

package/test/testdriver/hover-text.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Hover Text Test", () => {
   it("should click Sign In and verify error message", async (context) => {
-    const testdriver = TestDriver(context, { headless: false, newSandbox: true, cacheKey: 'hover-text-test' });
+    const testdriver = TestDriver(context, { headless: false, ip: '3.138.116.105'});
     await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
 
     // Click on Sign In button using new find() API

package/test/testdriver/scroll-keyboard.test.mjs

@@ -30,7 +30,7 @@ describe("Scroll Keyboard Test", () => {
     await testdriver.scroll("down", { amount: 1000 });
 
     // Assert the page is scrolled down
-    const result = await testdriver.assert("the page is scrolled down, the hamster dance h1 text heading is not visible on the webpage");
+    const result = await testdriver.assert("The text 'The Hamster Dance' is not visible on the webpage content. It's ok if it's visible in the tab title.");
 
     expect(result).toBeTruthy();
   });

package/test/testdriver/scroll.test.mjs

@@ -35,7 +35,7 @@ describe("Scroll Test", () => {
     await testdriver.scroll("down", { amount: 1000 });
 
     // Assert page is scrolled
-    const result = await testdriver.assert("the page is scrolled down, the hamster dance h1 text heading is not visible on the webpage");
+    const result = await testdriver.assert("The text 'The Hamster Dance' is not visible on the webpage content. It's ok if it's visible in the tab title.");
     expect(result).toBeTruthy();
   });
 });