testdriverai 7.2.3 → 7.2.10

package/.github/workflows/publish.yaml

@@ -1,12 +1,13 @@
-name: Publish Beta
+name: Publish
 permissions:
     contents: write
+    id-token: write  # Required for OIDC
 on:
   push:
     branches: [ main ]
 
 jobs:
-  publish-beta:
+  publish:
     runs-on: ubuntu-latest
     
     steps:
@@ -29,16 +30,23 @@ jobs:
     - name: Install dependencies
       run: npm ci
     
-    - name: Bump version (prerelease beta)
-      run: npm version prerelease --preid=beta --no-git-tag-version
+    - name: Bump version (patch)
+      run: npm version patch --no-git-tag-version
     
     - name: Commit and push version bump
       run: |
         git add package.json package-lock.json
-        git commit -m "chore: bump beta version to $(node -p "require('./package.json').version")"
+        git commit -m "chore: bump version to $(node -p "require('./package.json').version")"
         git push
     
-    - name: Publish to npm under beta tag
-      run: npm publish --tag beta
+    - name: Debug NPM Token
+      run: |
+        echo "NPM_TOKEN is set: ${{ secrets.NPM_TOKEN != '' }}"
+        echo "NPM_TOKEN first 4 chars: ${NPM_TOKEN:0:4}..."
+      env:
+        NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+    
+    - name: Publish to npm
+      run: npm publish --tag beta 
       env:
         NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.github/workflows/testdriver.yml

@@ -0,0 +1,163 @@
+name: TestDriver.ai Tests
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Run TestDriver.ai tests
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+      run: npx vitest run
+
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results
+        path: test-results/
+        retention-days: 30
+
+    # Init command test - only runs on PRs, not on main/master
+    - name: Create test directory for init
+      if: github.event_name == 'pull_request'
+      run: |
+        mkdir -p /tmp/test-init-project
+        cd /tmp/test-init-project
+
+    - name: Run init command (skip prompts)
+      if: github.event_name == 'pull_request'
+      working-directory: /tmp/test-init-project
+      run: |
+        # Create .env with API key first to skip the prompt
+        echo "TD_API_KEY=${{ secrets.TD_API_KEY }}" > .env
+        
+        # Run init command using the CLI from the repo
+        node ${{ github.workspace }}/bin/testdriverai.js init
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+
+    - name: Verify project structure
+      if: github.event_name == 'pull_request'
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking generated files..."
+        
+        # Check for package.json
+        if [ ! -f "package.json" ]; then
+          echo "❌ package.json not found"
+          exit 1
+        fi
+        echo "✓ package.json exists"
+        
+        # Check for vitest config
+        if [ ! -f "vitest.config.js" ]; then
+          echo "❌ vitest.config.js not found"
+          exit 1
+        fi
+        echo "✓ vitest.config.js exists"
+        
+        # Check for test file
+        if [ ! -f "tests/example.test.js" ]; then
+          echo "❌ tests/example.test.js not found"
+          exit 1
+        fi
+        echo "✓ tests/example.test.js exists"
+        
+        # Check for .env file
+        if [ ! -f ".env" ]; then
+          echo "❌ .env not found"
+          exit 1
+        fi
+        echo "✓ .env exists"
+        
+        # Check for .gitignore
+        if [ ! -f ".gitignore" ]; then
+          echo "❌ .gitignore not found"
+          exit 1
+        fi
+        echo "✓ .gitignore exists"
+        
+        # Check for GitHub workflow
+        if [ ! -f ".github/workflows/testdriver.yml" ]; then
+          echo "❌ .github/workflows/testdriver.yml not found"
+          exit 1
+        fi
+        echo "✓ .github/workflows/testdriver.yml exists"
+
+    - name: Verify vitest config contents
+      if: github.event_name == 'pull_request'
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking vitest.config.js contents..."
+        
+        # Check for TestDriver reporter
+        if ! grep -q "TestDriver()" vitest.config.js; then
+          echo "❌ TestDriver reporter not found in vitest.config.js"
+          cat vitest.config.js
+          exit 1
+        fi
+        echo "✓ TestDriver reporter is configured"
+        
+        # Check for setupFiles
+        if ! grep -q "setupFiles.*testdriverai/vitest/setup" vitest.config.js; then
+          echo "❌ setupFiles not configured correctly"
+          cat vitest.config.js
+          exit 1
+        fi
+        echo "✓ setupFiles is configured"
+
+    - name: Verify test file contents
+      if: github.event_name == 'pull_request'
+      working-directory: /tmp/test-init-project
+      run: |
+        echo "Checking test file contents..."
+        
+        # Check for .provision usage
+        if ! grep -q "\.provision\.chrome" tests/example.test.js; then
+          echo "❌ Test does not use .provision.chrome"
+          cat tests/example.test.js
+          exit 1
+        fi
+        echo "✓ Test uses .provision.chrome"
+        
+        # Check for TestDriver import
+        if ! grep -q "from 'testdriverai/vitest/hooks'" tests/example.test.js; then
+          echo "❌ Test does not import from testdriverai/vitest/hooks"
+          cat tests/example.test.js
+          exit 1
+        fi
+        echo "✓ Test imports TestDriver from vitest/hooks"
+
+    - name: Run the generated test
+      if: github.event_name == 'pull_request'
+      working-directory: /tmp/test-init-project
+      run: npm test
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+
+    - name: Upload init test results
+      if: always() && github.event_name == 'pull_request'
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-init-results
+        path: /tmp/test-init-project/test-results/
+        retention-days: 7
+        if-no-files-found: warn

package/.testdriver/last-sandbox

@@ -0,0 +1,7 @@
+{
+  "sandboxId": "i-034317335b2eb9678",
+  "os": "windows",
+  "ami": null,
+  "instanceType": null,
+  "timestamp": "2025-12-22T23:09:20.319Z"
+}

package/agent/events.js

@@ -99,6 +99,7 @@ const events = {
     disconnect: "sandbox:disconnected",
     sent: "sandbox:sent",
     received: "sandbox:received",
+    progress: "sandbox:progress",
   },
   redraw: {
     status: "redraw:status",

package/agent/index.js

@@ -17,7 +17,6 @@ const diff = require("diff");
 
 // global utilities
 const generator = require("./lib/generator.js");
-const promptCache = require("./lib/cache.js");
 const theme = require("./lib/theme.js");
 const SourceMapper = require("./lib/source-mapper.js");
 
@@ -110,6 +109,10 @@ class TestDriverAgent extends EventEmitter2 {
     // Create sandbox instance with this agent's emitter, analytics, and session
     this.sandbox = createSandbox(this.emitter, this.analytics, this.session);
 
+    // Attach Sentry log listeners to capture CLI logs as breadcrumbs
+      const sentry = require("../lib/sentry");
+      sentry.attachLogListeners(this.emitter);
+
     // Set the OS for the sandbox to use
     this.sandbox.os = this.sandboxOs;
 
@@ -191,6 +194,15 @@ class TestDriverAgent extends EventEmitter2 {
       this.redraw.cleanup();
     }
 
+    // Close sandbox connection to release the connection slot
+    if (this.sandbox) {
+      try {
+        this.sandbox.close();
+      } catch (err) {
+        // Ignore sandbox close errors during exit
+      }
+    }
+
     shouldRunPostrun =
       !this.hasRunPostrun &&
       (shouldRunPostrun || this.cliArgs?.command == "run");
@@ -356,7 +368,7 @@ class TestDriverAgent extends EventEmitter2 {
           image,
         },
         (chunk) => {
-          if (chunk.type === "data") {
+          if (chunk.type === "data" && chunk.data) {
             this.emitter.emit(events.log.markdown.chunk, streamId, chunk.data);
           }
         },
@@ -420,9 +432,6 @@ class TestDriverAgent extends EventEmitter2 {
     let mousePosition = await this.system.getMousePosition();
     let activeWindow = await this.system.activeWin();
 
-    const streamId = `check-${Date.now()}`;
-    this.emitter.emit(events.log.markdown.start, streamId);
-
     let response = await this.sdk.req(
       "check",
       {
@@ -430,15 +439,10 @@ class TestDriverAgent extends EventEmitter2 {
         images,
         mousePosition,
         activeWindow,
-      },
-      (chunk) => {
-        if (chunk.type === "data") {
-          this.emitter.emit(events.log.markdown.chunk, streamId, chunk.data);
-        }
-      },
+      }
     );
 
-    this.emitter.emit(events.log.markdown.end, streamId);
+    this.emitter.emit(events.log.markdown.static, response.data);
 
     this.lastScreenshot = thisScreenshot;
 
@@ -869,8 +873,7 @@ commands:
     currentTask,
     dry = false,
     validateAndLoop = false,
-    shouldSave = true,
-    useCache = true,
+    shouldSave = true
   ) {
     // Check if execution has been stopped
     if (this.stopped) {
@@ -889,56 +892,10 @@ commands:
 
     this.tasks.push(currentTask);
 
-    // Check cache first (if enabled via parameter)
-    const cachedYaml = useCache ? promptCache.readCache(currentTask) : null;
-
-    if (cachedYaml) {
-      // Cache hit - load and execute the cached YAML file
-      this.emitter.emit(
-        events.log.debug,
-        `Using cached response for prompt: "${currentTask}"`,
-      );
-      this.emitter.emit(events.log.log, theme.dim("(using cached response)"));
-
-      try {
-        // Load the YAML using hydrateFromYML
-        const parsed = await generator.hydrateFromYML(
-          cachedYaml,
-          this.sessionInstance,
-        );
-
-        // Execute the commands from the first step
-        if (parsed.steps && parsed.steps.length > 0) {
-          const step = parsed.steps[0];
-          if (step.commands) {
-            await this.executeCommands(
-              step.commands,
-              0,
-              false,
-              dry,
-              shouldSave,
-            );
-          }
-        }
-      } catch (err) {
-        this.emitter.emit(
-          events.log.debug,
-          `Error loading cached YAML: ${err.message}, falling back to API`,
-        );
-        // Fall through to make API call if cache is invalid
-      }
-
-      return;
-    }
-
-    // Cache miss - call the API
     this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     this.lastScreenshot = await this.system.captureScreenBase64();
 
-    const streamId = `input-${Date.now()}`;
-    this.emitter.emit(events.log.markdown.start, streamId);
-
     let message = await this.sdk.req(
       "input",
       {
@@ -946,59 +903,12 @@ commands:
         mousePosition: await this.system.getMousePosition(),
         activeWindow: await this.system.activeWin(),
         image: this.lastScreenshot,
-      },
-      (chunk) => {
-        if (chunk.type === "data") {
-          this.emitter.emit(events.log.markdown.chunk, streamId, chunk.data);
-        }
-      },
+      }
     );
 
-    this.emitter.emit(events.log.markdown.end, streamId);
+    this.emitter.emit(events.log.log, message.data);
 
     if (message && message.data) {
-      // Save the YAML to cache (if enabled)
-      if (useCache) {
-        try {
-          // Extract YAML code blocks from the markdown response
-          const codeblocks = await this.parser.findCodeBlocks(message.data);
-          if (codeblocks && codeblocks.length > 0) {
-            // Parse commands from all code blocks
-            const allCommands = [];
-            for (const block of codeblocks) {
-              const commands = await this.parser.getCommands(block);
-              allCommands.push(...commands);
-            }
-
-            // Create a proper step with prompt
-            const step = {
-              prompt: currentTask,
-              commands: allCommands,
-            };
-
-            // Use dumpToYML to create a valid testdriver yaml file
-            const yamlContent = await generator.dumpToYML(
-              [step],
-              this.sessionInstance,
-            );
-
-            const cachePath = promptCache.writeCache(currentTask, yamlContent);
-            if (cachePath) {
-              this.emitter.emit(
-                events.log.debug,
-                `Cached YAML saved to: ${cachePath}`,
-              );
-            }
-          }
-        } catch (err) {
-          // If we can't extract YAML, just skip caching
-          this.emitter.emit(
-            events.log.debug,
-            `Could not cache response: ${err.message}`,
-          );
-        }
-      }
-
       await this.aiExecute(message.data, validateAndLoop, dry, shouldSave);
       this.emitter.emit(
         events.log.debug,
@@ -1709,49 +1619,35 @@ ${regression}
     this.emitter.emit(events.log.log, `${inputFile} (end)`);
   }
 
-  // Returns sandboxId to use (either from file if recent, or null)
-  getRecentSandboxId() {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+  // Returns the path to the last sandbox file
+  getLastSandboxFilePath() {
+    const testdriverDir = path.join(process.cwd(), '.testdriver');
+    return path.join(testdriverDir, 'last-sandbox');
+  }
+
+  // Returns full sandbox info from last-sandbox file (no timeout - let API validate)
+  getLastSandboxId() {
+    const lastSandboxFile = this.getLastSandboxFilePath();
 
     if (fs.existsSync(lastSandboxFile)) {
       try {
-        const stats = fs.statSync(lastSandboxFile);
-        const mtime = new Date(stats.mtime);
-        const now = new Date();
-        const diffMinutes = (now - mtime) / (1000 * 60);
-        if (diffMinutes < 10) {
-          const fileContent = fs.readFileSync(lastSandboxFile, "utf-8").trim();
-
-          // Parse sandbox info (supports both old format and new format)
-          let sandboxInfo;
-          try {
-            sandboxInfo = JSON.parse(fileContent);
-          } catch {
-            return fileContent || null;
-          }
+        const fileContent = fs.readFileSync(lastSandboxFile, "utf-8").trim();
 
-          // Check if AMI and instance type match current requirements
-          const currentAmi = this.sandboxAmi || null;
-          const currentInstance = this.sandboxInstance || null;
-          const storedAmi = sandboxInfo.ami || null;
-          const storedInstance = sandboxInfo.instanceType || null;
-
-          if (currentAmi === storedAmi && currentInstance === storedInstance) {
-            // Return sandboxId (new format) or instanceId (old format for backwards compatibility)
-            return sandboxInfo.sandboxId || sandboxInfo.instanceId;
-          } else {
-            this.emitter.emit(
-              events.log.log,
-              theme.dim(
-                "Recent sandbox found but AMI/instance type doesn't match current requirements",
-              ),
-            );
-            return null;
-          }
+        // Parse sandbox info (supports both old format and new format)
+        let sandboxInfo;
+        try {
+          sandboxInfo = JSON.parse(fileContent);
+        } catch {
+          return { sandboxId: fileContent || null };
         }
+
+        return {
+          sandboxId: sandboxInfo.sandboxId || sandboxInfo.instanceId || null,
+          os: sandboxInfo.os || 'linux',
+          ami: sandboxInfo.ami || null,
+          instanceType: sandboxInfo.instanceType || null,
+          timestamp: sandboxInfo.timestamp || null,
+        };
       } catch {
         // ignore errors
       }
@@ -1759,12 +1655,43 @@ ${regression}
     return null;
   }
 
+  // Returns sandboxId to use if AMI/instance type match current requirements
+  getRecentSandboxId() {
+    const sandboxInfo = this.getLastSandboxId();
+    
+    if (!sandboxInfo || !sandboxInfo.sandboxId) {
+      return null;
+    }
+
+    // Check if AMI and instance type match current requirements
+    const currentAmi = this.sandboxAmi || null;
+    const currentInstance = this.sandboxInstance || null;
+    const storedAmi = sandboxInfo.ami || null;
+    const storedInstance = sandboxInfo.instanceType || null;
+
+    if (currentAmi === storedAmi && currentInstance === storedInstance) {
+      return sandboxInfo.sandboxId;
+    } else {
+      this.emitter.emit(
+        events.log.log,
+        theme.dim(
+          "Recent sandbox found but AMI/instance type doesn't match current requirements",
+        ),
+      );
+      return null;
+    }
+  }
+
   saveLastSandboxId(sandboxId, osType = "linux") {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+    const lastSandboxFile = this.getLastSandboxFilePath();
+    const testdriverDir = path.dirname(lastSandboxFile);
+    
     try {
+      // Ensure .testdriver directory exists
+      if (!fs.existsSync(testdriverDir)) {
+        fs.mkdirSync(testdriverDir, { recursive: true });
+      }
+      
       const sandboxInfo = {
         sandboxId: sandboxId,
         os: osType,
@@ -1772,7 +1699,7 @@ ${regression}
         instanceType: this.sandboxInstance || null,
         timestamp: new Date().toISOString(),
       };
-      fs.writeFileSync(lastSandboxFile, JSON.stringify(sandboxInfo), {
+      fs.writeFileSync(lastSandboxFile, JSON.stringify(sandboxInfo, null, 2), {
         encoding: "utf-8",
       });
     } catch {
@@ -1781,10 +1708,7 @@ ${regression}
   }
 
   clearRecentSandboxId() {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+    const lastSandboxFile = this.getLastSandboxFilePath();
     try {
       if (fs.existsSync(lastSandboxFile)) {
         fs.unlinkSync(lastSandboxFile);
@@ -1793,6 +1717,7 @@ ${regression}
       // ignore errors
     }
   }
+
   async buildEnv(options = {}) {
     // If instance already exists, do not build environment again
     if (this.instance) {
@@ -1875,6 +1800,7 @@ ${regression}
         let instance = await this.connectToSandboxDirect(
           this.sandboxId,
           true, // always persist by default
+          this.keepAlive, // pass keepAlive TTL
         );
 
         this.instance = instance;
@@ -1889,11 +1815,6 @@ ${regression}
         );
         console.error("Failed to reconnect to sandbox:", error);
       }
-    } else if (!createNew && !recentId) {
-      this.emitter.emit(
-        events.log.narration,
-        theme.dim(`no recent sandbox found, creating a new one.`),
-      );
     } else if (!createNew && this.sandboxId && !this.config.CI) {
       // Only attempt to connect to existing sandbox if not in CI mode and not creating new
       // Attempt to connect to known instance
@@ -1906,6 +1827,7 @@ ${regression}
         let instance = await this.connectToSandboxDirect(
           this.sandboxId,
           true, // always persist by default
+          this.keepAlive, // pass keepAlive TTL
         );
 
         this.instance = instance;
@@ -1948,6 +1870,7 @@ ${regression}
       let instance = await this.connectToSandboxDirect(
         this.sandboxId,
         true, // always persist by default
+        this.keepAlive, // pass keepAlive TTL
       );
       this.instance = instance;
       await this.renderSandbox(instance, headless);
@@ -2137,10 +2060,10 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     }
   }
 
-  async connectToSandboxDirect(sandboxId, persist = false) {
+  async connectToSandboxDirect(sandboxId, persist = false, keepAlive = null) {
     const { formatter } = require("../sdk-log-formatter.js");
     this.emitter.emit(events.log.narration, formatter.getPrefix("connect") + " " + theme.green.bold("Connecting") + " " + theme.cyan(`to sandbox...`));
-    let reply = await this.sandbox.connect(sandboxId, persist);
+    let reply = await this.sandbox.connect(sandboxId, persist, keepAlive);
 
     // reply includes { success, url, sandbox: {...} }
     // For renderSandbox, we need the sandbox object with url merged in
@@ -2169,6 +2092,10 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     if (this.sandboxInstance) {
       sandboxConfig.instanceType = this.sandboxInstance;
     }
+    // Add keepAlive TTL if specified
+    if (this.keepAlive !== undefined && this.keepAlive !== null) {
+      sandboxConfig.keepAlive = this.keepAlive;
+    }
 
     let instance = await this.sandbox.send(sandboxConfig, 60000 * 8);
 
@@ -2200,6 +2127,15 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     }
 
     this.session.set(sessionRes.data.id);
+    
+    // Set Sentry session trace context for distributed tracing
+    // This links CLI errors/logs to the same trace as API calls
+    try {
+      const sentry = require("../lib/sentry");
+      sentry.setSessionTraceContext(sessionRes.data.id);
+    } catch (e) {
+      // Sentry module may not be available, ignore
+    }
   }
 
   // Helper method to find testdriver directory by traversing up from a file path

package/agent/lib/sandbox.js

@@ -155,11 +155,12 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
       }
     }
 
-    async connect(sandboxId, persist = false) {
+    async connect(sandboxId, persist = false, keepAlive = null) {
       let reply = await this.send({
         type: "connect",
         persist,
         sandboxId,
+        keepAlive,
       });
 
       if (reply.success) {
@@ -227,6 +228,15 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
         this.socket.on("message", async (raw) => {
           let message = JSON.parse(raw);
 
+          // Handle progress messages (no requestId needed)
+          if (message.type === 'sandbox.progress') {
+            emitter.emit(events.sandbox.progress, {
+              step: message.step,
+              message: message.message,
+            });
+            return;
+          }
+
           if (!this.ps[message.requestId]) {
             console.warn(
               "No pending promise found for requestId:",