testdriverai 7.10.0-canary.0 → 7.10.1-test

package/.claude-plugin/marketplace.json

@@ -0,0 +1,17 @@
+{
+  "name": "testdriver",
+  "version": "7.9.54-test",
+  "description": "TestDriver.ai plugins for Claude Code — MCP server, agent, and skills for building natural-language end-to-end tests.",
+  "repository": "https://github.com/testdriverai/testdriverai",
+  "owner": {
+    "name": "TestDriver.ai",
+    "url": "https://testdriver.ai"
+  },
+  "plugins": [
+    {
+      "name": "testdriver",
+      "source": "./ai",
+      "description": "TestDriver.ai MCP server, TestDriver expert agent, and skills for authoring computer-use end-to-end tests with Vitest."
+    }
+  ]
+}

package/agent/index.js

@@ -494,65 +494,89 @@ class TestDriverAgent extends EventEmitter2 {
       );
     }
 
-    try {
-      let response;
-
-      // "run" and "if" commands are special meta commands
-      // that change the flow of execution
-      if (command.command == "run") {
-        response = await this.embed(command.file, depth, pushToHistory);
-      } else if (command.command == "if") {
-        response = await this.iffy(
-          command.condition,
-          command.then,
-          command.else,
-          depth,
-        );
-      } else {
-        response = await this.commander.run(command, depth);
-      }
+    const sentry = require("../lib/sentry");
+    return sentry.withSpan(
+      {
+        name: `command.${commandName}`,
+        op: "command",
+        attributes: { "command.name": commandName, "command.depth": depth },
+      },
+      async (span) => {
+        try {
+          let response;
+
+          // "run" and "if" commands are special meta commands
+          // that change the flow of execution
+          if (command.command == "run") {
+            response = await this.embed(command.file, depth, pushToHistory);
+          } else if (command.command == "if") {
+            response = await this.iffy(
+              command.condition,
+              command.then,
+              command.else,
+              depth,
+            );
+          } else {
+            response = await this.commander.run(command, depth);
+          }
 
-      const endTime = Date.now();
-      const duration = endTime - startTime;
+          const endTime = Date.now();
+          const duration = endTime - startTime;
 
-      // Emit command success event with source mapping
-      this.emitter.emit(events.command.success, {
-        command: commandName,
-        depth,
-        data: command,
-        duration,
-        response,
-        timestamp: endTime,
-        sourcePosition: sourcePosition,
-      });
+          if (span) {
+            span.setStatus({ code: 1 }); // OK
+          }
 
-      // if the result of a command contains more commands, we perform the process again
-      if (response && typeof response === "string") {
-        return await this.actOnMarkdown(response, depth, false, false, false);
-      }
-    } catch (error) {
-      const endTime = Date.now();
-      const duration = endTime - startTime;
+          // Emit command success event with source mapping
+          this.emitter.emit(events.command.success, {
+            command: commandName,
+            depth,
+            data: command,
+            duration,
+            response,
+            timestamp: endTime,
+            sourcePosition: sourcePosition,
+          });
 
-      // Emit command error event with source mapping
-      this.emitter.emit(events.command.error, {
-        command: commandName,
-        depth,
-        data: command,
-        error: error.message,
-        duration,
-        timestamp: endTime,
-        sourcePosition: sourcePosition,
-      });
+          // if the result of a command contains more commands, we perform the process again
+          if (response && typeof response === "string") {
+            return await this.actOnMarkdown(
+              response,
+              depth,
+              false,
+              false,
+              false,
+            );
+          }
+        } catch (error) {
+          const endTime = Date.now();
+          const duration = endTime - startTime;
 
-      return await this.haveAIResolveError(
-        error,
-        yaml.dump({ commands: [yml] }),
-        depth,
-        true,
-        shouldSave,
-      );
-    }
+          if (span) {
+            span.setStatus({ code: 2, message: error.message });
+          }
+
+          // Emit command error event with source mapping
+          this.emitter.emit(events.command.error, {
+            command: commandName,
+            depth,
+            data: command,
+            error: error.message,
+            duration,
+            timestamp: endTime,
+            sourcePosition: sourcePosition,
+          });
+
+          return await this.haveAIResolveError(
+            error,
+            yaml.dump({ commands: [yml] }),
+            depth,
+            true,
+            shouldSave,
+          );
+        }
+      },
+    );
   }
 
   async executeCommands(
@@ -1968,29 +1992,50 @@ ${regression}
   // Write session file for IDE preview (VSCode extension watches for these)
   writeIdeSessionFile(debuggerUrl, data) {
     const fs = require("fs");
+    const os = require("os");
     const path = require("path");
 
-    const sessionId = `${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
-    const previewsDir = path.join(process.cwd(), ".testdriver", ".previews");
+    // Base dir for the preview file. `process.cwd()` is read-only in some hosts
+    // (e.g. Vercel/Lambda serverless, where cwd is /var/task), which makes the
+    // mkdir below throw. Allow an explicit override and otherwise fall back to a
+    // writable temp dir, so a preview write never blocks the session.
+    const baseDir =
+      process.env.TD_PREVIEWS_DIR ||
+      (this.config && this.config.TD_PREVIEWS_DIR) ||
+      process.cwd();
+    const previewsDir = path.join(baseDir, ".testdriver", ".previews");
 
-    // Create the previews directory if it doesn't exist
-    if (!fs.existsSync(previewsDir)) {
-      fs.mkdirSync(previewsDir, { recursive: true });
-    }
+    const sessionId = `${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
 
-    const sessionData = {
-      sessionId,
-      debuggerUrl,
-      resolution: Array.isArray(data.resolution) ? data.resolution : (data.resolution ? data.resolution.split("x").map(Number) : [1920, 1080]),
-      testFile: data.testFile || this.testFile || null,
-      os: data.os || this.sandboxOs || "linux",
-      timestamp: Date.now(),
-    };
+    // The IDE preview file is a best-effort convenience (the VSCode extension
+    // watches for it). It must never break session start — if the target dir
+    // isn't writable, fall back to os.tmpdir(), and if that also fails, log and
+    // move on rather than throwing out of the provisioning path.
+    try {
+      const sessionData = {
+        sessionId,
+        debuggerUrl,
+        resolution: Array.isArray(data.resolution) ? data.resolution : (data.resolution ? data.resolution.split("x").map(Number) : [1920, 1080]),
+        testFile: data.testFile || this.testFile || null,
+        os: data.os || this.sandboxOs || "linux",
+        timestamp: Date.now(),
+      };
 
-    const filePath = path.join(previewsDir, `${sessionId}.json`);
-    fs.writeFileSync(filePath, JSON.stringify(sessionData, null, 2));
+      let dir = previewsDir;
+      try {
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+      } catch {
+        // cwd/base wasn't writable — retry under the OS temp dir.
+        dir = path.join(os.tmpdir(), ".testdriver", ".previews");
+        fs.mkdirSync(dir, { recursive: true });
+      }
 
-    logger.log(`IDE preview session written to ${filePath}`);
+      const filePath = path.join(dir, `${sessionId}.json`);
+      fs.writeFileSync(filePath, JSON.stringify(sessionData, null, 2));
+      logger.log(`IDE preview session written to ${filePath}`);
+    } catch (error) {
+      logger.log(`Skipped IDE preview session file (non-fatal): ${error.message}`);
+    }
   }
 
   // Find the VS Code instance that contains the test file

package/agent/interface.js

@@ -10,7 +10,19 @@ function createCommandDefinitions(agent) {
     init: {
       description: "Initialize a new TestDriver project with Vitest SDK examples",
       args: {},
-      flags: {},
+      flags: {
+        client: Flags.string({
+          description:
+            "AI client(s) to install into (comma-separated, or 'all'). e.g. --client claude-code,cursor. Omit for an interactive picker.",
+          multiple: false,
+        }),
+        "sample-test": Flags.boolean({
+          description:
+            "Scaffold the example test files (tests/example.test.js + tests/login.js). Use --no-sample-test to skip them (e.g. when an agent writes its own tests).",
+          default: true,
+          allowNo: true,
+        }),
+      },
       handler: async () => {
         // This handler is special - it doesn't need an agent instance
         // It just scaffolds files, so it will be handled by the CLI command

package/agent/lib/commands.js

@@ -302,10 +302,12 @@ const createCommands = (
       `🔍 assert() threshold: ${threshold} (cache ${threshold < 0 ? "DISABLED" : "ENABLED"}${cacheKey ? `, cacheKey: ${cacheKey.substring(0, 8)}...` : ""})`,
     );
     
-    // Use v7 endpoint for assert with caching support
+    // Use v7 endpoint for assert with caching support.
+    // captureScreenImage returns { imageKey } (fast S3-key path) or { image }
+    // (base64 fallback) — see system.captureScreenImage.
     let response = await sdk.req("assert", {
       expect: assertion,
-      image: await system.captureScreenBase64(),
+      ...(await system.captureScreenImage()),
       threshold,
       cacheKey,
       os,
@@ -815,7 +817,7 @@ const createCommands = (
 
       let response = await sdk.req("find", {
         element: description,
-        image: await system.captureScreenBase64(),
+        ...(await system.captureScreenImage()),
       });
 
       if (!response || !response.coordinates) {
@@ -855,7 +857,7 @@ const createCommands = (
 
       let response = await sdk.req("find", {
         element: description,
-        image: await system.captureScreenBase64(),
+        ...(await system.captureScreenImage()),
       });
 
       if (!response || !response.coordinates) {
@@ -1211,7 +1213,7 @@ const createCommands = (
       while (durationPassed < timeout && !passed) {
         const response = await sdk.req("find", {
           element: text,
-          image: await system.captureScreenBase64(),
+          ...(await system.captureScreenImage()),
         });
 
         passed = !!(response && response.coordinates);
@@ -1304,7 +1306,7 @@ const createCommands = (
       while (scrollDistance <= maxDistance && !passed) {
         const response = await sdk.req("find", {
           element: text,
-          image: await system.captureScreenBase64(),
+          ...(await system.captureScreenImage()),
         });
 
         passed = !!(response && response.coordinates);
@@ -1531,9 +1533,11 @@ const createCommands = (
       emitter.emit(events.log.narration, formatter.getPrefix("action") + " " + theme.cyan.bold("Exec") + " " + theme.magenta(`[${language}]`), true);
 
       // Log nested command details (truncate to first line)
-      const firstLine = code.split('\n')[0];
-      const codeDisplay = code.includes('\n') ? firstLine + '...' : firstLine;
-      emitter.emit(events.log.log, formatter.formatCodeLine(codeDisplay));
+      if (!silent) {
+        const firstLine = code.split('\n')[0];
+        const codeDisplay = code.includes('\n') ? firstLine + '...' : firstLine;
+        emitter.emit(events.log.log, formatter.formatCodeLine(codeDisplay));
+      }
 
       let plat = system.platform();
 
@@ -1565,11 +1569,26 @@ const createCommands = (
         let result = null;
 
         const execTimeout = timeout || 300000;
-        result = await sandbox.send({
-          type: "commands.run",
-          command: code,
-          timeout: execTimeout,
-        }, execTimeout);
+
+        // Stream output chunks in real-time as they arrive from the runner
+        let streamedOutput = false;
+        const onExecOutput = ({ chunk }) => {
+          if (!silent && chunk) {
+            emitter.emit(events.log.log, theme.dim(chunk), true);
+            streamedOutput = true;
+          }
+        };
+        emitter.on(events.exec.output, onExecOutput);
+
+        try {
+          result = await sandbox.send({
+            type: "commands.run",
+            command: code,
+            timeout: execTimeout,
+          }, execTimeout);
+        } finally {
+          emitter.off(events.exec.output, onExecOutput);
+        }
         
         const execActionEndTime = Date.now();
         const execDuration = execActionEndTime - execActionLogStart;
@@ -1595,7 +1614,8 @@ const createCommands = (
             true,
           );
           
-          if (!silent && result.out?.stdout) {
+          // Skip stdout log if already streamed in real-time to avoid duplication
+          if (!silent && !streamedOutput && result.out?.stdout) {
             emitter.emit(events.log.log, theme.dim(`  stdout:`), true);
             emitter.emit(events.log.log, theme.dim(`  ${result.out.stdout}`), true);
           }

package/agent/lib/provision-commands.js

@@ -13,6 +13,24 @@
 
 'use strict';
 
+/**
+ * Extract the major.minor portion of a semver-ish version string.
+ *
+ * Examples:
+ *   '7.9.59'         -> '7.9'
+ *   '7.9.59-test'    -> '7.9'
+ *   '7.10.0-canary.3'-> '7.10'
+ *   ''               -> ''
+ *
+ * @param {string} version
+ * @returns {string}
+ */
+function majorMinor(version) {
+  if (!version || typeof version !== 'string') return '';
+  var m = version.match(/^(\d+)\.(\d+)/);
+  return m ? m[1] + '.' + m[2] : '';
+}
+
 /**
  * Build the agent config object written to the sandbox.
  *
@@ -25,9 +43,10 @@
  * @param {string} [opts.sentryChannel]
  * @param {Object} opts.ablyToken - Ably token object
  * @param {string} opts.channelName - Ably channel name
+ * @param {string} [opts.sessionId] - Session ID for distributed tracing (traceId = MD5(sessionId))
  * @returns {Object} Agent config to serialize as JSON
  */
-function buildAgentConfig({ sandboxId, apiRoot, apiKey, sentryDsn, sentryEnvironment, sentryChannel, ablyToken, channelName }) {
+function buildAgentConfig({ sandboxId, apiRoot, apiKey, sentryDsn, sentryEnvironment, sentryChannel, ablyToken, channelName, sessionId }) {
   return {
     sandboxId,
     apiRoot,
@@ -35,6 +54,7 @@ function buildAgentConfig({ sandboxId, apiRoot, apiKey, sentryDsn, sentryEnviron
     sentryDsn: sentryDsn || undefined,
     sentryEnvironment: sentryEnvironment || 'production',
     sentryChannel: sentryChannel || undefined,
+    sessionId: sessionId || undefined,
     ably: {
       token: ablyToken,
       channel: channelName,
@@ -54,9 +74,10 @@ function buildAgentConfig({ sandboxId, apiRoot, apiKey, sentryDsn, sentryEnviron
  * @param {string} opts.configJson   - JSON.stringify'd agent config
  * @param {string} opts.sandboxId    - For logging
  * @param {string} [opts.s3DownloadUrl] - S3 pre-signed URL for dev/test (omit for npm install)
+ * @param {string} [opts.imageVersion] - Version baked into the AMI (skip npm install when it matches)
  * @returns {string[]} Array of PowerShell command strings
  */
-function windowsProvisionCommands({ channel, configJson, sandboxId, s3DownloadUrl }) {
+function windowsProvisionCommands({ channel, configJson, sandboxId, s3DownloadUrl, imageVersion }) {
   var useS3 = (channel === 'dev' || channel === 'test') && s3DownloadUrl;
   var commands = [];
 
@@ -92,11 +113,40 @@ function windowsProvisionCommands({ channel, configJson, sandboxId, s3DownloadUr
     // Canary/stable (or dev/test without S3 URL): npm install by dist-tag
     agentScript = 'node_modules/@testdriverai/runner/sandbox-agent.js';
     var runnerTag = channel === 'stable' ? 'latest' : channel;
-    commands.push(
-      "Write-Host 'Installing @testdriverai/runner@" + runnerTag + "...'",
-      'npm install @testdriverai/runner@' + runnerTag + ' --omit=dev 2>&1 | Write-Host',
-      "Write-Host 'Runner install complete'"
-    );
+
+    // If we know the image version, check whether the baked-in runner already
+    // matches — skip npm install entirely when it does (saves 5-15s per launch).
+    // Only the major.minor portion is compared (patch & prerelease suffix are
+    // ignored) so trivial patch bumps don't trigger an unnecessary npm install.
+    // Emits a structured RUNNER_VERSION_CHECK line so SSM output / runner logs
+    // capture the outcome for monitoring.
+    if (imageVersion) {
+      var expectedMinor = majorMinor(imageVersion);
+      commands.push(
+        "Write-Host 'Checking installed runner version...'",
+        "$installedVersion = ''",
+        "try { $pkg = Get-Content 'node_modules/@testdriverai/runner/package.json' -Raw | ConvertFrom-Json; $installedVersion = $pkg.version } catch {}",
+        "$installedMinor = if ($installedVersion) { ($installedVersion -split '[.-]')[0..1] -join '.' } else { '' }",
+        "if (('" + expectedMinor + "' -ne '') -and ($installedMinor -eq '" + expectedMinor + "')) {",
+        "  Write-Host \"Runner already at v$installedVersion (minor " + expectedMinor + "), skipping update\"",
+        "  Write-Host (\"RUNNER_VERSION_CHECK:\" + (ConvertTo-Json -Compress @{ action='skipped'; installedVersion=$installedVersion; expectedVersion='" + imageVersion + "'; expectedMinor='" + expectedMinor + "'; channel='" + channel + "'; sandboxId='" + sandboxId + "' }))",
+        "} else {",
+        "  Write-Host \"Installed minor: $installedMinor, expected: " + expectedMinor + " — updating...\"",
+        "  Write-Host 'Installing @testdriverai/runner@" + runnerTag + "...'",
+        '  npm install @testdriverai/runner@' + runnerTag + ' --omit=dev 2>&1 | Write-Host',
+        "  $newVersion = ''",
+        "  try { $newPkg = Get-Content 'node_modules/@testdriverai/runner/package.json' -Raw | ConvertFrom-Json; $newVersion = $newPkg.version } catch {}",
+        "  Write-Host (\"RUNNER_VERSION_CHECK:\" + (ConvertTo-Json -Compress @{ action='updated'; previousVersion=$installedVersion; expectedVersion='" + imageVersion + "'; expectedMinor='" + expectedMinor + "'; newVersion=$newVersion; channel='" + channel + "'; sandboxId='" + sandboxId + "' }))",
+        "  Write-Host 'Runner install complete'",
+        "}"
+      );
+    } else {
+      commands.push(
+        "Write-Host 'Installing @testdriverai/runner@" + runnerTag + "...'",
+        'npm install @testdriverai/runner@' + runnerTag + ' --omit=dev 2>&1 | Write-Host',
+        "Write-Host 'Runner install complete'"
+      );
+    }
   }
 
   // ── 3. Regenerate run_testdriver.ps1 ──────────────────────────────
@@ -138,16 +188,23 @@ function windowsProvisionCommands({ channel, configJson, sandboxId, s3DownloadUr
 /**
  * Generate the bash command to install/update the runner on Linux (E2B).
  *
+ * When `imageVersion` is provided with the non-S3 (npm) path, the generated
+ * command checks the installed version first and skips the install when it
+ * matches — mirroring the Windows version-check behaviour.
+ *
  * @param {Object} opts
- * @param {string} opts.channel        - Release channel
- * @param {string} [opts.s3DownloadUrl] - S3 pre-signed URL for dev/test
- * @param {string} [opts.runnerPath]   - Default '/opt/testdriver-runner'
+ * @param {string} opts.channel          - Release channel
+ * @param {string} [opts.s3DownloadUrl]  - S3 pre-signed URL for dev/test
+ * @param {string} [opts.runnerPath]     - Default '/opt/testdriver-runner'
+ * @param {string} [opts.imageVersion]   - Version baked into the E2B template (skip install when it matches)
+ * @param {string} [opts.sandboxId]      - For structured logging
  * @returns {string} Single bash command (steps joined with &&)
  */
-function linuxRunnerInstallCommand({ channel, s3DownloadUrl, runnerPath }) {
+function linuxRunnerInstallCommand({ channel, s3DownloadUrl, runnerPath, imageVersion, sandboxId }) {
   var rp = runnerPath || '/opt/testdriver-runner';
   var useS3 = (channel === 'dev' || channel === 'test') && s3DownloadUrl;
   var runnerTag = channel === 'stable' ? 'latest' : channel;
+  var sid = sandboxId || 'unknown';
 
   if (useS3) {
     return [
@@ -162,6 +219,29 @@ function linuxRunnerInstallCommand({ channel, s3DownloadUrl, runnerPath }) {
     ].join(' && ');
   }
 
+  // If we know the image version, wrap the install in a version check.
+  // Only the major.minor portion is compared (patch & prerelease suffix are
+  // ignored) so trivial patch bumps don't trigger an unnecessary npm install.
+  if (imageVersion) {
+    var expectedMinor = majorMinor(imageVersion);
+    return [
+      'INSTALLED_VERSION=$(node -p "try{require(\'' + rp + '/package.json\').version}catch(e){\'\'}" 2>/dev/null || echo "")',
+      'INSTALLED_MINOR=$(echo "$INSTALLED_VERSION" | sed -E "s/^([0-9]+\\.[0-9]+).*/\\1/")',
+      'if [ -n "' + expectedMinor + '" ] && [ "$INSTALLED_MINOR" = "' + expectedMinor + '" ]; then',
+      '  echo "RUNNER_VERSION_CHECK:{\\"action\\":\\"skipped\\",\\"installedVersion\\":\\"$INSTALLED_VERSION\\",\\"expectedVersion\\":\\"' + imageVersion + '\\",\\"expectedMinor\\":\\"' + expectedMinor + '\\",\\"channel\\":\\"' + channel + '\\",\\"sandboxId\\":\\"' + sid + '\\"}"',
+      '  echo "Runner already at v$INSTALLED_VERSION (minor ' + expectedMinor + '), skipping update"',
+      'else',
+      '  echo "Installed minor: $INSTALLED_MINOR, expected: ' + expectedMinor + ' — updating..."',
+      '  sudo npm install -g @testdriverai/runner@' + runnerTag + ' --omit=dev --no-audit --no-fund --loglevel=error',
+      '  sudo rm -rf ' + rp,
+      '  sudo ln -sf $(npm root -g)/@testdriverai/runner ' + rp,
+      '  NEW_VERSION=$(node -p "try{require(\'' + rp + '/package.json\').version}catch(e){\'\'}" 2>/dev/null || echo "")',
+      '  echo "RUNNER_VERSION_CHECK:{\\"action\\":\\"updated\\",\\"previousVersion\\":\\"$INSTALLED_VERSION\\",\\"expectedVersion\\":\\"' + imageVersion + '\\",\\"expectedMinor\\":\\"' + expectedMinor + '\\",\\"newVersion\\":\\"$NEW_VERSION\\",\\"channel\\":\\"' + channel + '\\",\\"sandboxId\\":\\"' + sid + '\\"}"',
+      '  echo "Runner install complete"',
+      'fi',
+    ].join(' && ');
+  }
+
   return [
     'sudo npm install -g @testdriverai/runner@' + runnerTag + ' --omit=dev --no-audit --no-fund --loglevel=error',
     'sudo rm -rf ' + rp,
@@ -173,4 +253,5 @@ module.exports = {
   buildAgentConfig,
   windowsProvisionCommands,
   linuxRunnerInstallCommand,
+  majorMinor,
 };