testdriverai 6.2.2 → 7.1.0

package/{testdriver → _testdriver}/examples/web/lifecycle/prerun.yaml

@@ -4,7 +4,12 @@ steps:
     commands:
       - command: exec
         lang: pwsh
-        code: dashcam track --name=TestDriver --type=application --pattern="C:\Users\testdriver\Documents\testdriver.log"
+        code: |
+          npm install dashcam@beta -g
+          npm list -g dashcam
+      - command: exec
+        lang: pwsh
+        code: dashcam track --name=TestDriver --type=app --pattern="C:\Users\testdriver\Documents\testdriver.log"
       - command: exec
         lang: pwsh
         code: dashcam start

package/{testdriver → _testdriver}/lifecycle/postrun.yaml

@@ -3,5 +3,6 @@ steps:
   - prompt: stop dashcam
     commands:
       - command: exec
-        lang: pwsh
-        code: dashcam -t 'Web Test Recording' -p
+        lang: sh
+        timeout: 120000
+        code: dashcam stop

package/_testdriver/lifecycle/prerun.yaml

@@ -0,0 +1,15 @@
+version: 6.0.0
+steps:
+  - prompt: dashcam
+    commands:
+      - command: exec
+        lang: sh
+        code: |
+          dashcam auth 4e93d8bf-3886-4d26-a144-116c4063522d
+          dashcam logs --add --name="TestDriver" --type=file --file="/tmp/testdriver.log"
+      - command: exec
+        lang: sh
+        code: dashcam record > /dev/null 2>&1 &
+      - command: exec
+        lang: sh
+        code: dashcam logs --add --name=TestDriver --type=file --file="/tmp/testdriver.log"

package/{testdriver → _testdriver}/lifecycle/provision.yaml

@@ -3,6 +3,11 @@ session: 67f00511acbd9ccac373edf7
 steps:
   - prompt: launch chrome
     commands:
+      - command: exec
+        lang: sh
+        code: |
+          npm install dashcam@beta -g
+          npm list -g dashcam
       - command: exec
         lang: sh
         code: |
@@ -13,8 +18,8 @@ steps:
             --no-default-browser-check \
             --no-first-run \
             --guest \
-            "${TD_WEBSITE}" \
+            "https://testdriver-sandbox.vercel.app/" \
             >/dev/null 2>&1 &
       - command: wait-for-text
-        text: ${TD_WEBSITE}
+        text: testdriver-sandbox
         timeout: 60000

package/agent/index.js

@@ -19,6 +19,7 @@ const diff = require("diff");
 
 // global utilities
 const generator = require("./lib/generator.js");
+const promptCache = require("./lib/cache.js");
 const theme = require("./lib/theme.js");
 const SourceMapper = require("./lib/source-mapper.js");
 
@@ -74,6 +75,7 @@ class TestDriverAgent extends EventEmitter2 {
     this.sandboxId = flags["sandbox-id"] || null;
     this.sandboxAmi = flags["sandbox-ami"] || null;
     this.sandboxInstance = flags["sandbox-instance"] || null;
+    this.sandboxOs = flags.os || "linux";
     this.ip = flags.ip || null;
     this.workingDir = flags.workingDir || process.cwd();
 
@@ -101,14 +103,17 @@ class TestDriverAgent extends EventEmitter2 {
     // Create outputs instance for this agent
     this.outputs = createOutputs();
 
-    // Create SDK instance with this agent's emitter, config, and session
+    // Create SDK instance with this agent's emitter, config, session, and abort signal
     this.sdk = createSDK(this.emitter, this.config, this.session);
 
     // Create analytics instance with this agent's emitter, config, and session
     this.analytics = createAnalytics(this.emitter, this.config, this.session);
 
-    // Create sandbox instance with this agent's emitter and analytics
-    this.sandbox = createSandbox(this.emitter, this.analytics);
+    // Create sandbox instance with this agent's emitter, analytics, and session
+    this.sandbox = createSandbox(this.emitter, this.analytics, this.session);
+
+    // Set the OS for the sandbox to use
+    this.sandbox.os = this.sandboxOs;
 
     // Create system instance with emitter, sandbox and config
     this.system = createSystem(this.emitter, this.sandbox, this.config);
@@ -121,6 +126,8 @@ class TestDriverAgent extends EventEmitter2 {
       this.config,
       this.session,
       () => this.sourceMapper.currentFilePath || this.thisFile,
+      this.cliArgs.options.redrawThreshold,
+      null, // getDashcamElapsedTime - will be set by SDK when dashcam is available
     );
     this.commands = commandsResult.commands;
     this.redraw = commandsResult.redraw;
@@ -272,6 +279,15 @@ class TestDriverAgent extends EventEmitter2 {
     // Get error message
     let eMessage = error.message ? error.message : error;
 
+    // Truncate error message if too long to prevent 400 errors from API
+    // Keep first 5000 characters as a reasonable limit for API payloads
+    const MAX_ERROR_LENGTH = 5000;
+    if (typeof eMessage === "string" && eMessage.length > MAX_ERROR_LENGTH) {
+      eMessage =
+        eMessage.substring(0, MAX_ERROR_LENGTH) +
+        "\n\n[Error message truncated - message was too long]";
+    }
+
     // we sanitize the error message to use it as a key in the errorCounts object
     let safeKey = JSON.stringify(error.message ? error.message : error);
     this.errorCounts[safeKey] = this.errorCounts[safeKey]
@@ -323,19 +339,40 @@ class TestDriverAgent extends EventEmitter2 {
     const streamId = `error-${Date.now()}`;
     this.emitter.emit(events.log.markdown.start, streamId);
 
-    let response = await this.sdk.req(
-      "error",
-      {
-        description: eMessage,
-        markdown,
-        image,
-      },
-      (chunk) => {
-        if (chunk.type === "data") {
-          this.emitter.emit(events.log.markdown.chunk, streamId, chunk.data);
-        }
-      },
-    );
+    // Truncate markdown if too long to prevent 400 errors
+    const MAX_MARKDOWN_LENGTH = 10000;
+    let truncatedMarkdown = markdown;
+    if (typeof markdown === "string" && markdown.length > MAX_MARKDOWN_LENGTH) {
+      truncatedMarkdown =
+        markdown.substring(0, MAX_MARKDOWN_LENGTH) +
+        "\n\n[Markdown truncated - content was too long]";
+    }
+
+    let response;
+    try {
+      response = await this.sdk.req(
+        "error",
+        {
+          description: eMessage,
+          markdown: truncatedMarkdown,
+          image,
+        },
+        (chunk) => {
+          if (chunk.type === "data") {
+            this.emitter.emit(events.log.markdown.chunk, streamId, chunk.data);
+          }
+        },
+      );
+    } catch (apiError) {
+      // If the error API call itself fails, prevent infinite loop
+      // by not retrying and instead treating as fatal
+      this.emitter.emit(
+        events.log.error,
+        theme.red(`Failed to get AI error resolution: ${apiError.message}`),
+      );
+      this.emitter.emit(events.log.log, "Original error: " + eMessage);
+      return await this.dieOnFatal(error);
+    }
 
     this.emitter.emit(events.log.markdown.end, streamId);
 
@@ -835,6 +872,7 @@ commands:
     dry = false,
     validateAndLoop = false,
     shouldSave = true,
+    useCache = true,
   ) {
     // Check if execution has been stopped
     if (this.stopped) {
@@ -853,6 +891,49 @@ commands:
 
     this.tasks.push(currentTask);
 
+    // Check cache first (if enabled via parameter)
+    const cachedYaml = useCache ? promptCache.readCache(currentTask) : null;
+
+    if (cachedYaml) {
+      // Cache hit - load and execute the cached YAML file
+      this.emitter.emit(
+        events.log.debug,
+        `Using cached response for prompt: "${currentTask}"`,
+      );
+      this.emitter.emit(events.log.log, theme.dim("(using cached response)"));
+
+      try {
+        // Load the YAML using hydrateFromYML
+        const parsed = await generator.hydrateFromYML(
+          cachedYaml,
+          this.sessionInstance,
+        );
+
+        // Execute the commands from the first step
+        if (parsed.steps && parsed.steps.length > 0) {
+          const step = parsed.steps[0];
+          if (step.commands) {
+            await this.executeCommands(
+              step.commands,
+              0,
+              false,
+              dry,
+              shouldSave,
+            );
+          }
+        }
+      } catch (err) {
+        this.emitter.emit(
+          events.log.debug,
+          `Error loading cached YAML: ${err.message}, falling back to API`,
+        );
+        // Fall through to make API call if cache is invalid
+      }
+
+      return;
+    }
+
+    // Cache miss - call the API
     this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     this.lastScreenshot = await this.system.captureScreenBase64();
@@ -877,7 +958,49 @@ commands:
 
     this.emitter.emit(events.log.markdown.end, streamId);
 
-    if (message) {
+    if (message && message.data) {
+      // Save the YAML to cache (if enabled)
+      if (useCache) {
+        try {
+          // Extract YAML code blocks from the markdown response
+          const codeblocks = await this.parser.findCodeBlocks(message.data);
+          if (codeblocks && codeblocks.length > 0) {
+            // Parse commands from all code blocks
+            const allCommands = [];
+            for (const block of codeblocks) {
+              const commands = await this.parser.getCommands(block);
+              allCommands.push(...commands);
+            }
+
+            // Create a proper step with prompt
+            const step = {
+              prompt: currentTask,
+              commands: allCommands,
+            };
+
+            // Use dumpToYML to create a valid testdriver yaml file
+            const yamlContent = await generator.dumpToYML(
+              [step],
+              this.sessionInstance,
+            );
+
+            const cachePath = promptCache.writeCache(currentTask, yamlContent);
+            if (cachePath) {
+              this.emitter.emit(
+                events.log.debug,
+                `Cached YAML saved to: ${cachePath}`,
+              );
+            }
+          }
+        } catch (err) {
+          // If we can't extract YAML, just skip caching
+          this.emitter.emit(
+            events.log.debug,
+            `Could not cache response: ${err.message}`,
+          );
+        }
+      }
+
       await this.aiExecute(message.data, validateAndLoop, dry, shouldSave);
       this.emitter.emit(
         events.log.debug,
@@ -1588,12 +1711,28 @@ ${regression}
     this.emitter.emit(events.log.log, `${inputFile} (end)`);
   }
 
+  // Returns the path to the sandbox file (project-local first, then global)
+  getSandboxFilePath(preferProjectLocal = true) {
+    const projectLocalFile = path.join(process.cwd(), ".testdriver-sandbox.json");
+    const globalFile = path.join(os.homedir(), ".testdriverai-last-sandbox");
+    
+    if (preferProjectLocal) {
+      // For reading: check project-local first, then global
+      if (fs.existsSync(projectLocalFile)) {
+        return projectLocalFile;
+      }
+      if (fs.existsSync(globalFile)) {
+        return globalFile;
+      }
+      // For writing: default to project-local
+      return projectLocalFile;
+    }
+    return globalFile;
+  }
+
   // Returns sandboxId to use (either from file if recent, or null)
   getRecentSandboxId() {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+    const lastSandboxFile = this.getSandboxFilePath(true);
 
     if (fs.existsSync(lastSandboxFile)) {
       try {
@@ -1619,7 +1758,8 @@ ${regression}
           const storedInstance = sandboxInfo.instanceType || null;
 
           if (currentAmi === storedAmi && currentInstance === storedInstance) {
-            return sandboxInfo.instanceId;
+            // Return sandboxId (new format) or instanceId (old format for backwards compatibility)
+            return sandboxInfo.sandboxId || sandboxInfo.instanceId;
           } else {
             this.emitter.emit(
               events.log.log,
@@ -1637,19 +1777,18 @@ ${regression}
     return null;
   }
 
-  saveLastSandboxId(instanceId) {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+  saveLastSandboxId(sandboxId, osType = "linux") {
+    // Save to project-local .testdriver-sandbox.json
+    const projectLocalFile = path.join(process.cwd(), ".testdriver-sandbox.json");
     try {
       const sandboxInfo = {
-        instanceId: instanceId,
+        sandboxId: sandboxId,
+        os: osType,
         ami: this.sandboxAmi || null,
         instanceType: this.sandboxInstance || null,
         timestamp: new Date().toISOString(),
       };
-      fs.writeFileSync(lastSandboxFile, JSON.stringify(sandboxInfo), {
+      fs.writeFileSync(projectLocalFile, JSON.stringify(sandboxInfo, null, 2), {
         encoding: "utf-8",
       });
     } catch {
@@ -1658,13 +1797,16 @@ ${regression}
   }
 
   clearRecentSandboxId() {
-    const lastSandboxFile = path.join(
-      os.homedir(),
-      ".testdriverai-last-sandbox",
-    );
+    // Clear project-local file first, then global
+    const projectLocalFile = path.join(process.cwd(), ".testdriver-sandbox.json");
+    const globalFile = path.join(os.homedir(), ".testdriverai-last-sandbox");
+    
     try {
-      if (fs.existsSync(lastSandboxFile)) {
-        fs.unlinkSync(lastSandboxFile);
+      if (fs.existsSync(projectLocalFile)) {
+        fs.unlinkSync(projectLocalFile);
+      }
+      if (fs.existsSync(globalFile)) {
+        fs.unlinkSync(globalFile);
       }
     } catch {
       // ignore errors
@@ -1682,6 +1824,11 @@ ${regression}
 
     let { headless = false, heal, new: createNew = false } = options;
 
+    // Prioritize this.newSandbox flag if it's set
+    if (this.newSandbox) {
+      createNew = true;
+    }
+
     // If CI environment variable is true, always create a new sandbox
     if (this.config.CI) {
       createNew = true;
@@ -1696,11 +1843,18 @@ ${regression}
     // If createNew flag is set, clear the recent sandbox file to force creating a new sandbox
     if (createNew) {
       this.clearRecentSandboxId();
-      if (!this.config.CI) {
+      // Also clear this.sandboxId to prevent reconnection attempts
+      this.sandboxId = null;
+      if (!this.config.CI && !this.newSandbox) {
         this.emitter.emit(
           events.log.log,
           theme.dim("--`new` flag detected, will create a new sandbox"),
         );
+      } else if (this.newSandbox) {
+        this.emitter.emit(
+          events.log.log,
+          theme.dim("--new-sandbox flag detected, will create a new sandbox"),
+        );
       }
     }
 
@@ -1708,6 +1862,9 @@ ${regression}
     await this.connectToSandboxService();
 
     const recentId = createNew ? null : this.getRecentSandboxId();
+    
+    // Track whether we reconnected to an existing sandbox or created a new one
+    this.isReconnected = false;
 
     // Set sandbox ID for reconnection (only if not creating new and recent ID exists)
     if (this.ip) {
@@ -1718,23 +1875,49 @@ ${regression}
         ip: this.ip,
       });
 
-      await this.renderSandbox(instance.instance, headless);
+      this.emitter.emit(events.sandbox.connected);
+
+      this.instance = instance.instance;
+      this.isReconnected = false; // Direct IP is considered new
+      await this.renderSandbox(this.instance, headless);
       await this.newSession();
       await this.runLifecycle("provision");
 
       return;
     } else if (!createNew && recentId) {
+      // Only attempt to connect to existing sandbox if not in CI mode and not creating new
       this.emitter.emit(
         events.log.narration,
         theme.dim(`using recent sandbox: ${recentId}`),
       );
       this.sandboxId = recentId;
-    } else if (!createNew) {
+      
+      try {
+        let instance = await this.connectToSandboxDirect(
+          this.sandboxId,
+          true, // always persist by default
+        );
+
+        this.instance = instance;
+        this.isReconnected = true; // Successfully reconnected to existing sandbox
+
+        await this.renderSandbox(instance, headless);
+        await this.newSession();
+        return;
+      } catch (error) {
+        // If connection fails, fall through to creating a new sandbox
+        this.emitter.emit(
+          events.log.narration,
+          theme.dim(`failed to connect to recent sandbox, creating new one...`),
+        );
+        console.error("Failed to reconnect to sandbox:", error);
+      }
+    } else if (!createNew && !recentId) {
       this.emitter.emit(
         events.log.narration,
         theme.dim(`no recent sandbox found, creating a new one.`),
       );
-    } else if (this.sandboxId && !this.config.CI) {
+    } else if (!createNew && this.sandboxId && !this.config.CI) {
       // Only attempt to connect to existing sandbox if not in CI mode and not creating new
       // Attempt to connect to known instance
       this.emitter.emit(
@@ -1749,51 +1932,55 @@ ${regression}
         );
 
         this.instance = instance;
+        this.isReconnected = true; // Successfully reconnected to existing sandbox
 
         await this.renderSandbox(instance, headless);
         await this.newSession();
         return;
       } catch (error) {
-        // But if it fails because the machine 404s, fall-through to `createNewSandbox()`
-        if (error?.name !== "InvalidInstanceID.NotFound") {
-          throw error;
-        }
+        // If connection fails, fall through to creating a new sandbox
+        this.emitter.emit(
+          events.log.narration,
+          theme.dim(`failed to connect to recent sandbox, creating new one...`),
+        );
+        console.error("Failed to reconnect to sandbox:", error);
       }
     }
-
-    this.emitter.emit(
-      events.log.narration,
-      theme.dim(`creating new sandbox (can take up to 2 minutes)...`),
-    );
-    // We don't have resiliency/retries baked in, so let's at least give it 1 attempt
-    // to see if that fixes the issue.
-    let newSandbox = await this.createNewSandbox().catch(() => {
+    
+    // Create new sandbox (either because createNew is true, or no existing sandbox to connect to)
+    if (!this.instance) {
+      this.isReconnected = false; // Creating new sandbox
       this.emitter.emit(
         events.log.narration,
-        theme.dim(`double-checking sandbox availability`),
+        theme.dim(`creating new sandbox...`),
       );
+      // We don't have resiliency/retries baked in, so let's at least give it 1 attempt
+      // to see if that fixes the issue.
+      let newSandbox = await this.createNewSandbox().catch(() => {
+        this.emitter.emit(
+          events.log.narration,
+          theme.dim(`double-checking sandbox availability`),
+        );
+        return this.createNewSandbox();
+      });
 
-      return this.createNewSandbox();
-    });
-
-    console.log("New sandbox created:", newSandbox);
-
-    let data = {
-      resolution: this.config.TD_RESOLUTION,
-      url: newSandbox.url,
-    };
-
-    const encodedData = Buffer.from(JSON.stringify(data)).toString('base64');
-
-    // Use the debugger URL instead of the VNC URL
-    const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`;
-
-    this.emitter.emit(events.showWindow, urlToOpen);
-
-    await this.newSession();
-    await this.runLifecycle("provision");
+      // Extract the sandbox ID from the newly created sandbox
+      this.sandboxId = newSandbox?.sandbox?.sandboxId || newSandbox?.sandbox?.instanceId;
+      
+      // Use the configured sandbox OS type
+      this.saveLastSandboxId(this.sandboxId, this.sandboxOs);
+      
+      let instance = await this.connectToSandboxDirect(
+        this.sandboxId,
+        true, // always persist by default
+      );
+      this.instance = instance;
+      await this.renderSandbox(instance, headless);
+      await this.newSession();
+      await this.runLifecycle("provision");
 
-    console.log("provision run");
+      console.log("provision run");
+    }
   }
 
   async start() {
@@ -1913,13 +2100,27 @@ ${regression}
   }
 
   async renderSandbox(instance, headless = false) {
+    console.log("renderSandbox", instance);
+
     if (!headless) {
-      let url =
-        "http://" +
-        instance.ip +
-        ":" +
-        instance.vncPort +
-        "/vnc_lite.html?token=V3b8wG9";
+      let url;
+
+      // If the instance already has a URL (from reconnection), use it
+      if (instance.url) {
+        url = instance.url;
+      } else if (instance.ip || instance.publicIp) {
+        // Otherwise construct it from IP and port
+        url =
+          "http://" +
+          (instance.ip || instance.publicIp) +
+          ":" +
+          (instance.vncPort || "5800") +
+          "/vnc_lite.html?token=V3b8wG9";
+      } else {
+        // If we don't have URL or IP, we can't render
+        console.warn("renderSandbox: Missing URL and IP in instance", instance);
+        return;
+      }
 
       let data = {
         resolution: this.config.TD_RESOLUTION,
@@ -1927,7 +2128,8 @@ ${regression}
         token: "V3b8wG9",
       };
 
-      const encodedData = encodeURIComponent(JSON.stringify(data));
+      // Base64 encode the data (the debugger expects base64, not URL encoding)
+      const encodedData = Buffer.from(JSON.stringify(data)).toString("base64");
 
       // Use the debugger URL instead of the VNC URL
       const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`;
@@ -1965,16 +2167,26 @@ Please check your network connection, TD_API_KEY, or the service status.`,
 
   async connectToSandboxDirect(sandboxId, persist = false) {
     this.emitter.emit(events.log.narration, theme.dim(`connecting...`));
-    let instance = await this.sandbox.connect(sandboxId, persist);
-    return instance;
+    let reply = await this.sandbox.connect(sandboxId, persist);
+
+    // reply includes { success, url, sandbox: {...} }
+    // For renderSandbox, we need the sandbox object with url merged in
+    const sandbox = reply.sandbox || {};
+
+    // If reply has a URL at top level, merge it into the sandbox object
+    if (reply.url && !sandbox.url) {
+      sandbox.url = reply.url;
+    }
+
+    return sandbox;
   }
 
   async createNewSandbox() {
     const sandboxConfig = {
       type: "create",
-      os: "linux",
       resolution: this.config.TD_RESOLUTION,
       ci: this.config.CI,
+      os: this.sandboxOs || "windows",
     };
 
     // Add AMI and instance type if specified
@@ -1985,11 +2197,14 @@ Please check your network connection, TD_API_KEY, or the service status.`,
       sandboxConfig.instanceType = this.sandboxInstance;
     }
 
-    console.log("sending create");
-
     let instance = await this.sandbox.send(sandboxConfig);
 
-    console.log("instance created", instance);
+    // Save the sandbox ID for reconnection with the correct OS type
+    if (instance.sandbox && instance.sandbox.sandboxId) {
+      this.saveLastSandboxId(instance.sandbox.sandboxId, this.sandboxOs);
+    } else if (instance.sandbox && instance.sandbox.instanceId) {
+      this.saveLastSandboxId(instance.sandbox.instanceId, this.sandboxOs);
+    }
 
     return instance;
   }

package/agent/interface.js

@@ -66,6 +66,11 @@ function createCommandDefinitions(agent) {
           description: "Generate JUnit XML test report to specified file",
           default: false,
         }),
+        os: Flags.string({
+          description: "Operating system for the sandbox (windows or linux)",
+          options: ["windows", "linux"],
+          default: "windows",
+        }),
       },
       handler: async (args, flags) => {
         // Use --path flag if provided, otherwise fall back to args.file
@@ -140,6 +145,11 @@ function createCommandDefinitions(agent) {
         summary: Flags.string({
           description: "Specify output file for summarize results",
         }),
+        os: Flags.string({
+          description: "Operating system for the sandbox (windows or linux)",
+          options: ["windows", "linux"],
+          default: "windows",
+        }),
       },
       handler: async () => {
         // Edit mode is handled by the CLI interface via factory.js
@@ -233,6 +243,11 @@ function createCommandDefinitions(agent) {
         "sandbox-instance": Flags.string({
           description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
         }),
+        os: Flags.string({
+          description: "Operating system for the sandbox (windows or linux)",
+          options: ["windows", "linux"],
+          default: "windows",
+        }),
       },
       handler: async (args, flags) => {
         // Call generate with the count and prompt