testdriverai 6.1.10 → 6.2.0

package/.github/workflows/acceptance-tests.yml

@@ -7,8 +7,6 @@ on:
   # So that we don't do expensive tests until approved
   push:
     branches: [main]
-    paths-ignore:
-      - "docs/**"
   # So that we can manually trigger tests when there's flake
   workflow_dispatch:
 

package/.github/workflows/acceptance-v6.yml

@@ -5,8 +5,6 @@ on:
   push:
     branches:
       - main
-    paths-ignore:
-      - "docs/**"
   pull_request:
     branches:
       - main

package/.github/workflows/lint.yml

@@ -1,10 +1,7 @@
 # Ensure affected code follows standards and is formatted correctly. Otherwise, automatic formatting in future changes will cause larger diffs.
 name: Lint + Prettier
 
-on:
-  push:
-    paths-ignore:
-      - "docs/**"
+on: push
 
 jobs:
   lint:

package/.github/workflows/publish-canary.yml

@@ -10,8 +10,6 @@ on:
   # So that we publish for every push to `main`, despite tests
   push:
     branches: [main]
-    paths-ignore:
-      - "docs/**"
   workflow_dispatch:
 
 jobs:

package/.github/workflows/publish-latest.yml

@@ -2,7 +2,6 @@
 name: Publish @latest to NPM
 
 on:
-  workflow_dispatch:
   workflow_run:
     workflows: ["Acceptance Tests"]
     branches: [main]

package/.prettierignore

@@ -1,4 +1,3 @@
 agent/lib/subimage/opencv.js
 node_modules
 schema.json
-docs

package/.vscode/settings.json

@@ -3,8 +3,5 @@
     "source.organizeImports": "explicit"
   },
   "editor.formatOnSave": true,
-  "editor.defaultFormatter": "esbenp.prettier-vscode",
-  "yaml.schemas": {
-    "https://raw.githubusercontent.com/testdriverai/testdriverai/main/schema.json": "file:///Users/kid/Desktop/td/internal/testdriverai/testdriver.yaml"
-  }
+  "editor.defaultFormatter": "esbenp.prettier-vscode"
 }

package/agent/events.js

@@ -1,4 +1,5 @@
 const { EventEmitter2 } = require("eventemitter2");
+const { censorSensitiveDataDeep } = require("./lib/censorship");
 
 // Factory function to create a new emitter instance with censoring middleware
 const createEmitter = () => {
@@ -12,6 +13,14 @@ const createEmitter = () => {
     ignoreErrors: false,
   });
 
+  // Override emit to censor sensitive data before emitting
+  const originalEmit = emitter.emit.bind(emitter);
+  emitter.emit = function (event, ...args) {
+    // Censor all arguments passed to emit
+    const censoredArgs = args.map(censorSensitiveDataDeep);
+    return originalEmit(event, ...censoredArgs);
+  };
+
   return emitter;
 };
 
@@ -37,7 +46,7 @@ const events = {
   status: "status",
   log: {
     markdown: {
-      static: "log:markdown",
+      static: "log:markdown:static",
       start: "log:markdown:start",
       chunk: "log:markdown:chunk",
       end: "log:markdown:end",

package/agent/index.js

@@ -63,18 +63,13 @@ class TestDriverAgent extends EventEmitter2 {
     // Derive properties from cliArgs
     const flags = cliArgs.options || {};
     const firstArg = cliArgs.args && cliArgs.args[0];
-
-    // All commands (run, edit, generate) use the same pattern:
-    // first argument is the main file to work with
     this.thisFile = firstArg || this.config.TD_DEFAULT_TEST_FILE;
-
     this.resultFile = flags.resultFile || null;
     this.newSandbox = flags.newSandbox || false;
     this.healMode = flags.healMode || flags.heal || false;
     this.sandboxId = flags["sandbox-id"] || null;
     this.sandboxAmi = flags["sandbox-ami"] || null;
     this.sandboxInstance = flags["sandbox-instance"] || null;
-    this.ip = flags.ip || null;
     this.workingDir = flags.workingDir || process.cwd();
 
     // Resolve thisFile to absolute path with proper extension
@@ -227,15 +222,7 @@ class TestDriverAgent extends EventEmitter2 {
     if (skipPostrun) {
       this.exit(true);
     } else {
-      try {
-        await this.summarize(error.message);
-      } catch (summarizeError) {
-        // If summarization fails, log it but don't let it prevent postrun from running
-        this.emitter.emit(
-          events.log.warn,
-          theme.yellow(`Failed to summarize: ${summarizeError.message}`),
-        );
-      }
+      await this.summarize(error.message);
       // Always run postrun lifecycle script, even for fatal errors
       return await this.exit(true, false, true);
     }
@@ -432,7 +419,6 @@ class TestDriverAgent extends EventEmitter2 {
 
     // Log current execution position for debugging
     if (this.sourceMapper.currentFileSourceMap) {
-      this.emitter.emit(events.log.log, "");
       this.emitter.emit(
         events.log.log,
         theme.dim(`${this.sourceMapper.getCurrentPositionDescription()}`),
@@ -490,13 +476,14 @@ class TestDriverAgent extends EventEmitter2 {
         sourcePosition: sourcePosition,
       });
 
-      return await this.haveAIResolveError(
+      await this.haveAIResolveError(
         error,
         yaml.dump({ commands: [yml] }),
         depth,
         true,
         shouldSave,
       );
+      throw error;
     }
   }
 
@@ -892,33 +879,30 @@ commands:
   // based on the current state of the system (primarily the current screenshot)
   // it will generate files that contain only "prompts"
   // @todo revit the generate command
-  async generate(count = 1, prompt = null) {
-    this.emitter.emit(
-      events.log.debug,
-      `generate called with count: ${count}, prompt: ${prompt}`,
-    );
-
-    await this.runLifecycle("prerun");
+  async generate(type, count, baseYaml, skipYaml = false) {
+    this.emitter.emit(events.log.debug, "generate called, %s", type);
 
     this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
+    if (baseYaml && !skipYaml) {
+      await this.runLifecycle("prerun");
+      await this.run(baseYaml, false, false);
+      await this.runLifecycle("postrun");
+    }
+
     let image = await this.system.captureScreenBase64();
 
     const streamId = `generate-${Date.now()}`;
     this.emitter.emit(events.log.markdown.start, streamId);
 
-    let mouse = await this.system.getMousePosition();
-    let activeWindow = await this.system.activeWin();
-
     let message = await this.sdk.req(
       "generate",
       {
-        prompt: prompt || "make sure to do a spellcheck",
+        type,
         image,
-        mousePosition: mouse,
-        activeWindow: activeWindow,
+        mousePosition: await this.system.getMousePosition(),
+        activeWindow: await this.system.activeWin(),
         count,
-        stream: false,
       },
       (chunk) => {
         if (chunk.type === "data") {
@@ -941,36 +925,35 @@ commands:
           .replace(/['"`]/g, "")
           .replace(/[^a-zA-Z0-9-]/g, "") // remove any non-alphanumeric chars except hyphens
           .toLowerCase() + ".yaml";
-
       let path1 = path.join(
         this.workingDir,
         "testdriver",
         "generate",
         fileName,
       );
+
       // create generate directory if it doesn't exist
-      const generateDir = path.join(this.workingDir, "testdriver", "generate");
-      if (!fs.existsSync(generateDir)) {
-        fs.mkdirSync(generateDir);
-        console.log("Created generate directory:", generateDir);
-      } else {
-        console.log("Generate directory already exists:", generateDir);
+      if (!fs.existsSync(path.join(this.workingDir, "generate"))) {
+        fs.mkdirSync(path.join(this.workingDir, "generate"));
       }
 
       let list = testPrompt.steps;
 
+      if (baseYaml && fs.existsSync(baseYaml)) {
+        list.unshift({
+          step: {
+            command: "run",
+            file: baseYaml,
+          },
+        });
+      }
       let contents = yaml.dump({
         version: packageJson.version,
         steps: list,
       });
-
-      this.emitter.emit(events.log.debug, `writing file ${path1} ${contents}`);
-
       fs.writeFileSync(path1, contents);
     }
 
-    await this.runLifecycle("postrun");
-
     this.exit(false);
   }
 
@@ -1156,8 +1139,21 @@ ${yml}
 
     // Create diff if file exists and content has changed
     let diffResult = null;
+    console.log("Checking for diff. File exists:", fileExists);
+    console.log(
+      "Content changed:",
+      fileExists && existingContent !== regression,
+    );
+    if (fileExists) {
+      console.log(
+        "Existing content preview:",
+        existingContent.substring(0, 100),
+      );
+      console.log("New content preview:", regression.substring(0, 100));
+    }
 
     if (fileExists && existingContent !== regression) {
+      console.log("Creating diff - content has changed");
       const patches = diff.structuredPatch(
         filepath,
         filepath,
@@ -1245,6 +1241,8 @@ ${yml}
           diff: diffResult,
           timestamp: endTime,
         });
+      } else {
+        console.log("No diff result to emit");
       }
 
       // Emit file save completion event
@@ -1521,8 +1519,6 @@ ${regression}
   }
 
   async embed(file, depth, pushToHistory) {
-    let inputFile = JSON.parse(JSON.stringify(file));
-
     this.analytics.track("embed", { file });
 
     this.emitter.emit(
@@ -1532,7 +1528,7 @@ ${regression}
 
     depth = depth + 1;
 
-    this.emitter.emit(events.log.log, `${inputFile} (start)`);
+    this.emitter.emit(events.log.log, `${file} (start)`);
 
     // Use the new helper method to resolve file paths relative to testdriver directory
     const currentFilePath = this.sourceMapper.currentFilePath || this.thisFile;
@@ -1585,7 +1581,7 @@ ${regression}
       this.sourceMapper.restoreContext(previousContext);
     }
 
-    this.emitter.emit(events.log.log, `${inputFile} (end)`);
+    this.emitter.emit(events.log.log, `${file} (end)`);
   }
 
   // Returns sandboxId to use (either from file if recent, or null)
@@ -1710,20 +1706,7 @@ ${regression}
     const recentId = createNew ? null : this.getRecentSandboxId();
 
     // Set sandbox ID for reconnection (only if not creating new and recent ID exists)
-    if (this.ip) {
-      let instance = await this.sandbox.send({
-        type: "direct",
-        resolution: this.config.TD_RESOLUTION,
-        ci: this.config.CI,
-        ip: this.ip,
-      });
-
-      await this.renderSandbox(instance.instance, headless);
-      await this.newSession();
-      await this.runLifecycle("provision");
-
-      return;
-    } else if (!createNew && recentId) {
+    if (!createNew && recentId) {
       this.emitter.emit(
         events.log.narration,
         theme.dim(`using recent sandbox: ${recentId}`),
@@ -1734,8 +1717,10 @@ ${regression}
         events.log.narration,
         theme.dim(`no recent sandbox found, creating a new one.`),
       );
-    } else if (this.sandboxId && !this.config.CI) {
-      // Only attempt to connect to existing sandbox if not in CI mode and not creating new
+    }
+
+    // Only attempt to connect to existing sandbox if not in CI mode and not creating new
+    if (this.sandboxId && !this.config.CI && !createNew) {
       // Attempt to connect to known instance
       this.emitter.emit(
         events.log.narration,
@@ -1776,15 +1761,24 @@ ${regression}
       return this.createNewSandbox();
     });
 
-    this.saveLastSandboxId(newSandbox.sandbox.instanceId);
-    let instance = await this.connectToSandboxDirect(
-      newSandbox.sandbox.instanceId,
-      true, // always persist by default
-    );
-    this.instance = instance;
-    await this.renderSandbox(instance, headless);
+    console.log("New sandbox created:", newSandbox);
+
+    let data = {
+      resolution: this.config.TD_RESOLUTION,
+      url: newSandbox.url,
+    };
+
+    const encodedData = encodeURIComponent(JSON.stringify(data));
+
+    // Use the debugger URL instead of the VNC URL
+    const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`;
+
+    this.emitter.emit(events.showWindow, urlToOpen);
+
     await this.newSession();
     await this.runLifecycle("provision");
+
+    console.log("provision run");
   }
 
   async start() {
@@ -1793,16 +1787,13 @@ ${regression}
         events.log.log,
         theme.green(`Howdy! I'm TestDriver v${packageJson.version}`),
       );
-
-      // Emit test start event for the entire test execution
-      this.emitter.emit(events.test.start, {
-        filePath: this.thisFile,
-        timestamp: Date.now(),
-      });
-
       // Start the debugger server as early as possible to ensure event listeners are attached
       if (!debuggerStarted) {
         debuggerStarted = true; // Prevent multiple starts, especially when running test in parallel
+        this.emitter.emit(
+          events.log.narration,
+          theme.green(`Starting debugger server...`),
+        );
         debuggerProcess = await createDebuggerProcess(
           this.config,
           this.emitter,
@@ -1810,7 +1801,6 @@ ${regression}
       }
       this.debuggerUrl = debuggerProcess.url || null; // Store the debugger URL
       this.emitter.emit(events.log.log, `This is beta software!`);
-      this.emitter.emit(events.log.log, ``);
       this.emitter.emit(
         events.log.log,
         theme.yellow(`Join our Discord for help`),
@@ -1819,7 +1809,6 @@ ${regression}
         events.log.log,
         `https://discord.com/invite/cWDFW8DzPm`,
       );
-      this.emitter.emit(events.log.log, ``);
 
       // make testdriver directory if it doesn't exist
       let testdriverFolder = path.join(this.workingDir);
@@ -1833,10 +1822,7 @@ ${regression}
       }
 
       // if the directory for thisFile doesn't exist, create it
-      if (
-        this.cliArgs.command !== "sandbox" &&
-        this.cliArgs.command !== "generate"
-      ) {
+      if (this.cliArgs.command !== "sandbox") {
         const dir = path.dirname(this.thisFile);
         if (!fs.existsSync(dir)) {
           fs.mkdirSync(dir, { recursive: true });
@@ -1861,10 +1847,7 @@ ${regression}
         await this.sdk.auth();
       }
 
-      if (
-        this.cliArgs.command !== "sandbox" &&
-        this.cliArgs.command !== "generate"
-      ) {
+      if (this.cliArgs.command !== "sandbox") {
         this.emitter.emit(
           events.log.log,
           theme.dim(`Working on ${this.thisFile}`),
@@ -1963,6 +1946,7 @@ Please check your network connection, TD_API_KEY, or the service status.`,
   async createNewSandbox() {
     const sandboxConfig = {
       type: "create",
+      os: "linux",
       resolution: this.config.TD_RESOLUTION,
       ci: this.config.CI,
     };
@@ -1975,7 +1959,12 @@ Please check your network connection, TD_API_KEY, or the service status.`,
       sandboxConfig.instanceType = this.sandboxInstance;
     }
 
+    console.log("sending create");
+
     let instance = await this.sandbox.send(sandboxConfig);
+
+    console.log("instance created", instance);
+
     return instance;
   }
 
@@ -2061,20 +2050,6 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     // Use the current file path from sourceMapper to find the lifecycle directory
     // If sourceMapper doesn't have a current file, use thisFile which should be the file being run
     let currentFilePath = this.sourceMapper.currentFilePath || this.thisFile;
-
-    this.emitter.emit(events.log.log, ``);
-    this.emitter.emit(events.log.log, "Running lifecycle: " + lifecycleName);
-
-    // If we still don't have a currentFilePath, fall back to the default testdriver directory
-    if (!currentFilePath) {
-      currentFilePath = path.join(
-        this.workingDir,
-        "testdriver",
-        "testdriver.yaml",
-      );
-      console.log("No currentFilePath found, using fallback:", currentFilePath);
-    }
-
     // Ensure we have an absolute path
     if (currentFilePath && !path.isAbsolute(currentFilePath)) {
       currentFilePath = path.resolve(this.workingDir, currentFilePath);
@@ -2111,9 +2086,6 @@ Please check your network connection, TD_API_KEY, or the service status.`,
         }
       }
     }
-
-    this.emitter.emit(events.log.log, lifecycleFile);
-
     if (lifecycleFile) {
       // Store current source mapping state before running lifecycle file
       const previousContext = this.sourceMapper.saveContext();
@@ -2183,7 +2155,7 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     }
 
     // Move environment setup and special handling here
-    if (["edit", "run", "generate"].includes(commandName)) {
+    if (["edit", "run"].includes(commandName)) {
       await this.buildEnv(options);
     }
 

package/agent/interface.js

@@ -55,10 +55,6 @@ function createCommandDefinitions(agent) {
         "sandbox-instance": Flags.string({
           description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
         }),
-        ip: Flags.string({
-          description:
-            "Connect directly to a sandbox at the specified IP address",
-        }),
         summary: Flags.string({
           description: "Specify output file for summarize results",
         }),
@@ -72,6 +68,12 @@ function createCommandDefinitions(agent) {
         const file = normalizeFilePath(args.file);
         const testStartTime = Date.now();
 
+        // Emit test start event for the entire test execution
+        agent.emitter.emit(events.test.start, {
+          filePath: file,
+          timestamp: testStartTime,
+        });
+
         try {
           await agent.runLifecycle("prerun");
           // When run() is called through run.js CLI command, shouldExit should be true
@@ -133,10 +135,6 @@ function createCommandDefinitions(agent) {
         "sandbox-instance": Flags.string({
           description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
         }),
-        ip: Flags.string({
-          description:
-            "Connect directly to a sandbox at the specified IP address",
-        }),
         summary: Flags.string({
           description: "Specify output file for summarize results",
         }),
@@ -204,41 +202,6 @@ function createCommandDefinitions(agent) {
         console.log(`TestDriver.ai v${packageJson.version}`);
       },
     },
-
-    generate: {
-      description: "Generate test files based on current screen state",
-      args: {
-        prompt: Args.string({
-          description: "Multi-line text prompt describing what to generate",
-          required: false,
-        }),
-      },
-      flags: {
-        count: Flags.integer({
-          description: "Number of test files to generate",
-          default: 3,
-        }),
-        headless: Flags.boolean({
-          description: "Run in headless mode (no GUI)",
-          default: false,
-        }),
-        new: Flags.boolean({
-          description:
-            "Create a new sandbox instead of reconnecting to an existing one",
-          default: false,
-        }),
-        "sandbox-ami": Flags.string({
-          description: "Specify AMI ID for sandbox instance (e.g., ami-1234)",
-        }),
-        "sandbox-instance": Flags.string({
-          description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
-        }),
-      },
-      handler: async (args, flags) => {
-        // Call generate with the count and prompt
-        await agent.generate(flags.count || 3, args.prompt);
-      },
-    },
   };
 }
 

package/agent/lib/censorship.js

@@ -38,23 +38,18 @@ const censorSensitiveData = (message) => {
 
 // Function to censor sensitive data in any value (recursive for objects/arrays)
 const censorSensitiveDataDeep = (value) => {
-  try {
-    if (typeof value === "string") {
-      return censorSensitiveData(value);
-    } else if (Array.isArray(value)) {
-      return value.map(censorSensitiveDataDeep);
-    } else if (value && typeof value === "object") {
-      const result = {};
-      for (const [key, val] of Object.entries(value)) {
-        result[key] = censorSensitiveDataDeep(val);
-      }
-      return result;
+  if (typeof value === "string") {
+    return censorSensitiveData(value);
+  } else if (Array.isArray(value)) {
+    return value.map(censorSensitiveDataDeep);
+  } else if (value && typeof value === "object") {
+    const result = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = censorSensitiveDataDeep(val);
     }
-    return value;
-  } catch {
-    // If we hit any error (like circular reference), just return a safe placeholder
-    return "[Object]";
+    return result;
   }
+  return value;
 };
 
 // Function to update interpolation variables (for runtime updates)