testdriverai 6.2.0 → 6.2.1

package/.github/workflows/acceptance-tests.yml

@@ -7,6 +7,8 @@ on:
   # So that we don't do expensive tests until approved
   push:
     branches: [main]
+    paths-ignore:
+      - "docs/**"
   # So that we can manually trigger tests when there's flake
   workflow_dispatch:
 

package/.github/workflows/acceptance-v6.yml

@@ -5,6 +5,8 @@ on:
   push:
     branches:
       - main
+    paths-ignore:
+      - "docs/**"
   pull_request:
     branches:
       - main

package/.github/workflows/lint.yml

@@ -1,7 +1,10 @@
 # Ensure affected code follows standards and is formatted correctly. Otherwise, automatic formatting in future changes will cause larger diffs.
 name: Lint + Prettier
 
-on: push
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
 
 jobs:
   lint:

package/.github/workflows/publish-canary.yml

@@ -10,6 +10,8 @@ on:
   # So that we publish for every push to `main`, despite tests
   push:
     branches: [main]
+    paths-ignore:
+      - "docs/**"
   workflow_dispatch:
 
 jobs:

package/.github/workflows/publish-latest.yml

@@ -2,6 +2,7 @@
 name: Publish @latest to NPM
 
 on:
+  workflow_dispatch:
   workflow_run:
     workflows: ["Acceptance Tests"]
     branches: [main]

package/.github/workflows/self-hosted.yml

@@ -0,0 +1,102 @@
+name: AWS
+
+on:
+  workflow_dispatch:
+  push:
+    paths-ignore:
+      - "docs/**"
+
+jobs:
+  gather:
+    name: Gather Test Files
+    runs-on: ubuntu-latest
+    outputs:
+      test_files: ${{ steps.test_list.outputs.files }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Find all test files
+        id: test_list
+        run: |
+          FILES=$(ls ./testdriver/acceptance/*.yaml)
+          FILENAMES=$(basename -a $FILES)
+          FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]')
+          echo "files=$FILES_JSON" >> $GITHUB_OUTPUT
+
+  test:
+    needs: gather
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test: ${{ fromJson(needs.gather.outputs.test_files) }}
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      # only needed for `act`
+      # - name: Install AWS CLI
+      #   run: |
+      #     apt-get update
+      #     apt-get install curl unzip -y
+      #     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      #     unzip awscliv2.zip
+      #     ./aws/install
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+      - name: Install dependencies
+        run: NODE_ENV=production npm ci
+      - name: Setup AWS Instance
+        id: aws-setup
+        run: |
+          OUTPUT=$(./setup/aws/spawn-runner.sh | tee /dev/stderr) # Capture and display output
+          echo "$OUTPUT"
+          PUBLIC_IP=$(echo "$OUTPUT" | grep "PUBLIC_IP=" | cut -d'=' -f2)
+          INSTANCE_ID=$(echo "$OUTPUT" | grep "INSTANCE_ID=" | cut -d'=' -f2)
+          AWS_REGION=$(echo "$OUTPUT" | grep "AWS_REGION=" | cut -d'=' -f2)
+          echo "public-ip=$PUBLIC_IP" >> $GITHUB_OUTPUT
+          echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT
+          echo "aws-region=$AWS_REGION" >> $GITHUB_OUTPUT
+        env:
+          FORCE_COLOR: 3
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-2
+          AWS_LAUNCH_TEMPLATE_ID: lt-00d02f31cfc602f27
+          AMI_ID: ami-085f872ca0cd80fed
+          RESOLUTION_WIDTH: 1920
+          RESOLUTION_HEIGHT: 1080
+      - name: Run TestDriver
+        run: node bin/testdriverai.js run testdriver/acceptance/${{ matrix.test }} --ip="${{ steps.aws-setup.outputs.public-ip }}" --junit=out.xml
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TD_WEBSITE: https://testdriver-sandbox.vercel.app
+          TD_THIS_FILE: ${{ matrix.test }}
+      - name: Upload TestDriver AI CLI logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: testdriverai-cli-logs-${{ matrix.test }}
+          path: /tmp/testdriverai-cli-*.log
+          if-no-files-found: warn
+          retention-days: 30
+      - name: Upload test results as artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.test }}
+          path: out.xml
+          retention-days: 30
+      - name: Shutdown AWS Instance
+        if: always()
+        run: aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids "$INSTANCE_ID"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ steps.aws-setup.outputs.aws-region }}
+          INSTANCE_ID: ${{ steps.aws-setup.outputs.instance-id }}

package/.prettierignore

@@ -1,3 +1,4 @@
 agent/lib/subimage/opencv.js
 node_modules
 schema.json
+docs

package/.vscode/settings.json

@@ -3,5 +3,8 @@
     "source.organizeImports": "explicit"
   },
   "editor.formatOnSave": true,
-  "editor.defaultFormatter": "esbenp.prettier-vscode"
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "yaml.schemas": {
+    "https://raw.githubusercontent.com/testdriverai/testdriverai/main/schema.json": "file:///Users/kid/Desktop/td/internal/testdriverai/testdriver.yaml"
+  }
 }

package/agent/events.js

@@ -1,5 +1,4 @@
 const { EventEmitter2 } = require("eventemitter2");
-const { censorSensitiveDataDeep } = require("./lib/censorship");
 
 // Factory function to create a new emitter instance with censoring middleware
 const createEmitter = () => {
@@ -13,14 +12,6 @@ const createEmitter = () => {
     ignoreErrors: false,
   });
 
-  // Override emit to censor sensitive data before emitting
-  const originalEmit = emitter.emit.bind(emitter);
-  emitter.emit = function (event, ...args) {
-    // Censor all arguments passed to emit
-    const censoredArgs = args.map(censorSensitiveDataDeep);
-    return originalEmit(event, ...censoredArgs);
-  };
-
   return emitter;
 };
 
@@ -46,7 +37,7 @@ const events = {
   status: "status",
   log: {
     markdown: {
-      static: "log:markdown:static",
+      static: "log:markdown",
       start: "log:markdown:start",
       chunk: "log:markdown:chunk",
       end: "log:markdown:end",

package/agent/index.js

@@ -63,13 +63,18 @@ class TestDriverAgent extends EventEmitter2 {
     // Derive properties from cliArgs
     const flags = cliArgs.options || {};
     const firstArg = cliArgs.args && cliArgs.args[0];
+
+    // All commands (run, edit, generate) use the same pattern:
+    // first argument is the main file to work with
     this.thisFile = firstArg || this.config.TD_DEFAULT_TEST_FILE;
+
     this.resultFile = flags.resultFile || null;
     this.newSandbox = flags.newSandbox || false;
     this.healMode = flags.healMode || flags.heal || false;
     this.sandboxId = flags["sandbox-id"] || null;
     this.sandboxAmi = flags["sandbox-ami"] || null;
     this.sandboxInstance = flags["sandbox-instance"] || null;
+    this.ip = flags.ip || null;
     this.workingDir = flags.workingDir || process.cwd();
 
     // Resolve thisFile to absolute path with proper extension
@@ -222,7 +227,15 @@ class TestDriverAgent extends EventEmitter2 {
     if (skipPostrun) {
       this.exit(true);
     } else {
-      await this.summarize(error.message);
+      try {
+        await this.summarize(error.message);
+      } catch (summarizeError) {
+        // If summarization fails, log it but don't let it prevent postrun from running
+        this.emitter.emit(
+          events.log.warn,
+          theme.yellow(`Failed to summarize: ${summarizeError.message}`),
+        );
+      }
       // Always run postrun lifecycle script, even for fatal errors
       return await this.exit(true, false, true);
     }
@@ -419,6 +432,7 @@ class TestDriverAgent extends EventEmitter2 {
 
     // Log current execution position for debugging
     if (this.sourceMapper.currentFileSourceMap) {
+      this.emitter.emit(events.log.log, "");
       this.emitter.emit(
         events.log.log,
         theme.dim(`${this.sourceMapper.getCurrentPositionDescription()}`),
@@ -476,14 +490,13 @@ class TestDriverAgent extends EventEmitter2 {
         sourcePosition: sourcePosition,
       });
 
-      await this.haveAIResolveError(
+      return await this.haveAIResolveError(
         error,
         yaml.dump({ commands: [yml] }),
         depth,
         true,
         shouldSave,
       );
-      throw error;
     }
   }
 
@@ -879,30 +892,33 @@ commands:
   // based on the current state of the system (primarily the current screenshot)
   // it will generate files that contain only "prompts"
   // @todo revit the generate command
-  async generate(type, count, baseYaml, skipYaml = false) {
-    this.emitter.emit(events.log.debug, "generate called, %s", type);
+  async generate(count = 1, prompt = null) {
+    this.emitter.emit(
+      events.log.debug,
+      `generate called with count: ${count}, prompt: ${prompt}`,
+    );
 
-    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
+    await this.runLifecycle("prerun");
 
-    if (baseYaml && !skipYaml) {
-      await this.runLifecycle("prerun");
-      await this.run(baseYaml, false, false);
-      await this.runLifecycle("postrun");
-    }
+    this.emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
     let image = await this.system.captureScreenBase64();
 
     const streamId = `generate-${Date.now()}`;
     this.emitter.emit(events.log.markdown.start, streamId);
 
+    let mouse = await this.system.getMousePosition();
+    let activeWindow = await this.system.activeWin();
+
     let message = await this.sdk.req(
       "generate",
       {
-        type,
+        prompt: prompt || "make sure to do a spellcheck",
         image,
-        mousePosition: await this.system.getMousePosition(),
-        activeWindow: await this.system.activeWin(),
+        mousePosition: mouse,
+        activeWindow: activeWindow,
         count,
+        stream: false,
       },
       (chunk) => {
         if (chunk.type === "data") {
@@ -925,35 +941,36 @@ commands:
           .replace(/['"`]/g, "")
           .replace(/[^a-zA-Z0-9-]/g, "") // remove any non-alphanumeric chars except hyphens
           .toLowerCase() + ".yaml";
+
       let path1 = path.join(
         this.workingDir,
         "testdriver",
         "generate",
         fileName,
       );
-
       // create generate directory if it doesn't exist
-      if (!fs.existsSync(path.join(this.workingDir, "generate"))) {
-        fs.mkdirSync(path.join(this.workingDir, "generate"));
+      const generateDir = path.join(this.workingDir, "testdriver", "generate");
+      if (!fs.existsSync(generateDir)) {
+        fs.mkdirSync(generateDir);
+        console.log("Created generate directory:", generateDir);
+      } else {
+        console.log("Generate directory already exists:", generateDir);
       }
 
       let list = testPrompt.steps;
 
-      if (baseYaml && fs.existsSync(baseYaml)) {
-        list.unshift({
-          step: {
-            command: "run",
-            file: baseYaml,
-          },
-        });
-      }
       let contents = yaml.dump({
         version: packageJson.version,
         steps: list,
       });
+
+      this.emitter.emit(events.log.debug, `writing file ${path1} ${contents}`);
+
       fs.writeFileSync(path1, contents);
     }
 
+    await this.runLifecycle("postrun");
+
     this.exit(false);
   }
 
@@ -1139,21 +1156,8 @@ ${yml}
 
     // Create diff if file exists and content has changed
     let diffResult = null;
-    console.log("Checking for diff. File exists:", fileExists);
-    console.log(
-      "Content changed:",
-      fileExists && existingContent !== regression,
-    );
-    if (fileExists) {
-      console.log(
-        "Existing content preview:",
-        existingContent.substring(0, 100),
-      );
-      console.log("New content preview:", regression.substring(0, 100));
-    }
 
     if (fileExists && existingContent !== regression) {
-      console.log("Creating diff - content has changed");
       const patches = diff.structuredPatch(
         filepath,
         filepath,
@@ -1241,8 +1245,6 @@ ${yml}
           diff: diffResult,
           timestamp: endTime,
         });
-      } else {
-        console.log("No diff result to emit");
       }
 
       // Emit file save completion event
@@ -1519,6 +1521,8 @@ ${regression}
   }
 
   async embed(file, depth, pushToHistory) {
+    let inputFile = JSON.parse(JSON.stringify(file));
+
     this.analytics.track("embed", { file });
 
     this.emitter.emit(
@@ -1528,7 +1532,7 @@ ${regression}
 
     depth = depth + 1;
 
-    this.emitter.emit(events.log.log, `${file} (start)`);
+    this.emitter.emit(events.log.log, `${inputFile} (start)`);
 
     // Use the new helper method to resolve file paths relative to testdriver directory
     const currentFilePath = this.sourceMapper.currentFilePath || this.thisFile;
@@ -1581,7 +1585,7 @@ ${regression}
       this.sourceMapper.restoreContext(previousContext);
     }
 
-    this.emitter.emit(events.log.log, `${file} (end)`);
+    this.emitter.emit(events.log.log, `${inputFile} (end)`);
   }
 
   // Returns sandboxId to use (either from file if recent, or null)
@@ -1706,7 +1710,20 @@ ${regression}
     const recentId = createNew ? null : this.getRecentSandboxId();
 
     // Set sandbox ID for reconnection (only if not creating new and recent ID exists)
-    if (!createNew && recentId) {
+    if (this.ip) {
+      let instance = await this.sandbox.send({
+        type: "direct",
+        resolution: this.config.TD_RESOLUTION,
+        ci: this.config.CI,
+        ip: this.ip,
+      });
+
+      await this.renderSandbox(instance.instance, headless);
+      await this.newSession();
+      await this.runLifecycle("provision");
+
+      return;
+    } else if (!createNew && recentId) {
       this.emitter.emit(
         events.log.narration,
         theme.dim(`using recent sandbox: ${recentId}`),
@@ -1717,10 +1734,8 @@ ${regression}
         events.log.narration,
         theme.dim(`no recent sandbox found, creating a new one.`),
       );
-    }
-
-    // Only attempt to connect to existing sandbox if not in CI mode and not creating new
-    if (this.sandboxId && !this.config.CI && !createNew) {
+    } else if (this.sandboxId && !this.config.CI) {
+      // Only attempt to connect to existing sandbox if not in CI mode and not creating new
       // Attempt to connect to known instance
       this.emitter.emit(
         events.log.narration,
@@ -1768,7 +1783,7 @@ ${regression}
       url: newSandbox.url,
     };
 
-    const encodedData = encodeURIComponent(JSON.stringify(data));
+    const encodedData = Buffer.from(JSON.stringify(data)).toString('base64');
 
     // Use the debugger URL instead of the VNC URL
     const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`;
@@ -1787,13 +1802,16 @@ ${regression}
         events.log.log,
         theme.green(`Howdy! I'm TestDriver v${packageJson.version}`),
       );
+
+      // Emit test start event for the entire test execution
+      this.emitter.emit(events.test.start, {
+        filePath: this.thisFile,
+        timestamp: Date.now(),
+      });
+
       // Start the debugger server as early as possible to ensure event listeners are attached
       if (!debuggerStarted) {
         debuggerStarted = true; // Prevent multiple starts, especially when running test in parallel
-        this.emitter.emit(
-          events.log.narration,
-          theme.green(`Starting debugger server...`),
-        );
         debuggerProcess = await createDebuggerProcess(
           this.config,
           this.emitter,
@@ -1801,6 +1819,7 @@ ${regression}
       }
       this.debuggerUrl = debuggerProcess.url || null; // Store the debugger URL
       this.emitter.emit(events.log.log, `This is beta software!`);
+      this.emitter.emit(events.log.log, ``);
       this.emitter.emit(
         events.log.log,
         theme.yellow(`Join our Discord for help`),
@@ -1809,6 +1828,7 @@ ${regression}
         events.log.log,
         `https://discord.com/invite/cWDFW8DzPm`,
       );
+      this.emitter.emit(events.log.log, ``);
 
       // make testdriver directory if it doesn't exist
       let testdriverFolder = path.join(this.workingDir);
@@ -1822,7 +1842,10 @@ ${regression}
       }
 
       // if the directory for thisFile doesn't exist, create it
-      if (this.cliArgs.command !== "sandbox") {
+      if (
+        this.cliArgs.command !== "sandbox" &&
+        this.cliArgs.command !== "generate"
+      ) {
         const dir = path.dirname(this.thisFile);
         if (!fs.existsSync(dir)) {
           fs.mkdirSync(dir, { recursive: true });
@@ -1847,7 +1870,10 @@ ${regression}
         await this.sdk.auth();
       }
 
-      if (this.cliArgs.command !== "sandbox") {
+      if (
+        this.cliArgs.command !== "sandbox" &&
+        this.cliArgs.command !== "generate"
+      ) {
         this.emitter.emit(
           events.log.log,
           theme.dim(`Working on ${this.thisFile}`),
@@ -2050,6 +2076,20 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     // Use the current file path from sourceMapper to find the lifecycle directory
     // If sourceMapper doesn't have a current file, use thisFile which should be the file being run
     let currentFilePath = this.sourceMapper.currentFilePath || this.thisFile;
+
+    this.emitter.emit(events.log.log, ``);
+    this.emitter.emit(events.log.log, "Running lifecycle: " + lifecycleName);
+
+    // If we still don't have a currentFilePath, fall back to the default testdriver directory
+    if (!currentFilePath) {
+      currentFilePath = path.join(
+        this.workingDir,
+        "testdriver",
+        "testdriver.yaml",
+      );
+      console.log("No currentFilePath found, using fallback:", currentFilePath);
+    }
+
     // Ensure we have an absolute path
     if (currentFilePath && !path.isAbsolute(currentFilePath)) {
       currentFilePath = path.resolve(this.workingDir, currentFilePath);
@@ -2086,6 +2126,9 @@ Please check your network connection, TD_API_KEY, or the service status.`,
         }
       }
     }
+
+    this.emitter.emit(events.log.log, lifecycleFile);
+
     if (lifecycleFile) {
       // Store current source mapping state before running lifecycle file
       const previousContext = this.sourceMapper.saveContext();
@@ -2155,7 +2198,7 @@ Please check your network connection, TD_API_KEY, or the service status.`,
     }
 
     // Move environment setup and special handling here
-    if (["edit", "run"].includes(commandName)) {
+    if (["edit", "run", "generate"].includes(commandName)) {
       await this.buildEnv(options);
     }
 

package/agent/interface.js

@@ -55,6 +55,10 @@ function createCommandDefinitions(agent) {
         "sandbox-instance": Flags.string({
           description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
         }),
+        ip: Flags.string({
+          description:
+            "Connect directly to a sandbox at the specified IP address",
+        }),
         summary: Flags.string({
           description: "Specify output file for summarize results",
         }),
@@ -68,12 +72,6 @@ function createCommandDefinitions(agent) {
         const file = normalizeFilePath(args.file);
         const testStartTime = Date.now();
 
-        // Emit test start event for the entire test execution
-        agent.emitter.emit(events.test.start, {
-          filePath: file,
-          timestamp: testStartTime,
-        });
-
         try {
           await agent.runLifecycle("prerun");
           // When run() is called through run.js CLI command, shouldExit should be true
@@ -135,6 +133,10 @@ function createCommandDefinitions(agent) {
         "sandbox-instance": Flags.string({
           description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
         }),
+        ip: Flags.string({
+          description:
+            "Connect directly to a sandbox at the specified IP address",
+        }),
         summary: Flags.string({
           description: "Specify output file for summarize results",
         }),
@@ -202,6 +204,41 @@ function createCommandDefinitions(agent) {
         console.log(`TestDriver.ai v${packageJson.version}`);
       },
     },
+
+    generate: {
+      description: "Generate test files based on current screen state",
+      args: {
+        prompt: Args.string({
+          description: "Multi-line text prompt describing what to generate",
+          required: false,
+        }),
+      },
+      flags: {
+        count: Flags.integer({
+          description: "Number of test files to generate",
+          default: 3,
+        }),
+        headless: Flags.boolean({
+          description: "Run in headless mode (no GUI)",
+          default: false,
+        }),
+        new: Flags.boolean({
+          description:
+            "Create a new sandbox instead of reconnecting to an existing one",
+          default: false,
+        }),
+        "sandbox-ami": Flags.string({
+          description: "Specify AMI ID for sandbox instance (e.g., ami-1234)",
+        }),
+        "sandbox-instance": Flags.string({
+          description: "Specify EC2 instance type for sandbox (e.g., i3.metal)",
+        }),
+      },
+      handler: async (args, flags) => {
+        // Call generate with the count and prompt
+        await agent.generate(flags.count || 3, args.prompt);
+      },
+    },
   };
 }
 

package/agent/lib/censorship.js

@@ -38,18 +38,23 @@ const censorSensitiveData = (message) => {
 
 // Function to censor sensitive data in any value (recursive for objects/arrays)
 const censorSensitiveDataDeep = (value) => {
-  if (typeof value === "string") {
-    return censorSensitiveData(value);
-  } else if (Array.isArray(value)) {
-    return value.map(censorSensitiveDataDeep);
-  } else if (value && typeof value === "object") {
-    const result = {};
-    for (const [key, val] of Object.entries(value)) {
-      result[key] = censorSensitiveDataDeep(val);
+  try {
+    if (typeof value === "string") {
+      return censorSensitiveData(value);
+    } else if (Array.isArray(value)) {
+      return value.map(censorSensitiveDataDeep);
+    } else if (value && typeof value === "object") {
+      const result = {};
+      for (const [key, val] of Object.entries(value)) {
+        result[key] = censorSensitiveDataDeep(val);
+      }
+      return result;
     }
-    return result;
+    return value;
+  } catch {
+    // If we hit any error (like circular reference), just return a safe placeholder
+    return "[Object]";
   }
-  return value;
 };
 
 // Function to update interpolation variables (for runtime updates)