testdriverai 6.0.16-canary.a5849eb.0 → 6.0.16-canary.ea858d7.0

package/agent/index.js

@@ -192,7 +192,7 @@ class TestDriverAgent extends EventEmitter2 {
 
   // fatal errors always exit the program
   // this ensure we log the error, summarize it, and exit cleanly
-  async dieOnFatal(error) {
+  async dieOnFatal(error, skipPostrun = false) {
     // Show error with source context if available
     const errorContext = this.sourceMapper.getErrorWithSourceContext(error);
     if (errorContext) {
@@ -204,9 +204,13 @@ class TestDriverAgent extends EventEmitter2 {
       );
     }
 
-    await this.summarize(error.message);
-    // Always run postrun lifecycle script, even for fatal errors
-    return await this.exit(true, false, true);
+    if (skipPostrun) {
+      this.exit(true);
+    } else {
+      await this.summarize(error.message);
+      // Always run postrun lifecycle script, even for fatal errors
+      return await this.exit(true, false, true);
+    }
   }
 
   // creates a new "thread" in which the AI is given an error
@@ -447,13 +451,14 @@ class TestDriverAgent extends EventEmitter2 {
         sourcePosition: sourcePosition,
       });
 
-      return await this.haveAIResolveError(
+      await this.haveAIResolveError(
         error,
         yaml.dump({ commands: [yml] }),
         depth,
         true,
         shouldSave,
       );
+      throw error;
     }
   }
 
@@ -1583,9 +1588,12 @@ ${regression}
       this.emitter.emit(events.log.log, `This is beta software!`);
       this.emitter.emit(
         events.log.log,
-        theme.yellow(`Join our Forums for help`),
+        theme.yellow(`Join our Discord for help`),
+      );
+      this.emitter.emit(
+        events.log.log,
+        `https://discord.com/invite/cWDFW8DzPm`,
       );
-      this.emitter.emit(events.log.log, `https://forums.testdriver.ai`);
 
       // make testdriver directory if it doesn't exist
       let testdriverFolder = path.join(this.workingDir);
@@ -1692,9 +1700,26 @@ ${regression}
       events.log.log,
       theme.gray(`- establishing connection...`),
     );
-    await this.sandbox.boot(this.config.TD_API_ROOT);
+    let ableToBoot = await this.sandbox.boot(this.config.TD_API_ROOT);
+
+    if (!ableToBoot) {
+      return await this.dieOnFatal(
+        `Unable to connect to TestDriver sandbox service at ${this.config.TD_API_ROOT}.
+Please check your network connection, TD_API_KEY, or the service status.`,
+        true,
+      );
+    }
+
     this.emitter.emit(events.log.log, theme.gray(`- authenticating...`));
-    await this.sandbox.auth(this.config.TD_API_KEY);
+    let ableToAuth = await this.sandbox.auth(this.config.TD_API_KEY);
+
+    if (!ableToAuth) {
+      return await this.dieOnFatal(
+        `Unable to authorize with TestDriver sandbox service at ${this.config.TD_API_ROOT}.
+Please check your network connection, TD_API_KEY, or the service status.`,
+        true,
+      );
+    }
   }
 
   async connectToSandboxDirect(sandboxId, persist = false) {

package/agent/lib/sandbox.js

@@ -84,10 +84,6 @@ const createSandbox = (emitter, analytics) => {
         this.socket.on("close", () => {
           clearInterval(this.heartbeat);
           // Emit a clear error event for API key issues
-          emitter.emit(events.error.fatal, {
-            message: "Socket closed. Check your API KEY (TD_API_KEY)",
-            code: "API_KEY_MISSING_OR_INVALID",
-          });
           reject();
           this.apiSocketConnected = false;
         });

package/docs/action/performance.mdx

@@ -23,9 +23,9 @@ Parallel testing allows you to split your test actions into multiple files and r
 - Use a test matrix strategy to execute these files in parallel.
 
 <Warning>
-  The `run` command is used in your test files to run other test files. This is
-  useful for breaking down large tests into smaller, more manageable pieces. To
-  run multiple separate tests, use a test matrix strategy.
+  The [`run`](/commands/run) command is used in your test files to run other
+  test files. This is useful for breaking down large tests into smaller, more
+  manageable pieces. To run multiple separate tests, use a test matrix strategy.
 </Warning>
 
 Example:
@@ -43,12 +43,12 @@ strategy:
 
 ### 2. Use optimized matching methods
 
-For actions like `hover-text`, `wait-for-text`, and `scroll-until-text`, use the `turbo` matching method instead of `ai`. The `turbo` method uses text similarity to quickly compute the most relevant match, making it about 40% faster than the `ai` method.
+For actions like [`hover-text`](/commands/hover-text), [`wait-for-text`](/commands/wait-for-text), and [`scroll-until-text`](/commands/scroll-until-text), use the `turbo` matching method instead of `ai`. The `turbo` method uses text similarity to quickly compute the most relevant match, making it about 40% faster than the `ai` method.
 
 #### Example
 
 ```yaml
-command: hover-text
+command: [`hover-text`](/commands/hover-text)
 text: Sign In
 description: login button
 action: click
@@ -59,12 +59,12 @@ method: turbo
 
 ### 3. Use `async` asserts
 
-The `assert` command supports the `async: true` property, allowing you to create non-blocking assertions. This means your tests can continue running while the assertion is being validated, saving valuable time.
+The [`assert`](/commands/assert) command supports the `async: true` property, allowing you to create non-blocking assertions. This means your tests can continue running while the assertion is being validated, saving valuable time.
 
 #### Example
 
 ```yaml
-command: assert
+command: [`assert`](/commands/assert)
 expect: The user is logged in
 async: true
 ```

package/docs/docs.json

@@ -220,8 +220,8 @@
         "href": "https://github.com/testdriverai/testdriverai"
       },
       {
-        "label": "Forums",
-        "href": "https://forums.testdriver.ai"
+        "label": "Discord",
+        "href": "https://discord.com/invite/cWDFW8DzPm"
       },
       {
         "label": "Report an Issue",

package/docs/guide/protips.mdx

@@ -13,6 +13,7 @@ When executing the `hover-image` and `hover-text` command, TestDriver may use th
 
 TestDriver may not accurately locate matches in certain cases. This can lead to unexpected behavior when using the `hover-image` or `hover-text` command. For example, if you specify a description that's too generic or similar to other images on the screen, TestDriver may not be able to identify the correct image to interact with. To improve accuracy, use specific and unique descriptions for images and text.
 When executing the `hover-image` and `hover-text` commands, TestDriver uses the provided description to identify the target image or text. If the description is too generic or matches multiple elements on the screen, TestDriver may not be able to determine which one to interact with, leading to unexpected behavior. To improve accuracy, always use specific and unique descriptions for both images and text.
+
 ### Avoid using `hover-text` for single characters (ex: `1`, `>`, `|`)
 
 To improve accuracy of our model, we do not include single characters or symbols in the matching data. This means that `hover-text` may not work as expected for these cases. If you need to interact with single characters or symbols, consider using the `match-image` command with a screenshot of the target element instead.
@@ -29,14 +30,14 @@ Assertions use the current state of the page to determine if the step passes or
 
 ## Reporting issues
 
-If you encounter any issues while using TestDriver, please report them to us. You can do this by creating a new issue in the [Feedback Forum](https://forums.testdriver.ai/c/feedback/10). We appreciate your feedback and will work to resolve any problems as quickly as possible.
+If you encounter any issues while using TestDriver, please report them to us. You can do this by creating a new issue in the [discord](https://discord.com/invite/cWDFW8DzPm). We appreciate your feedback and will work to resolve any problems as quickly as possible.
 
 <Card
   title="Report an Issue"
   icon="arrow-turn-down-right"
   iconType="duotone"
   horizontal
-  href="https://forums.testdriver.ai/c/feedback/10"
+  href="https://discord.com/invite/cWDFW8DzPm"
 >
   We'd love to hear from you!
 </Card>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "6.0.16-canary.a5849eb.0",
+  "version": "6.0.16-canary.ea858d7.0",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {

package/testdriver/examples/web/lifecycle/prerun.yaml

@@ -22,16 +22,33 @@ steps:
 
           # Define paths
           $extensionPath = Join-Path (Get-Location) "node_modules/dashcam-chrome/build"
-          $profilePath = Join-Path $env:TEMP "chrome-profile-$(Get-Random)"
+          $profilePath   = Join-Path $env:TEMP "chrome-profile-$(Get-Random)"
+          $defaultDir    = Join-Path $profilePath "Default"
+          $prefsPath     = Join-Path $defaultDir "Preferences"
 
           Write-Host "Extension path: $extensionPath"
           Write-Host "Chrome user data dir: $profilePath"
 
-          # Validate extension path
-          if (-not (Test-Path $extensionPath)) {
-              Write-Host "Extension not found at $extensionPath"
-          }
+          # Create a clean profile + Preferences to disable password manager & autofill
+          Remove-Item $profilePath -Recurse -Force -ErrorAction SilentlyContinue
+          New-Item $defaultDir -ItemType Directory -Force | Out-Null
 
+          $prefs = @{
+            "credentials_enable_service" = $false
+            "profile" = @{
+              "password_manager_enabled" = $false
+            }
+            "autofill" = @{
+              "profile_enabled"       = $false
+              "address_enabled"       = $false
+              "credit_card_enabled"   = $false
+            }
+          } | ConvertTo-Json -Depth 6
+
+          # Write Preferences before Chrome starts (ASCII/UTF8 is fine)
+          $prefs | Set-Content -Path $prefsPath -Encoding ASCII
+
+          # Build args
           $chromeArgs = @(
             "--start-maximized",
             "--load-extension=$extensionPath",
@@ -43,9 +60,6 @@ steps:
           ) -join ' '
 
           Start-Process "cmd.exe" -ArgumentList "/c", "npx @puppeteer/browsers launch chrome -- $chromeArgs"
-
-          Write-Host "Script complete."
-          exit
       - command: wait-for-text
         text: ${TD_WEBSITE}
         timeout: 60000

package/curl.sh

@@ -1,13 +0,0 @@
-curl -X POST "https://forums.testdriver.ai/users" \
-  -H "Content-Type: application/json" \
-  -H "Api-Key: YOUR_ADMIN_API_KEY" \
-  -H "Api-Username: admin" \
-  -d '{
-    "name": "Jane Doe",
-    "email": "jane.doe@example.com",
-    "username": "janedoe",
-    "active": false,
-    "approved": true,
-    "email_verified": false,
-    "staged": false
-  }'

package/debug-ast.js

@@ -1,38 +0,0 @@
-#!/usr/bin/env node
-
-const yamlAst = require("@stoplight/yaml-ast-parser");
-const yaml = require("js-yaml");
-const fs = require("fs");
-
-function debugAST() {
-  const testFile =
-    "/Users/ianjennings/Development/testdriverai/test-source-mapping.yaml";
-  const yamlContent = fs.readFileSync(testFile, "utf-8");
-
-  console.log("YAML Content:");
-  console.log(yamlContent);
-  console.log("\n" + "=".repeat(50) + "\n");
-
-  const ast = yamlAst.load(yamlContent);
-
-  console.log("AST Structure:");
-  console.log(
-    JSON.stringify(
-      ast,
-      (key, value) => {
-        if (key === "parent") return "[parent]"; // Avoid circular reference
-        return value;
-      },
-      2,
-    ),
-  );
-
-  console.log("\n" + "=".repeat(50) + "\n");
-
-  // Test with js-yaml for comparison
-  const yamlObj = yaml.load(yamlContent);
-  console.log("js-yaml result:");
-  console.log(JSON.stringify(yamlObj, null, 2));
-}
-
-debugAST();

package/testdriver/examples/doom/.env.example

@@ -1 +0,0 @@
-TD_API_KEY=

package/testdriver/examples/doom/lifecycle/provision.yaml

@@ -1,46 +0,0 @@
-version: 6.0.0
-session: 67f00511acbd9ccac373edf7
-steps:
-  - prompt: download arc browser
-    commands:
-      - command: exec
-        lang: pwsh
-        code: |
-          # === Setup ===
-          $chocoUrl = "https://www.chocolate-doom.org/downloads/3.0.1/chocolate-doom-3.0.1-win32.zip"
-          $wadUrl = "https://archive.org/download/DOOMIWADFILE/DOOM.WAD"
-          $basePath = "$env:USERPROFILE\choco-doom"
-          $extractDir = "$basePath\choco-extracted"
-          $wadFile = "$basePath\DOOM.WAD"
-          $chocoExe = "$extractDir\chocolate-doom.exe"
-          $zipPath = "$basePath\choco.zip"
-
-          # === Ensure clean folders ===
-          New-Item -ItemType Directory -Force -Path $basePath | Out-Null
-          New-Item -ItemType Directory -Force -Path $extractDir | Out-Null
-
-          # === Download Chocolate DOOM if missing ===
-          if (-not (Test-Path $zipPath)) {
-              Write-Host "Downloading Chocolate DOOM..."
-              Invoke-WebRequest -Uri $chocoUrl -OutFile $zipPath
-          }
-
-          # === Extract if not already extracted ===
-          if (-not (Test-Path $chocoExe)) {
-              Write-Host "Extracting Chocolate DOOM..."
-              Expand-Archive -LiteralPath $zipPath -DestinationPath $extractDir -Force
-          }
-
-          # === Download DOOM.WAD if missing ===
-          if (-not (Test-Path $wadFile)) {
-              Write-Host "Downloading DOOM.WAD..."
-              Invoke-WebRequest -Uri $wadUrl -OutFile $wadFile
-          }
-
-          # === Set SDL variables for software rendering ===
-          $env:SDL_VIDEODRIVER = "windows"
-          $env:SDL_RENDER_DRIVER = "software"
-
-          # === Launch DOOM ===
-          Write-Host "Launching Chocolate DOOM with software rendering..."
-          Start-Process -FilePath $chocoExe -ArgumentList "-iwad `"$wadFile`""

package/testdriver/examples/doom/readme.md

@@ -1 +0,0 @@
-Can launch doom, but can not interact with it (probably due to emulation).

package/testdriver/examples/mobile/readme.md

@@ -1,3 +0,0 @@
-# TBD
-
-- Alex is working on this

package/testdriver/examples/npm/lifecycle/prerun.yaml

@@ -1,19 +0,0 @@
-version: 6.0.0
-session: 67f00511acbd9ccac373edf7
-steps:
-  - prompt: download and run npm
-    commands:
-      - command: exec
-        lang: pwsh
-        code: |
-          # Download and install Chocolatey:
-            powershell -c "irm https://community.chocolatey.org/install.ps1|iex"
-
-            # Download and install Node.js:
-            choco install nodejs --version="22.17.1"
-
-            # Verify the Node.js version:
-            node -v # Should print "v22.17.1".
-
-            # Verify npm version:
-            npm -v # Should print "10.9.2".

package/testdriver/examples/performance/lifecycle/prerun.yaml

@@ -1,40 +0,0 @@
-version: 6.0.0
-session: 67f00511acbd9ccac373edf7
-steps:
-  - prompt: download and run npm
-    commands:
-      - command: exec
-        lang: pwsh
-        code: |
-          # Set the output log file path
-          $logFile = "$PSScriptRoot\SystemUsageLog.csv"
-
-          # Write header if the log file does not exist
-          if (!(Test-Path $logFile)) {
-              "Timestamp,CPU_Usage_Percent,Memory_Used_MB,Memory_Total_MB,Memory_Usage_Percent" | Out-File $logFile
-          }
-
-          # Monitor loop
-          while ($true) {
-              $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
-
-              # Get CPU usage
-              $cpuUsage = Get-Counter '\Processor(_Total)\% Processor Time'
-              $cpuValue = [math]::Round($cpuUsage.CounterSamples.CookedValue, 2)
-
-              # Get memory usage
-              $comp = Get-CimInstance Win32_OperatingSystem
-              $totalMemory = [math]::Round($comp.TotalVisibleMemorySize / 1024, 2)
-              $freeMemory = [math]::Round($comp.FreePhysicalMemory / 1024, 2)
-              $usedMemory = [math]::Round($totalMemory - $freeMemory, 2)
-              $memPercent = [math]::Round(($usedMemory / $totalMemory) * 100, 2)
-
-              # Append log entry
-              "$timestamp,$cpuValue,$usedMemory,$totalMemory,$memPercent" | Out-File -Append -Encoding utf8 $logFile
-
-              # Output to screen (optional)
-              Write-Host "$timestamp | CPU: $cpuValue% | RAM: $usedMemory / $totalMemory MB ($memPercent%)"
-
-              # Wait before next poll
-              Start-Sleep -Seconds 5
-          }

package/testdriver/examples/performance/monitor.yaml

@@ -1,40 +0,0 @@
-version: 6.0.0
-session: 67f00511acbd9ccac373edf7
-steps:
-  - prompt: download and run npm
-    commands:
-      - command: exec
-        lang: pwsh
-        code: |
-          # Set the output log file path
-          $logFile = "$PSScriptRoot\SystemUsageLog.csv"
-
-          # Write header if the log file does not exist
-          if (!(Test-Path $logFile)) {
-              "Timestamp,CPU_Usage_Percent,Memory_Used_MB,Memory_Total_MB,Memory_Usage_Percent" | Out-File $logFile
-          }
-
-          # Monitor loop
-          while ($true) {
-              $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
-
-              # Get CPU usage
-              $cpuUsage = Get-Counter '\Processor(_Total)\% Processor Time'
-              $cpuValue = [math]::Round($cpuUsage.CounterSamples.CookedValue, 2)
-
-              # Get memory usage
-              $comp = Get-CimInstance Win32_OperatingSystem
-              $totalMemory = [math]::Round($comp.TotalVisibleMemorySize / 1024, 2)
-              $freeMemory = [math]::Round($comp.FreePhysicalMemory / 1024, 2)
-              $usedMemory = [math]::Round($totalMemory - $freeMemory, 2)
-              $memPercent = [math]::Round(($usedMemory / $totalMemory) * 100, 2)
-
-              # Append log entry
-              "$timestamp,$cpuValue,$usedMemory,$totalMemory,$memPercent" | Out-File -Append -Encoding utf8 $logFile
-
-              # Output to screen (optional)
-              Write-Host "$timestamp | CPU: $cpuValue% | RAM: $usedMemory / $totalMemory MB ($memPercent%)"
-
-              # Wait before next poll
-              Start-Sleep -Seconds 5
-          }

package/testdriver/examples/performance/readme.md

@@ -1 +0,0 @@
-Needs `dashcam track` call

package/testdriver/examples/playwright-test-recording/hello-world.yaml

@@ -1,2 +0,0 @@
-steps:
-  - prompt: interact with the webpage

package/testdriver/examples/playwright-test-recording/lifecycle/prerun.yaml

@@ -1,28 +0,0 @@
-version: 6.0.0
-session: 67f00511acbd9ccac373edf7
-steps:
-  - prompt: launch playwright test recorder
-    commands:
-      - command: exec
-        lang: pwsh
-        code: |
-          powershell -c "irm https://community.chocolatey.org/install.ps1|iex"
-
-          # Download and install Node.js:
-          choco install nodejs --version="22.17.1"
-
-          # Verify the Node.js version:
-          node -v # Should print "v22.17.1".
-
-          # Verify npm version:
-          npm -v # Should print "10.9.2".
-
-          echo "Step 3: Installing Playwright dependencies"
-          npx playwright install --with-deps chromium
-
-          echo "Step 4: Generating Playwright script"
-          $process = Start-Process -FilePath "npx" -ArgumentList "playwright codegen --target playwright-test https://airbnb.com" -PassThru
-          Write-Output "Started Playwright with PID: $($process.Id)"
-
-          echo "Step 5: Completed"
-          exit

package/testdriver/examples/playwright-test-recording/readme.md

@@ -1 +0,0 @@
-Worked in v5, `npx` may be failing