testdriverai 7.2.3 → 7.2.10

package/test/manual/reconnect-provision.test.mjs

@@ -0,0 +1,49 @@
+/**
+ * TestDriver SDK - Reconnect Test Part 1: Provision
+ * 
+ * This test provisions a new sandbox and navigates to the login page.
+ * The sandbox ID is saved to .testdriver/last-sandbox for the next test.
+ * 
+ * The sandbox has keepAlive: 120000 (2 minutes) after disconnect.
+ * Run reconnect-signin.test.mjs within 2 minutes of this test completing.
+ * 
+ * Usage:
+ *   1. npm test -- test/testdriver/reconnect-provision.test.mjs
+ *   2. (within 2 minutes) npm test -- test/testdriver/reconnect-signin.test.mjs
+ */
+
+import { afterAll, describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Reconnect Test - Part 1: Provision", () => {
+
+  afterAll(async () => {
+    // Explicitly DO NOT disconnect - we want the sandbox to stay alive
+    // for the reconnect test. The sandbox will auto-terminate after keepAlive TTL.
+    console.log("\n⚠️  NOT disconnecting - sandbox will stay alive for ~2 minutes (keepAlive: 120000)");
+    console.log("   Run reconnect-signin.test.mjs within 2 minutes to test reconnect\n");
+  });
+
+  it("should provision sandbox and navigate to login page", async (context) => {
+
+    const testdriver = TestDriver(context, { newSandbox: true, headless: false });
+    
+    // Provision Chrome and navigate to login page
+    await testdriver.provision.chrome({
+      url: 'http://testdriver-sandbox.vercel.app/login',
+    });
+
+
+    // Verify we're on the login page
+    const result = await testdriver.assert("I can see a Sign In button");
+    expect(result).toBeTruthy();
+
+    // Get the sandbox ID that was saved
+    const lastSandbox = testdriver.getLastSandboxId();
+    console.log("\n✅ Sandbox provisioned:", lastSandbox?.sandboxId);
+    console.log("   Sandbox info saved to .testdriver/last-sandbox");
+    
+    expect(lastSandbox).toBeTruthy();
+    expect(lastSandbox.sandboxId).toBeTruthy();
+  });
+});

package/test/manual/reconnect-signin.test.mjs

@@ -0,0 +1,41 @@
+/**
+ * TestDriver SDK - Reconnect Test Part 2: Sign In
+ * 
+ * This test reconnects to the sandbox provisioned by reconnect-provision.test.mjs
+ * and clicks the Sign In button.
+ * 
+ * IMPORTANT: Run this within 2 minutes of reconnect-provision.test.mjs completing.
+ * The sandbox auto-terminates after the keepAlive TTL (default 2 minutes).
+ * 
+ * Usage:
+ *   1. npm test -- test/testdriver/reconnect-provision.test.mjs
+ *   2. (within 2 minutes) npm test -- test/testdriver/reconnect-signin.test.mjs
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Reconnect Test - Part 2: Sign In", () => {
+
+  it("should reconnect to existing sandbox and click Sign In", async (context) => {
+
+    const testdriver = TestDriver(context, { newSandbox: true, headless: false, reconnect: true });
+    
+    // Provision Chrome and navigate to login page
+    await testdriver.provision.chrome({
+      url: 'http://testdriver-sandbox.vercel.app/login',
+    });
+
+    // Click on Sign In button - the page should already be loaded from provision test
+    const signInButton = await testdriver.find(
+      "Sign In, black button below the password field",
+    );
+    await signInButton.click();
+
+    // Assert that an error shows that fields are required
+    const result = await testdriver.assert(
+      "an error shows that fields are required",
+    );
+    expect(result).toBeTruthy();
+  });
+});

package/test/testdriver/act.test.mjs

@@ -0,0 +1,30 @@
+/**
+ * TestDriver SDK - Act Test (Vitest)
+ * Tests the AI exploratory loop (act) functionality
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Act Test", () => {
+  it("should use act to search for testdriver on Google", async (context) => {
+    const testdriver = TestDriver(context, { newSandbox: true });
+    
+    // provision.chrome() automatically calls ready() and starts dashcam
+    await testdriver.provision.chrome({
+      url: 'https://www.google.com',
+    });
+
+    // Use act to search for testdriver
+    let actRes = await testdriver.act("click on the empty search box, type 'testdriver', and hit enter. do not click the plus button in the search bar");
+
+    console.log("Act response:", actRes);
+
+    // Assert the search results are displayed
+    const result = await testdriver.assert(
+      "search results for testdriver are visible",
+    );
+
+    expect(result).toBeTruthy();
+  });
+});

package/test/testdriver/ai.test.mjs

@@ -0,0 +1,30 @@
+/**
+ * TestDriver SDK - AI Test (Vitest)
+ * Tests the AI exploratory loop (ai) functionality
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("AI Test", () => {
+  it("should use ai to search for testdriver on Google", async (context) => {
+    const testdriver = TestDriver(context, { newSandbox: true });
+    
+    // provision.chrome() automatically calls ready() and starts dashcam
+    await testdriver.provision.chrome({
+      url: 'https://duckduckgo.com',
+    });
+
+    // Use ai to search for testdriver
+    let aiRes = await testdriver.ai("click on the empty search box, type 'testdriver', and hit enter.");
+
+    console.log("AI response:", aiRes);
+
+    // Assert the search results are displayed
+    const result = await testdriver.assert(
+      "search results for testdriver are visible",
+    );
+
+    expect(result).toBeTruthy();
+  });
+});

package/test/testdriver/assert.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Assert Test", () => {
   it("should assert the testdriver login page shows", async (context) => {
-    const testdriver = TestDriver(context, { newSandbox: true });
+    const testdriver = TestDriver(context, { newSandbox: true, headless: false });
     
     // provision.chrome() automatically calls ready() and starts dashcam
     await testdriver.provision.chrome({

package/test/testdriver/hover-text.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Hover Text Test", () => {
   it("should click Sign In and verify error message", async (context) => {
-    const testdriver = TestDriver(context, { headless: true, newSandbox: true, cacheKey: 'hover-text-test' });
+    const testdriver = TestDriver(context, { headless: false, newSandbox: true, cacheKey: 'hover-text-test' });
     await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
 
     // Click on Sign In button using new find() API

package/test/testdriver/setup/testHelpers.mjs

@@ -5,8 +5,6 @@
 
 import crypto from "crypto";
 import { config } from "dotenv";
-import fs from "fs";
-import os from "os";
 import path, { dirname } from "path";
 import { fileURLToPath } from "url";
 import TestDriver from "../../../sdk.js";
@@ -51,85 +49,6 @@ console.log(
   process.env.TD_OS || "Not set (will default to linux)",
 );
 
-// Global test results storage
-const testResults = {
-  tests: [],
-  startTime: Date.now(),
-};
-
-/**
- * Store test result with dashcam URL
- * @param {string} testName - Name of the test
- * @param {string} testFile - Test file path
- * @param {string|null} dashcamUrl - Dashcam URL if available
- * @param {Object} sessionInfo - Session information
- */
-export function storeTestResult(
-  testName,
-  testFile,
-  dashcamUrl,
-  sessionInfo = {},
-) {
-
-  // Extract replay object ID from dashcam URL
-  let replayObjectId = null;
-  if (dashcamUrl) {
-    const replayIdMatch = dashcamUrl.match(/\/replay\/([^?]+)/);
-    replayObjectId = replayIdMatch ? replayIdMatch[1] : null;
-    if (replayObjectId) {
-      console.log(`   Replay Object ID: ${replayObjectId}`);
-    }
-  }
-
-  testResults.tests.push({
-    name: testName,
-    file: testFile,
-    dashcamUrl,
-    replayObjectId,
-    sessionId: sessionInfo.sessionId,
-    timestamp: new Date().toISOString(),
-  });
-}
-
-/**
- * Get all test results
- * @returns {Object} All collected test results
- */
-export function getTestResults() {
-  return {
-    ...testResults,
-    endTime: Date.now(),
-    duration: Date.now() - testResults.startTime,
-  };
-}
-
-/**
- * Save test results to a JSON file
- * @param {string} outputPath - Path to save the results
- */
-export function saveTestResults(outputPath = "test-results/sdk-summary.json") {
-  const results = getTestResults();
-  const dir = path.dirname(outputPath);
-
-  // Create directory if it doesn't exist
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
-
-  fs.writeFileSync(outputPath, JSON.stringify(results, null, 2));
-  console.log(`\n📊 Test results saved to: ${outputPath}`);
-
-  // Also print dashcam URLs to console
-  console.log("\n🎥 Dashcam URLs:");
-  results.tests.forEach((test) => {
-    if (test.dashcamUrl) {
-      console.log(`  ${test.name}: ${test.dashcamUrl}`);
-    }
-  });
-
-  return results;
-}
-
 /**
  * Intercept console logs and forward to TestDriver sandbox
  * @param {TestDriver} client - TestDriver client instance
@@ -494,21 +413,9 @@ export async function teardownTest(client, options = {}) {
       console.log("⏭️  Postrun skipped (disabled in options)");
     }
 
-    // Write test result to a file for the reporter to pick up (cross-process communication)
+    // Use Vitest's task.meta for cross-process communication with the reporter
     if (options.task) {
-      const testResultFile = path.join(
-        os.tmpdir(),
-        "testdriver-results",
-        `${options.task.id}.json`,
-      );
-
       try {
-        // Ensure directory exists
-        const dir = path.dirname(testResultFile);
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir, { recursive: true });
-        }
-
         // Get test file path - make it relative to project root
         const absolutePath =
           options.task.file?.filepath || options.task.file?.name || "unknown";
@@ -523,33 +430,15 @@ export async function teardownTest(client, options = {}) {
           testOrder = options.task.suite.tasks.indexOf(options.task);
         }
 
-        // Note: Duration is calculated by Vitest and passed via result.duration
-        // We include it in the test result file so the reporter can use it
-
-        // Get duration from Vitest result
-        const result = options.task.result?.();
-        const duration = result?.duration || 0;
-
-        // Write test result with dashcam URL, platform, and metadata
-        const testResult = {
-          testId: options.task.id,
-          testName: options.task.name,
-          testFile: testFile,
-          testOrder: testOrder,
-          dashcamUrl: dashcamUrl,
-          replayObjectId: dashcamUrl
-            ? dashcamUrl.match(/\/replay\/([^?]+)/)?.[1]
-            : null,
-          platform: client.os, // Include platform from SDK client (source of truth)
-          timestamp: Date.now(),
-          duration: duration, // Include duration from Vitest
-        };
-
-        fs.writeFileSync(testResultFile, JSON.stringify(testResult, null, 2));
-   
+        // Set metadata on task for the reporter to pick up
+        options.task.meta.dashcamUrl = dashcamUrl;
+        options.task.meta.platform = client.os; // Include platform from SDK client (source of truth)
+        options.task.meta.testFile = testFile;
+        options.task.meta.testOrder = testOrder;
+        options.task.meta.sessionId = client.getSessionId?.() || null;
       } catch (error) {
         console.error(
-          `[TestHelpers] ❌ Failed to write test result file:`,
+          `[TestHelpers] ❌ Failed to set test metadata:`,
           error.message,
         );
       }

package/test/testdriver/windows-installer.test.mjs

@@ -0,0 +1,61 @@
+/**
+ * TestDriver SDK - Windows Installer Example (Vitest)
+ * 
+ * This example demonstrates how to download and install a Windows application
+ * using PowerShell commands, then launch and interact with it.
+ * 
+ * Based on the v6 GitButler provisioning workflow.
+ * 
+ * Run: TD_OS=windows npx vitest run examples/windows-installer.test.mjs
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+const isLinux = (process.env.TD_OS || "linux") === "linux";
+
+describe("Windows App Installation", () => {
+    
+  it.skipIf(isLinux)("should download, install, and launch GitButler on Windows", async (context) => {
+    // Alternative approach using provision.installer helper
+    const testdriver = TestDriver(context, { 
+      newSandbox: true,
+      os: 'windows'
+    });
+
+    // Download the MSI installer
+    const installerPath = await testdriver.provision.installer({
+      url: 'https://app.gitbutler.com/downloads/release/windows/x86_64/msi',
+      launch: false, // Don't auto-launch, we'll install manually
+    });
+
+    // The installer should be an .msi or .exe file
+    expect(installerPath).toMatch(/\.(msi|exe)$/i);
+
+    // Install the MSI silently (check which type it is)
+    if (installerPath.toLowerCase().endsWith('.msi')) {
+      await testdriver.exec('pwsh', 
+        `Start-Process msiexec.exe -ArgumentList "/i \`"${installerPath}\`" /qn /norestart" -Wait`,
+        120000
+      );
+    } else {
+      await testdriver.exec('pwsh', 
+        `Start-Process "${installerPath}" -ArgumentList "/S" -Wait`,
+        120000
+      );
+    }
+
+    // Verify installation by checking if executable exists
+    const verifyScript = `
+      $exePath = "C:\\Program Files\\GitButler\\gitbutler-tauri.exe"
+      if (Test-Path $exePath) {
+          Write-Host "GitButler installed successfully at $exePath"
+      } else {
+          Write-Error "GitButler not found"
+          exit 1
+      }
+    `;
+
+    await testdriver.exec('pwsh', verifyScript, 5000);
+  });
+});

package/tests/example.test.js

@@ -0,0 +1,33 @@
+import { test, expect } from 'vitest';
+import { TestDriver } from 'testdriverai/vitest/hooks';
+import { login } from './login.js';
+
+test('should login and add item to cart', async (context) => {
+
+  // Create TestDriver instance - automatically connects to sandbox
+  const testdriver = TestDriver(context);
+
+  // Launch chrome and navigate to demo app
+  await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
+
+  // Use the login snippet to handle authentication
+  // This demonstrates how to reuse test logic across multiple tests
+  await login(testdriver);
+
+  // Add item to cart
+  const addToCartButton = await testdriver.find(
+    'add to cart button under TestDriver Hat'
+  );
+  await addToCartButton.click();
+
+  // Open cart
+  const cartButton = await testdriver.find(
+    'cart button in the top right corner'
+  );
+  await cartButton.click();
+
+  // Verify item in cart
+  const result = await testdriver.assert('TestDriver Hat is in the cart');
+  expect(result).toBeTruthy();
+  
+});

package/tests/login.js

@@ -0,0 +1,28 @@
+/**
+ * Login snippet - reusable login function
+ * 
+ * This demonstrates how to create reusable test snippets that can be
+ * imported and used across multiple test files.
+ */
+export async function login(testdriver) {
+
+  // The password is displayed on screen, have TestDriver extract it
+  const password = await testdriver.extract('the password');
+
+  // Find the username field
+  const usernameField = await testdriver.find(
+    'Username, label above the username input field on the login form'
+  );
+  await usernameField.click();
+
+  // Type username
+  await testdriver.type('standard_user');
+
+  // Enter password form earlier 
+  // Marked as secret so it's not logged or stored
+  await testdriver.pressKeys(['tab']);
+  await testdriver.type(password, { secret: true });
+
+  // Submit the form
+  await testdriver.find('submit button on the login form').click();
+}

package/tests/table-sort-enrollments.test.mjs

@@ -0,0 +1,72 @@
+/**
+ * Test case 7: Sort by Enrollments (ascending, numeric)
+ * 
+ * 1. Open page: https://practicetestautomation.com/practice-test-table/
+ * 2. Set Sort by = Enrollments
+ * 3. Verify visible rows are ordered from smallest to largest enrollment
+ * 4. Verify numbers with commas sort correctly
+ * 5. Fails if the sort is lexicographic (string-based) instead of numeric
+ * 
+ * EXPECTED: This test should PASS if enrollments are sorted ascending numerically
+ *           This test should FAIL if sort is lexicographic or descending
+ */
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+
+describe("Table Sort - Enrollments", () => {
+
+  it("should sort enrollments numerically ascending", async (context) => {
+    const testdriver = TestDriver(context, { 
+      newSandbox: true, 
+      headless: false 
+    });
+    
+    // Step 1: Open the page
+    await testdriver.provision.chrome({
+      url: 'https://practicetestautomation.com/practice-test-table/',
+    });
+
+    // Scroll down to see the Sort by dropdown
+    await testdriver.scroll("down");
+    await testdriver.scroll("down");
+    
+    // Step 2: Find and click the Sort by dropdown
+    const sortDropdown = await testdriver.find("Sort by dropdown", { timeout: 15000 });
+    console.log("Sort by dropdown found:", sortDropdown.found());
+    expect(sortDropdown.found()).toBeTruthy();
+    await sortDropdown.click();
+    
+    // Select "Enrollments" from the dropdown options
+    const enrollmentsOption = await testdriver.find("Enrollments option", { timeout: 10000 });
+    console.log("Enrollments option found:", enrollmentsOption.found());
+    expect(enrollmentsOption.found()).toBeTruthy();
+    await enrollmentsOption.click();
+    
+    // Scroll to the TOP of the page to see the first rows of the sorted table
+    await testdriver.scroll("up");
+    await testdriver.scroll("up");
+    await testdriver.scroll("up");
+    
+    // Small wait for sort to complete
+    await testdriver.pressKeys([""]); // No-op to add small delay
+    
+    // Scroll down just enough to see the table header and first data rows
+    await testdriver.scroll("down");
+    
+    // Step 3: Verify the sort is NUMERIC ASCENDING (smallest to largest)
+    // For numeric ascending: 10 < 11 < 50 < 1000 < 1200 < 1365
+    // If lexicographic: "1,000" < "10" < "11" < "1,200" < "1365" < "50" (wrong - strings compare char by char)
+    // 
+    // The TEST PASSES if first rows show small numbers like 10, 11, 50
+    // The TEST FAILS if first rows show large numbers like 1000, 1200, 1365
+    const sortResult = await testdriver.assert(
+      "The table is sorted by Enrollments in ASCENDING numeric order. " +
+      "The FIRST visible enrollment numbers in the table should be the SMALLEST values. " +
+      "For example, if the data contains values like 10, 11, 50, 1000, and 1365, " +
+      "then 10 or 11 should appear at the TOP of the sorted table. " +
+      "If a large number like 1365 appears first, the sort is WRONG (descending or lexicographic)."
+    );
+    console.log("Numeric ascending sort assertion result:", sortResult);
+    expect(sortResult).toBeTruthy();
+  });
+});

package/tests/table-sort-experiment.test.mjs

@@ -0,0 +1,42 @@
+/**
+ * Experiment file - reconnects to existing sandbox
+ * Test case 7: Sort by Enrollments (ascending, numeric)
+ * Run AFTER table-sort-setup.test.mjs passes (within 2 minutes)
+ */
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+
+describe("Table Sort Experiment", () => {
+
+  it("should sort by Enrollments and verify numeric ordering", async (context) => {
+    const testdriver = TestDriver(context, { 
+      newSandbox: true, 
+      headless: false,
+      reconnect: true  // Reconnects to last sandbox
+    });
+    
+    // NO provision here! The sandbox is already running from setup.test.mjs
+    
+    // Click on the Sort By dropdown and select Enrollments
+    const sortDropdown = await testdriver.find("Sort By dropdown");
+    console.log("Sort dropdown found:", sortDropdown.found());
+    await sortDropdown.click();
+    
+    // Select Enrollments from the dropdown
+    const enrollmentsOption = await testdriver.find("Enrollments option in dropdown");
+    console.log("Enrollments option found:", enrollmentsOption.found());
+    await enrollmentsOption.click();
+    
+    // Verify the table is now sorted by enrollments (ascending, numeric)
+    // The numbers should be ordered from smallest to largest: 10, 11, 50, 1000, 1,200
+    // If lexicographic: 1,000, 1,200, 10, 11, 50 (wrong - strings sort by first character)
+    const result = await testdriver.assert("The table shows enrollment numbers in ascending numeric order where smaller numbers like 10 or 11 appear before larger numbers like 1000 or 1200");
+    console.log("Sort assertion result:", result);
+    expect(result).toBeTruthy();
+    
+    // Also verify numbers with commas sort correctly (1,200 should be after 1,000)
+    const commaResult = await testdriver.assert("Numbers with commas like 1,000 and 1,200 are sorted correctly as numbers, not as text");
+    console.log("Comma sort assertion:", commaResult);
+    expect(commaResult).toBeTruthy();
+  });
+});

package/tests/table-sort-setup.test.mjs

@@ -0,0 +1,59 @@
+/**
+ * Test case 7: Sort by Enrollments (ascending, numeric)
+ * 
+ * 1. Open page: https://practicetestautomation.com/practice-test-table/
+ * 2. Set Sort by = Enrollments
+ * 3. Verify visible rows are ordered from smallest to largest enrollment
+ * 4. Verify numbers with commas sort correctly
+ * 5. Fails if the sort is lexicographic
+ */
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+
+describe("Table Sort - Enrollments", () => {
+
+  it("should sort enrollments numerically ascending", async (context) => {
+    const testdriver = TestDriver(context, { 
+      newSandbox: true, 
+      headless: false 
+    });
+    
+    // Step 1: Open the page
+    await testdriver.provision.chrome({
+      url: 'https://practicetestautomation.com/practice-test-table/',
+    });
+
+    // Wait for page to load - find the Sort By dropdown
+    const sortDropdown = await testdriver.find("Sort By dropdown or select box", { timeout: 30000 });
+    console.log("Sort dropdown found:", sortDropdown.found());
+    expect(sortDropdown.found()).toBeTruthy();
+    
+    // Step 2: Click the Sort By dropdown
+    await sortDropdown.click();
+    
+    // Select "Enrollments" from the dropdown options
+    const enrollmentsOption = await testdriver.find("Enrollments option in the dropdown list", { timeout: 10000 });
+    console.log("Enrollments option found:", enrollmentsOption.found());
+    await enrollmentsOption.click();
+    
+    // Step 3 & 4: Verify the sort is numeric ascending (not lexicographic)
+    // If numeric: smaller numbers (10, 11, 50) come before larger numbers (1000, 1200)
+    // If lexicographic: would sort as strings (1,000 before 10 because '1' < '1')
+    const sortResult = await testdriver.assert(
+      "The table enrollment column is sorted in ascending NUMERIC order: " +
+      "smaller enrollment numbers like 10, 11, 50 appear BEFORE larger numbers like 1000 or 1200. " +
+      "This is correct numeric sorting, not alphabetical/lexicographic sorting."
+    );
+    console.log("Numeric sort assertion result:", sortResult);
+    expect(sortResult).toBeTruthy();
+    
+    // Step 5: Verify numbers with commas are handled correctly
+    // 1,200 should be treated as 1200 (greater than 1000), not as a string
+    const commaResult = await testdriver.assert(
+      "Any enrollment numbers with commas (like 1,200 or 1,000) are sorted correctly as numbers, " +
+      "with 1,000 appearing before 1,200 in ascending order"
+    );
+    console.log("Comma number sorting result:", commaResult);
+    expect(commaResult).toBeTruthy();
+  });
+});

package/vitest.config.mjs

@@ -10,8 +10,10 @@ export default defineConfig({
   test: {
     testTimeout: 900000,
     hookTimeout: 900000,
+    disableConsoleIntercept: true,
+    maxConcurrency: 3,
     reporters: [
-      'verbose',
+      'default',
       TestDriver(),
       ['junit', { outputFile: 'test-report.junit.xml' }]
     ],