testdriverai 6.2.2 → 7.1.0

package/testdriver/acceptance-sdk/setup/testHelpers.mjs

@@ -0,0 +1,420 @@
+/**
+ * Repo-Specific Test Helpers
+ * 
+ * These helpers are specific to this repository's test infrastructure.
+ * For reusable plugin helpers, import from 'testdriverai/vitest' (or src/vitest/index.mjs locally)
+ */
+
+import crypto from "crypto";
+import { config } from "dotenv";
+import fs from "fs";
+import os from "os";
+import path, { dirname } from "path";
+import { fileURLToPath } from "url";
+
+// Import TestDriver SDK locally (for repo development)
+import TestDriver from "../../../sdk.js";
+
+// Import plugin helpers
+import {
+  addDashcamLog,
+  authDashcam,
+  launchChrome,
+  launchChromeExtension,
+  launchChromeForTesting,
+  runPostrun,
+  runPrerun,
+  runPrerunChromeExtension,
+  runPrerunChromeForTesting,
+  startDashcam,
+  stopDashcam,
+  waitForPage,
+} from "../../../src/vitest/lifecycle.mjs";
+
+import {
+  retryAsync,
+  setupEventLogging,
+  sleep,
+  waitFor,
+} from "../../../src/vitest/utils.mjs";
+
+// Re-export plugin lifecycle helpers for backward compatibility
+export {
+  addDashcamLog,
+  authDashcam,
+  launchChrome,
+  launchChromeExtension,
+  launchChromeForTesting,
+  runPostrun,
+  runPrerun,
+  runPrerunChromeExtension,
+  runPrerunChromeForTesting,
+  startDashcam,
+  stopDashcam,
+  waitForPage
+};
+
+// Re-export plugin utilities for backward compatibility
+  export {
+    retryAsync,
+    setupEventLogging,
+    sleep,
+    waitFor
+  };
+
+// Get the directory of the current module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Load environment variables from .env file in the project root
+const envPath = path.resolve(__dirname, "../../../.env");
+config({ path: envPath });
+
+// Log loaded env vars for debugging
+console.log("🔧 Environment variables loaded from:", envPath);
+console.log("   TD_API_KEY:", process.env.TD_API_KEY ? "✓ Set" : "✗ Not set");
+console.log("   TD_API_ROOT:", process.env.TD_API_ROOT || "Not set");
+console.log("   TD_OS:", process.env.TD_OS || "Not set (will default to linux)");
+
+// =============================================================================
+// TEST RESULTS STORAGE (Repo-specific CI/CD integration)
+// =============================================================================
+
+const testResults = {
+  tests: [],
+  startTime: Date.now(),
+};
+
+/**
+ * Store test result with dashcam URL
+ * @param {string} testName - Name of the test
+ * @param {string} testFile - Test file path
+ * @param {string|null} dashcamUrl - Dashcam URL if available
+ * @param {Object} sessionInfo - Session information
+ */
+export function storeTestResult(testName, testFile, dashcamUrl, sessionInfo = {}) {
+  console.log(`📝 Storing test result: ${testName}`);
+  console.log(`   Dashcam URL: ${dashcamUrl || "none"}`);
+
+  let replayObjectId = null;
+  if (dashcamUrl) {
+    const replayIdMatch = dashcamUrl.match(/\/replay\/([^?]+)/);
+    replayObjectId = replayIdMatch ? replayIdMatch[1] : null;
+    if (replayObjectId) {
+      console.log(`   Replay Object ID: ${replayObjectId}`);
+    }
+  }
+
+  testResults.tests.push({
+    name: testName,
+    file: testFile,
+    dashcamUrl,
+    replayObjectId,
+    sessionId: sessionInfo.sessionId,
+    timestamp: new Date().toISOString(),
+  });
+}
+
+/**
+ * Get all test results
+ * @returns {Object} All collected test results
+ */
+export function getTestResults() {
+  return {
+    ...testResults,
+    endTime: Date.now(),
+    duration: Date.now() - testResults.startTime,
+  };
+}
+
+/**
+ * Save test results to a JSON file
+ * @param {string} outputPath - Path to save the results
+ */
+export function saveTestResults(outputPath = "test-results/sdk-summary.json") {
+  const results = getTestResults();
+  const dir = path.dirname(outputPath);
+
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  fs.writeFileSync(outputPath, JSON.stringify(results, null, 2));
+  console.log(`\n📊 Test results saved to: ${outputPath}`);
+
+  console.log("\n🎥 Dashcam URLs:");
+  results.tests.forEach((test) => {
+    if (test.dashcamUrl) {
+      console.log(`  ${test.name}: ${test.dashcamUrl}`);
+    }
+  });
+
+  return results;
+}
+
+// =============================================================================
+// REPO-SPECIFIC CLIENT CREATION
+// =============================================================================
+
+/**
+ * Create a configured TestDriver client for this repo's tests
+ * Uses local SDK path for development
+ * @param {Object} options - Additional options
+ * @returns {TestDriver} Configured client
+ */
+export function createTestClient(options = {}) {
+  if (!process.env.TD_API_KEY) {
+    console.error("\n❌ Error: TD_API_KEY is not set!");
+    console.error("Please set it in one of the following ways:");
+    console.error("  1. Create a .env file in the project root with: TD_API_KEY=your_key");
+    console.error("  2. Pass it as an environment variable: TD_API_KEY=your_key npm run test:sdk");
+    console.error("  3. Export it in your shell: export TD_API_KEY=your_key\n");
+    throw new Error("TD_API_KEY environment variable is required");
+  }
+
+  const osConfig = process.env.TEST_PLATFORM || "linux";
+  const { task, ...clientOptions } = options;
+  const taskId = task?.id || task?.name || null;
+
+  const client = new TestDriver(process.env.TD_API_KEY, {
+    resolution: "1366x768",
+    analytics: true,
+    os: osConfig,
+    apiKey: process.env.TD_API_KEY,
+    apiRoot: process.env.TD_API_ROOT || "https://testdriver-api.onrender.com",
+    newSandbox: true,
+    ...clientOptions,
+  });
+
+  console.log("🔧 createTestClient: SDK created, cacheThresholds =", client.cacheThresholds);
+  console.log(`[TestHelpers] Client OS configured as: ${client.os}`);
+
+  if (taskId) {
+    console.log(`[TestHelpers] Storing task ID on client: ${taskId}`);
+    client.vitestTaskId = taskId;
+  }
+
+  if (process.env.DEBUG_EVENTS === "true") {
+    setupEventLogging(client);
+  }
+
+  return client;
+}
+
+// =============================================================================
+// REPO-SPECIFIC LOGIN HELPER (for TestDriver Sandbox)
+// =============================================================================
+
+/**
+ * Perform login flow on TestDriver Sandbox
+ * This is specific to http://testdriver-sandbox.vercel.app/login
+ * @param {TestDriver} client - TestDriver client
+ * @param {string} username - Username (default: 'standard_user')
+ * @param {string} password - Password (default: retrieved from screen)
+ */
+export async function performLogin(client, username = "standard_user", password = null) {
+  await client.focusApplication("Google Chrome");
+
+  if (!password) {
+    password = await client.remember("the password");
+  }
+
+  const usernameField = await client.find(
+    "Username, label above the username input field on the login form",
+  );
+  await usernameField.click();
+  await client.type(username);
+
+  await client.pressKeys(["tab"]);
+  await client.type(password, { secret: true });
+
+  await client.pressKeys(["tab"]);
+  await client.pressKeys(["enter"]);
+}
+
+// =============================================================================
+// SUITE TEST RUN MANAGEMENT (CI/CD Integration)
+// =============================================================================
+
+/**
+ * Initialize a test run for the entire suite
+ * @param {Object} suiteTask - Vitest suite task context
+ * @returns {Promise<Object>} Test run info { runId, testRunDbId, token }
+ */
+export async function initializeSuiteTestRun(suiteTask) {
+  const apiKey = process.env.TD_API_KEY;
+  const apiRoot = process.env.TD_API_ROOT || "https://testdriver-api.onrender.com";
+
+  if (!apiKey || !globalThis.__testdriverPlugin) {
+    console.log(`[TestHelpers] Skipping suite test run initialization - no API key or plugin`);
+    return null;
+  }
+
+  const existingRun = globalThis.__testdriverPlugin.getSuiteTestRun(suiteTask.id);
+  if (existingRun) {
+    console.log(`[TestHelpers] Test run already exists for suite: ${existingRun.runId}`);
+    return existingRun;
+  }
+
+  try {
+    console.log(`[TestHelpers] Initializing test run for suite: ${suiteTask.name}`);
+
+    const token = await globalThis.__testdriverPlugin.authenticateWithApiKey(apiKey, apiRoot);
+    console.log(`[TestHelpers] ✅ Authenticated for suite`);
+
+    const runId = `${Date.now()}-${crypto.randomBytes(4).toString("hex")}`;
+    const testFile = suiteTask.file?.name || "unknown";
+    const testRunData = {
+      runId,
+      suiteName: suiteTask.name || testFile,
+    };
+
+    const testRunResponse = await globalThis.__testdriverPlugin.createTestRunDirect(
+      token,
+      apiRoot,
+      testRunData,
+    );
+    const testRunDbId = testRunResponse.data?.id;
+
+    const runInfo = { runId, testRunDbId, token };
+    globalThis.__testdriverPlugin.setSuiteTestRun(suiteTask.id, runInfo);
+
+    process.env.TD_TEST_RUN_ID = runId;
+    process.env.TD_TEST_RUN_DB_ID = testRunDbId;
+    process.env.TD_TEST_RUN_TOKEN = token;
+
+    console.log(`[TestHelpers] ✅ Created test run for suite: ${runId} (DB ID: ${testRunDbId})`);
+    return runInfo;
+  } catch (error) {
+    console.error(`[TestHelpers] ❌ Failed to initialize suite test run:`, error.message);
+    return null;
+  }
+}
+
+// =============================================================================
+// SETUP/TEARDOWN HELPERS
+// =============================================================================
+
+/**
+ * Setup function to run before each test
+ * @param {TestDriver} client - TestDriver client
+ * @param {Object} options - Connection options
+ * @returns {Promise<Object>} Sandbox instance
+ */
+export async function setupTest(client, options = {}) {
+  await client.auth();
+  const instance = await client.connect({ ...options });
+
+  if (options.prerun !== false) {
+    await runPrerun(client);
+  }
+
+  return instance;
+}
+
+/**
+ * Teardown function to run after each test
+ * @param {TestDriver} client - TestDriver client
+ * @param {Object} options - Teardown options
+ * @returns {Promise<Object>} Session info including dashcam URL
+ */
+export async function teardownTest(client, options = {}) {
+  let dashcamUrl = options.dashcamUrl || null;
+
+  console.log("🧹 Running teardown...");
+
+  try {
+    if (options.postrun !== false && !dashcamUrl) {
+      dashcamUrl = await runPostrun(client);
+
+      if (dashcamUrl && options.task) {
+        const replayIdMatch = dashcamUrl.match(/\/replay\/([^?]+)/);
+        const replayObjectId = replayIdMatch ? replayIdMatch[1] : null;
+
+        console.log(`🎥 Dashcam URL: ${dashcamUrl}`);
+        if (replayObjectId) {
+          console.log(`📝 Replay Object ID: ${replayObjectId}`);
+        }
+
+        options.task.meta.testdriverDashcamUrl = dashcamUrl;
+        options.task.meta.testdriverReplayObjectId = replayObjectId;
+        console.log(`[TestHelpers] ✅ Stored dashcam URL in task.meta for test: ${options.task.name}`);
+      }
+    } else {
+      console.log("⏭️  Postrun skipped (disabled in options)");
+    }
+
+    // Write test result to temp file for reporter
+    if (options.task) {
+      const testResultFile = path.join(
+        os.tmpdir(),
+        "testdriver-results",
+        `${options.task.id}.json`,
+      );
+
+      try {
+        const dir = path.dirname(testResultFile);
+        if (!fs.existsSync(dir)) {
+          fs.mkdirSync(dir, { recursive: true });
+        }
+
+        const testFile = options.task.file?.filepath || options.task.file?.name || "unknown";
+        let testOrder = 0;
+        if (options.task.suite && options.task.suite.tasks) {
+          testOrder = options.task.suite.tasks.indexOf(options.task);
+        }
+
+        const result = options.task.result?.();
+        const duration = result?.duration || 0;
+
+        const testResult = {
+          testId: options.task.id,
+          testName: options.task.name,
+          testFile: testFile,
+          testOrder: testOrder,
+          dashcamUrl: dashcamUrl,
+          replayObjectId: dashcamUrl ? dashcamUrl.match(/\/replay\/([^?]+)/)?.[1] : null,
+          platform: client.os,
+          timestamp: Date.now(),
+          duration: duration,
+        };
+
+        fs.writeFileSync(testResultFile, JSON.stringify(testResult, null, 2));
+        console.log(`[TestHelpers] ✅ Wrote test result to file: ${testResultFile}`);
+      } catch (error) {
+        console.error(`[TestHelpers] ❌ Failed to write test result file:`, error.message);
+      }
+    }
+  } catch (error) {
+    console.error("❌ Error in postrun:", error);
+    console.error("❌ Error stack:", error.stack);
+  } finally {
+    if (options.disconnect !== false) {
+      console.log("🔌 Disconnecting client...");
+      try {
+        await client.disconnect();
+        console.log("✅ Client disconnected");
+      } catch (disconnectError) {
+        console.error("❌ Error disconnecting:", disconnectError.message);
+      }
+    } else {
+      console.log("⏭️  Disconnect skipped (disabled in options)");
+    }
+  }
+
+  let replayObjectId = null;
+  if (dashcamUrl) {
+    const replayIdMatch = dashcamUrl.match(/\/replay\/([^?]+)/);
+    replayObjectId = replayIdMatch ? replayIdMatch[1] : null;
+  }
+
+  const sessionInfo = {
+    sessionId: client.getSessionId(),
+    dashcamUrl: dashcamUrl,
+    replayObjectId: replayObjectId,
+    instance: client.getInstance(),
+  };
+
+  console.log("📊 Session info:", JSON.stringify(sessionInfo, null, 2));
+  return sessionInfo;
+}

package/testdriver/acceptance-sdk/setup/vitestSetup.mjs

@@ -0,0 +1,40 @@
+/**
+ * Vitest Setup File
+ * Runs once before all tests in each worker process
+ * This ensures the TestDriver plugin global state is available in test processes
+ */
+
+// Import the plugin functions
+import {
+  authenticateWithApiKey,
+  clearDashcamUrls,
+  clearSuiteTestRun,
+  createTestRunDirect,
+  getDashcamUrl,
+  getPluginState,
+  getSuiteTestRun,
+  pluginState,
+  recordTestCaseDirect,
+  registerDashcamUrl,
+  setSuiteTestRun,
+} from "../../../interfaces/vitest-plugin.mjs";
+
+// Make the plugin API available globally in the test worker process
+if (typeof globalThis !== "undefined") {
+  globalThis.__testdriverPlugin = {
+    registerDashcamUrl,
+    getDashcamUrl,
+    clearDashcamUrls,
+    authenticateWithApiKey,
+    createTestRunDirect,
+    recordTestCaseDirect,
+    getSuiteTestRun,
+    setSuiteTestRun,
+    clearSuiteTestRun,
+    getPluginState,
+    state: pluginState,
+  };
+  console.log(
+    "[Vitest Setup] TestDriver plugin API initialized in worker process",
+  );
+}

package/testdriver/acceptance-sdk/sully-ai.test.mjs

@@ -0,0 +1,234 @@
+import { describe, expect, it } from "vitest";
+import { chrome } from "../../src/presets/index.mjs";
+
+/**
+ * CHECKPOINT SYSTEM GUIDE FOR TESTDRIVER TESTS
+ * ============================================
+ * 
+ * When newSandbox is set to false in chrome(), the sandbox persists between test runs.
+ * This allows you to skip completed steps when debugging or developing tests.
+ * 
+ * HOW TO USE CHECKPOINTS:
+ * 
+ * 1. Run your test normally until it fails at a certain step
+ * 2. Find the checkpoint comment just before the failing section (e.g., "// CHECKPOINT: Login Complete")
+ * 3. Comment out all code BEFORE that checkpoint using /* ... *\/ block comments
+ * 4. Re-run the test - it will resume from that checkpoint using the existing sandbox state
+ * 5. Once the test passes completely, uncomment all sections to verify end-to-end
+ * 
+ * CHECKPOINT LOCATIONS IN THIS TEST:
+ * - CHECKPOINT 1: Cookie banner handled
+ * - CHECKPOINT 2: Login complete
+ * - CHECKPOINT 3: Patients page loaded
+ * - CHECKPOINT 4: Patient selected/created
+ * - CHECKPOINT 5: Patient profile opened
+ * 
+ * IMPORTANT: Always set newSandbox: false in chrome() options to enable this feature!
+ * 
+ * FOR AI ASSISTANTS:
+ * When helping users debug TestDriver tests:
+ * - If a test fails at a certain step, suggest commenting out earlier checkpoints
+ * - Remind them to set newSandbox: false
+ * - After fixes, remind them to uncomment all code for full end-to-end validation
+ */
+
+// Helper function to poll for an element until it's found
+async function waitForElement(testdriver, description, maxAttempts = 10, delayMs = 1000) {
+  for (let i = 0; i < maxAttempts; i++) {
+    try {
+      const element = await testdriver.find(description);
+      if (element.found()) {
+        return element;
+      }
+    } catch (e) {
+      if (i === maxAttempts - 1) throw e;
+    }
+    await new Promise(resolve => setTimeout(resolve, delayMs));
+  }
+  throw new Error(`Element not found after ${maxAttempts} attempts: ${description}`);
+}
+
+describe("Sully.ai Patient Management Workflow", () => {
+  it("should complete full patient workflow: login, view patients, update note, and edit profile", async (context) => {
+    const { testdriver } = await chrome(context, {
+      url: 'https://app.sully.ai',
+      redrawThreshold: { enabled: true }
+    });
+
+    // ========================================================================
+    // CHECKPOINT 1: Cookie Banner
+    // If test fails after this point, comment out this section (lines above)
+    // ========================================================================
+    
+    // Handle cookie banner if present
+    try {
+      const acceptCookies = await testdriver.find("Accept All button for cookies");
+      await acceptCookies.click();
+    } catch {
+      console.log("No cookie banner or already dismissed");
+    }
+
+    // Wait for login page to load by polling for email field
+    console.log("Waiting for login page to load...");
+    const emailField = await waitForElement(testdriver, "Email input field");
+    console.log("Found email field");
+    await emailField.click();
+    await testdriver.type("razeen+testdriver@sully.ai");
+
+    const passwordField = await testdriver.find("Password input field");
+    await passwordField.click();
+    await testdriver.type("plmokn7@A", {secret: true});
+
+    const loginButton = await testdriver.find("Login button");
+    await loginButton.click();
+
+    // ========================================================================
+    // CHECKPOINT 2: Login Complete
+    // If test fails after this point, comment out all code above this checkpoint
+    // ========================================================================
+    
+    // Wait for navigation to complete after login
+    console.log("Waiting for login to process...");
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    
+    // Take a screenshot to see where we are
+    await testdriver.screenshot();
+    console.log("Screenshot taken after login");
+    
+    // Verify login successful
+    const loginSuccess = await testdriver.assert("user is logged in and on the main application page or dashboard");
+    expect(loginSuccess).toBeTruthy();
+
+    // Poll for dashboard element to ensure login completed
+    console.log("Looking for Patients link...");
+    await waitForElement(testdriver, "Patients button or link in the navigation menu", 15, 1000);
+
+    // Dismiss password save dialog first if it appears
+    try {
+      const neverButton = await testdriver.find("Never button or Never save button");
+      await neverButton.click();
+      console.log("Dismissed password save dialog");
+    } catch {
+      console.log("Password save dialog not found");
+    }
+
+    await testdriver.find("save profile changes popup or banner").click();
+
+    // Click Patients
+    const patientsLink = await testdriver.find("Patients navigation link or menu item");
+    await patientsLink.click();
+
+    // ========================================================================
+    // CHECKPOINT 3: Patients Page Loaded
+    // If test fails after this point, comment out all code above this checkpoint
+    // ========================================================================
+    
+    // Approach 1: Search for existing patient
+      const existingPatientInput = await testdriver.find("Patient search input box");
+      await existingPatientInput.click();
+      
+      // Type a search term to find a patient
+      await testdriver.type("Patient Test");
+      
+      // Poll for search results to appear
+      const firstPatient = await waitForElement(testdriver, "first patient name in the dropdown or list", 5, 500);
+      await firstPatient.click();
+    
+      console.log("Found patient via search");
+
+    // Verify patient visit page or details are loaded
+    const patientLoaded = await testdriver.assert("we are on a patient recording page");
+    expect(patientLoaded).toBeTruthy();
+
+    // ========================================================================
+    // CHECKPOINT 4: Patient Selected/Created
+    // If test fails after this point, comment out all code above this checkpoint
+    // ========================================================================
+
+    await testdriver.find('input box at the bottom of the screen').click();
+    await testdriver.type("This is a test note added by TestDriver automation.");
+
+    // Regenerate note - click make note more concise
+    console.log("Looking for note regeneration option...");
+    try {
+      const makeNoteConcise = await testdriver.find("Make note more concise button or option");
+      await makeNoteConcise.click();
+      console.log("Clicked make note more concise");
+
+      // Wait for note to regenerate
+      await new Promise(resolve => setTimeout(resolve, 3000));
+
+      // Verify note regeneration completed
+      const noteRegenerated = await testdriver.assert("note has been regenerated or updated");
+      expect(noteRegenerated).toBeTruthy();
+    } catch {
+      console.log("Could not find or click make note more concise option");
+    }
+
+    // Look for kebab menu on the right to access patient profile
+    console.log("Looking for kebab menu on the right...");
+
+    // Try to find the menu - it might be a three-dot menu or settings icon on the right side
+    const kebabMenu = await testdriver.find("three dots menu or kebab menu on the right side of the page");
+    await kebabMenu.click();
+
+    // Wait for menu to open 
+    await new Promise(resolve => setTimeout(resolve, 2000));
+
+    // Look for patient profile option
+    const patientProfile = await testdriver.find("Patient Profile or Edit Profile option in menu");
+    await patientProfile.click();
+    
+    // Wait for patient profile form to load
+    await new Promise(resolve => setTimeout(resolve, 3000));
+
+    // Verify patient profile form is displayed
+    const profileForm = await testdriver.assert("patient profile form or edit patient page is visible");
+    expect(profileForm).toBeTruthy();
+
+    // ========================================================================
+    // CHECKPOINT 5: Patient Profile Opened
+    // If test fails after this point, comment out all code above this checkpoint
+    // ========================================================================
+
+    // Fill out patient details
+    console.log("Filling out patient details...");
+    try {
+      // Look for phone number field as an example
+      const phoneField = await testdriver.find("phone number field or mobile number input");
+      await phoneField.click();
+      await testdriver.type("555-0123");
+      console.log("Filled phone number");
+
+      // Look for email field
+      try {
+        const emailFieldProfile = await testdriver.find("email address field in the patient profile");
+        await emailFieldProfile.click();
+        await testdriver.type("patient@example.com");
+        console.log("Filled email address");
+      } catch {
+        console.log("Email field not found");
+      }
+
+      // Look for and click the save button
+      const saveButton = await testdriver.find("Save button or Submit button");
+      await saveButton.click();
+      console.log("Clicked save button");
+
+      // Wait for save to complete
+      await new Promise(resolve => setTimeout(resolve, 2000));
+
+      // Verify save was successful
+      const saveSuccess = await testdriver.assert("changes were saved successfully or save confirmation appears");
+      expect(saveSuccess).toBeTruthy();
+    } catch (error) {
+      console.log("Could not complete patient profile form:", error.message);
+    }
+
+    // Take final screenshot
+    const finalScreenshot = await testdriver.screenshot();
+    expect(finalScreenshot).toBeDefined();
+
+    console.log("✅ Test completed successfully! Remember to uncomment all checkpoints for final validation.");
+  });
+});