testdriverai 6.2.2 → 7.0.0

package/docs/v7/playwright.mdx

@@ -0,0 +1,342 @@
+---
+title: "Get Started with TestDriver & Playwright"
+sidebarTitle: "Playwright"
+tag: "NEW"
+description: "In this guide, you'll setup your TestDriver account, create a new Playwright project, and leverage TestDriver's AI to convert tests to natural language."
+icon: "masks-theater"
+---
+
+## Overview
+
+`@testdriver.ai/playwright` is a backwards-compatible wrapper around `@playwright/test` that uses TestDriver's Vision AI to:
+
+- Make [natural language assertions](#assertions-with-expect-tomatchprompt)
+- [Replace brittle selectors](#locating-elements-with-testdriver-locate) with natural language
+- [Perform actions](#performing-actions-with-testdriver-act) with a prompt
+- Test with an [automated agent](#agentic-tests-with-test-agent)
+
+We'll be incrementally converting Playwright's example test from the sample code:
+
+```typescript tests/example.spec.ts icon=square-js
+// Before
+test("get started link", async ({ page }) => {
+  await page.goto("https://playwright.dev/");
+  await page.getByRole("link", { name: "Get started" }).click();
+  await expect(
+    page.getByRole("heading", { name: "Installation" }),
+  ).toBeVisible();
+});
+```
+
+To using natural language with TestDriver:
+
+```typescript tests/example.spec.ts icon=square-js
+// After
+import { test } from "@testdriver.ai/playwright";
+
+test.describe("get started link", () => {
+  test.beforeEach(async ({ page }) => page.goto("https://playwright.dev/"));
+  test.agent(`
+    - Click the 'Get started' link
+    - Verify the 'Installation' heading is visible
+  `);
+});
+```
+
+## Prerequisites
+
+<Steps>
+  <Step title="Create a TestDriver Account">
+    You will need a [Free TestDriver Account](https://app.testdriver.ai/team) to get an API key.
+
+    <Card title="Sign Up for TestDriver" icon="user-plus" horizontal href="https://app.testdriver.ai/team">
+
+    </Card>
+
+  </Step>
+  <Step title="Set up your environment">
+    Copy your API key from [the TestDriver dashboard](https://app.testdriver.ai/team), and set it as an environment variable.
+
+    <Tabs>
+      <Tab title="macOS / Linux">
+        ```bash Export an environment variable on macOS or Linux systems
+        export TD_API_KEY="your_api_key_here"
+        ```
+      </Tab>
+      <Tab title="Windows">
+        ```powershell Export an environment variable in PowerShell
+        setx TD_API_KEY "your_api_key_here"
+        ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+</Steps>
+
+## Setup Playwright
+
+<Steps>
+  <Step title="Initialize Playwright">
+    <Info>
+      This is a condensed version of [Playwright's Installation Instructions](https://playwright.dev/docs/intro).
+
+      **If you're new to Playwright, you should follow their guide first.**
+    </Info>
+    In a new folder or an existing, run:
+
+    <Tabs>
+      <Tab title="npm">
+        ```bash
+        npm init playwright@latest
+        ```
+      </Tab>
+      <Tab title="yarn">
+        ```bash
+        yarn create playwright
+        ```
+      </Tab>
+      <Tab title="pnpm">
+        ```bash
+        pnpm create playwright
+        ```
+      </Tab>
+    </Tabs>
+    Select the following options when prompted:
+
+    ```console
+    ✔ Do you want to use TypeScript or JavaScript?
+    > TypeScript
+    ✔ Where to put your end-to-end tests?
+    > tests
+    ✔ Add a GitHub Actions workflow? (y/N)
+    > N
+    ✔ Install Playwright browsers (can be done manually via 'npx playwright install')? (Y/n)
+    > Y
+    ```
+
+    Then, confirm Playwright works by running:
+
+    <Tabs>
+      <Tab title="npm">
+        ```bash
+        npx playwright test
+        ```
+      </Tab>
+      <Tab title="yarn">
+        ```bash
+        yarn playwright test
+        ```
+      </Tab>
+      <Tab title="pnpm">
+        ```bash
+        pnpm exec playwright test
+        ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+</Steps>
+
+## Setup `@testdriver.ai/playwright`
+
+<Steps>
+  <Step title="Install TestDriver">
+    `@testdriver.ai/playwright` as a backwards-compatible wrapper around `@playwright/test`:
+
+    <Tabs>
+      <Tab title="npm">
+        ```bash
+        npm install @testdriver.ai/playwright
+        ```
+      </Tab>
+      <Tab title="yarn">
+        ```bash
+        yarn add @testdriver.ai/playwright
+        ```
+      </Tab>
+      <Tab title="pnpm">
+        ```bash
+        pnpm add @testdriver.ai/playwright
+        ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+  <Step title="Run Playwright">
+    Before we start using TestDriver in our tests, run Playwright in [UI Mode](https://playwright.dev/docs/test-ui-mode):
+
+    <Tabs>
+      <Tab title="npm">
+        ```bash
+        npx playwright test --ui
+        ```
+      </Tab>
+      <Tab title="yarn">
+        ```bash
+        yarn playwright test --ui
+        ```
+      </Tab>
+      <Tab title="pnpm">
+        ```bash
+        pnpm exec playwright test --ui
+        ```
+      </Tab>
+    </Tabs>
+    ![Playwright UI Mode](https://playwright.dev/assets/ideal-img/ui-mode.4e54d6b.3598.png)
+
+    Clicking the ▶️ button should successfully run the tests in the UI,
+    just as they did before with `playwright test` in the CLI.
+
+  </Step>
+  <Step title="Import TestDriver">
+    For the sake of simplicity, we'll be working with one test file for now.
+
+    Open `tests/example.spec.ts` in your editor & rename the `@playwright/test`
+    import to `@testdriver.ai/playwright`:
+
+    ```typescript tests/example.spec.ts icon=square-js
+    import { test, expect } from '@playwright/test'; // [!code --]
+    import { test, expect } from '@testdriver.ai/playwright'; // [!code ++]
+    ```
+
+    Click the <Icon icon="play" /> button to run the test and verify everything still works.
+
+    <Tip>
+      Click the <Icon icon="eye" /> button to automatically re-run tests on save.
+    </Tip>
+
+  </Step>
+</Steps>
+
+## Usage
+
+Because TestDriver uses AI vision instead of selectors, we can use natural language for
+[assertions](#assertions-with-expect-tomatchprompt),
+[locating](#locating-elements-with-testdriver-locate),
+performing [actions](#performing-actions-with-testdriver-act),
+or even having an [agent](#agentic-tests-with-test-agent) test for you!
+
+### Assertions with `expect.toMatchPrompt`
+
+Replace `toBeVisible` with `toMatchPrompt` to assert that the element is visible on the screen:
+
+```typescript tests/example.spec.ts icon=square-js
+// [!code --:2]
+// Expects page to have a heading with the name of Installation.
+await expect(page.getByRole("heading", { name: "Installation" })).toBeVisible();
+// [!code ++:2]
+await expect(page).toMatchPrompt("'Installation' heading is visible");
+```
+
+Before, the test needed code comments to describe what the assertion is _actually checking_.
+
+With `toMatchPrompt`, natural language acts as a description, selector, and assertion in one
+
+<Tip>
+  TestDriver can reduce the amount of complexity in your tests, but you can
+  still "opt-in" to Playwright assertions and selectors if you need to (e.g.
+  validating accessibility with `page.getByRole`).
+</Tip>
+
+### Locating elements with `testdriver.locate`
+
+TestDriver can replace `data-testid`s, `getByRole`, and CSS selectors with natural language.
+
+First, update your test to get access to the `testdriver` fixture:
+
+```typescript tests/example.spec.ts icon=square-js
+// [!code --]
+test('get started link', async ({ page, }) => {
+// [!code ++]
+test('get started link', async ({ page, testdriver }) => {
+```
+
+Then, replace `getByRole` with `testdriver.locate`:
+
+```typescript tests/example.spec.ts icon=square-js
+test('get started link', async ({ page, testdriver }) => {
+  await page.goto('https://playwright.dev/');
+
+  // [!code --:2]
+  // Click the get started link.
+  await page.getByRole("link", { name: "Get started" }).click();
+  // [!code ++:2]
+  const link = await testdriver(page).locate("Get started link");
+  await link.click();
+
+  await expect(page).toMatchPrompt("'Installation' heading is visible");
+```
+
+Now, our test uses natural language to both describe & locate the element.
+
+<Tip>
+  In the example above, you can still use Playwright to assert that the element is indeed a link for accessibility:
+
+```typescript tests/example.spec.ts icon=square-js
+const link = await testdriver(page).locate("Get started link");
+// [!code ++]
+expect(link).toHaveRole("link");
+await link.click();
+```
+
+This way you can write user-centric tests _and_ validate the implementation.
+
+</Tip>
+
+### Performing actions with `testdriver.act`
+
+We can combine `locate` and `click` from the previous example into one line with `testdriver.act`:
+
+```typescript tests/example.spec.ts icon=square-js
+test("get started link", async ({ page, testdriver }) => {
+  await page.goto("https://playwright.dev/");
+
+  // [!code --:2]
+  const link = await testdriver(page).locate("Get started link");
+  await link.click();
+  // [!code ++]
+  await testdriver(page).act("Click the 'Get started' link");
+
+  await expect(page).toMatchPrompt("'Installation' heading is visible");
+});
+```
+
+Now the test uses the page the way a user would!
+
+### Agentic tests with `test.agent`
+
+TestDriver can automatically perform the entire test for you with an AI agent:
+
+```typescript tests/example.spec.ts icon=square-js
+// [!code --:6]
+test("get started link", async ({ page, testdriver }) => {
+  await page.goto("https://playwright.dev/");
+  await testdriver(page).act("Click the 'Get started' link");
+  await expect(page).toMatchPrompt("'Installation' heading is visible");
+});
+
+// [!code ++:7]
+test.describe("get started link", () => {
+  test.beforeEach(async ({ page }) => page.goto("https://playwright.dev/"));
+  test.agent(`
+    - Click the 'Get started' link
+    - Verify the 'Installation' heading is visible
+  `);
+});
+```
+
+Instead of writing the test implementation, we've used [`test.describe`](https://playwright.dev/docs/api/class-test#test-describe) to describe the test still,
+but replaced the `test` itself with `test.agent`.
+
+<Tip>
+  Use `test.beforeEach` to prepare the page for the agent (e.g.
+  [`page.goto`](https://playwright.dev/docs/api/class-page#page-goto), calling
+  an API to create a user). Use
+  [`test.afterEach`](https://playwright.dev/docs/api/class-test#test-after-each)
+  to clean up after the agent (e.g. `page.close`) or perform additional logic
+  (e.g. clearing the session).
+</Tip>
+
+## Conclusion
+
+With `@testdriver.ai/playwright`, you can use as much or as little of Playwright's _or_ TestDriver's API as you need to validate correctness. It's up to you!

package/eslint.config.js

@@ -35,9 +35,27 @@ module.exports = [
       },
     },
   },
+  {
+    // Config for ES Module files (.mjs) - used in SDK tests
+    files: ["**/*.mjs"],
+    languageOptions: {
+      sourceType: "module",
+      ecmaVersion: 2022,
+      globals: {
+        ...globals.node,
+      },
+    },
+  },
   {
     // this needs to be it's own object for some reason
     // https://github.com/eslint/eslint/issues/17400
-    ignores: ["agent/lib/subimage/**", "node_modules/**", ".git"],
+    ignores: [
+      "agent/lib/subimage/**",
+      "node_modules/**",
+      ".git",
+      "test-results/**",
+      "examples/test-recording-example.test.js",
+      "vitest.config.example.js",
+    ],
   },
 ];

package/examples/run-tests-with-recording.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Example script showing how to run tests with TestDriver + Dashcam integration
+# This script demonstrates the complete flow:
+# 1. Start dashcam
+# 2. Run tests with Vitest reporter
+# 3. Stop and publish dashcam recording
+# 4. View results in TestDriver dashboard
+
+set -e
+
+echo "======================================"
+echo "TestDriver + Dashcam Test Recording"
+echo "======================================"
+echo ""
+
+# Check for required environment variables
+if [ -z "$TD_API_KEY" ]; then
+  echo "Error: TD_API_KEY environment variable is required"
+  echo "Get your API key from: https://app.testdriver.ai/settings/api-keys"
+  exit 1
+fi
+
+if [ -z "$DASHCAM_PROJECT_ID" ]; then
+  echo "Warning: DASHCAM_PROJECT_ID not set, replays won't be published"
+fi
+
+# Start Dashcam recording
+echo "📹 Starting Dashcam..."
+dashcam start
+
+# Track test logs
+echo "📝 Tracking test output..."
+dashcam track --name="TestDriver Tests" --type=application --pattern="*.log"
+
+echo ""
+
+# Run tests with Vitest and TestDriver reporter
+echo "🧪 Running tests..."
+npx vitest run --config vitest.config.example.js
+
+# Capture exit code
+TEST_EXIT_CODE=$?
+
+# Stop dashcam
+echo ""
+echo "🛑 Stopping Dashcam..."
+dashcam stop
+
+# Publish to dashcam project if configured
+if [ -n "$DASHCAM_PROJECT_ID" ]; then
+  echo "📤 Publishing replay to project: $DASHCAM_PROJECT_ID"
+  REPLAY_URL=$(dashcam publish -p "$DASHCAM_PROJECT_ID" --json | jq -r '.replayUrl')
+  echo "✅ Replay URL: $REPLAY_URL"
+  echo ""
+  echo "View your test recording at:"
+  echo "$REPLAY_URL"
+else
+  echo "⚠️  Skipping publish (DASHCAM_PROJECT_ID not set)"
+fi
+
+echo ""
+echo "======================================"
+echo "📊 View Results"
+echo "======================================"
+echo "Dashboard: https://app.testdriver.ai/dashboard/test-runs"
+echo ""
+
+# Exit with the same code as tests
+exit $TEST_EXIT_CODE

package/examples/screenshot-example.js

@@ -0,0 +1,63 @@
+/**
+ * TestDriver SDK - Screenshot Example
+ *
+ * This example demonstrates how to use the screenshot() method
+ * to capture screenshots of the sandbox environment.
+ */
+
+const TestDriver = require("../sdk.js");
+const fs = require("fs");
+const path = require("path");
+
+async function main() {
+  // Initialize TestDriver SDK
+  const client = new TestDriver(process.env.TD_API_KEY);
+
+  try {
+    // Connect to sandbox
+    console.log("Connecting to sandbox...");
+    await client.connect();
+
+    // Navigate to a website
+    console.log("Opening Chrome and navigating to example.com...");
+    await client.focusApplication("Google Chrome");
+    const urlBar = await client.find("URL bar in Chrome");
+    await urlBar.click();
+    await client.type("https://example.com");
+    await client.pressKeys(["enter"]);
+
+    // Wait a moment for page to load
+    await client.wait(2000);
+
+    // Capture a screenshot
+    console.log("Capturing screenshot...");
+    const screenshot = await client.screenshot();
+
+    // Save screenshot to file
+    const outputPath = path.join(__dirname, "example-screenshot.png");
+    fs.writeFileSync(outputPath, Buffer.from(screenshot, "base64"));
+    console.log(`Screenshot saved to: ${outputPath}`);
+
+    // You can also capture with the mouse cursor visible
+    console.log("Capturing screenshot with mouse cursor...");
+    await client.hover(400, 300);
+    const screenshotWithMouse = await client.screenshot(1, false, true);
+
+    const outputPathWithMouse = path.join(
+      __dirname,
+      "example-screenshot-with-mouse.png",
+    );
+    fs.writeFileSync(
+      outputPathWithMouse,
+      Buffer.from(screenshotWithMouse, "base64"),
+    );
+    console.log(`Screenshot with mouse saved to: ${outputPathWithMouse}`);
+  } catch (error) {
+    console.error("Error:", error);
+  } finally {
+    // Clean up
+    await client.disconnect();
+  }
+}
+
+main();

package/examples/sdk-awesome-logs-demo.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+
+/**
+ * TestDriver SDK - AWESOME Logs Demo 🎨
+ *
+ * This example showcases the beautiful, emoji-rich logging with great DX
+ * that makes your test output a joy to read!
+ *
+ * Run: TD_API_KEY=your_key node examples/sdk-awesome-logs-demo.js
+ */
+
+const TestDriver = require("../sdk.js");
+const { formatter } = require("../sdk-log-formatter.js");
+
+(async () => {
+  try {
+    console.log(
+      formatter.formatHeader("TestDriver SDK - AWESOME Logs Demo", "🚀"),
+    );
+
+    // Create client with logging enabled
+    const client = new TestDriver(process.env.TD_API_KEY, {
+      os: "windows",
+      logging: true,
+    });
+
+    // Demo 1: Connection
+    console.log(
+      formatter.formatConnection("connect", {
+        sandboxId: "demo-sandbox-123",
+        os: "Windows",
+      }),
+    );
+
+    await client.connect({ headless: true });
+
+    // Demo 2: Navigation
+    console.log(
+      "\n" + formatter.formatAction("navigate", "https://example.com"),
+    );
+    await client.focusApplication("Google Chrome");
+    await client.type("https://example.com");
+    await client.pressKeys(["enter"]);
+
+    // Demo 3: Wait with loading indicator
+    console.log("\n" + formatter.formatWaiting("for page to load", 2000));
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    // Demo 4: Finding elements (this will use the real formatElementFound from sdk.js)
+    console.log("\n" + formatter.formatHeader("Finding Elements", "🔍"));
+
+    const heading = await client.find("heading that says Example Domain");
+    // The actual find() call will emit the formatted log automatically
+
+    if (heading.found()) {
+      await heading.click();
+      // The click will also emit a formatted log automatically
+    }
+
+    // Demo 5: Multiple actions
+    console.log("\n" + formatter.formatHeader("User Actions", "👆"));
+
+    const link = await client.find("More information link");
+    if (link.found()) {
+      await link.hover();
+      await link.click();
+    }
+
+    // Demo 6: Typing
+    console.log(
+      "\n" +
+        formatter.formatAction("type", "search query", {
+          text: "TestDriver AI",
+        }),
+    );
+
+    // Demo 7: Scroll
+    console.log("\n" + formatter.formatAction("scroll", "down the page"));
+    await client.scroll("down", 300);
+
+    // Demo 8: Assertions
+    console.log("\n" + formatter.formatHeader("Assertions", "✅"));
+    console.log(
+      formatter.formatAssertion("Page title is correct", true, {
+        duration: "45ms",
+      }),
+    );
+    console.log(
+      formatter.formatAssertion("Footer is visible", true, {
+        duration: "23ms",
+      }),
+    );
+    console.log(
+      formatter.formatAssertion("Login button exists", false, {
+        duration: "1234ms",
+      }),
+    );
+
+    // Demo 9: Cache status
+    console.log("\n" + formatter.formatHeader("Cache Performance", "⚡"));
+    console.log(
+      formatter.formatCacheStatus(true, {
+        similarity: 0.98,
+        strategy: "image",
+      }),
+    );
+    console.log(
+      formatter.formatCacheStatus(false, {
+        strategy: "text",
+      }),
+    );
+
+    // Demo 10: Screenshot
+    console.log(
+      "\n" +
+        formatter.formatScreenshot({
+          path: "/tmp/testdriver-debug/screenshot-123.png",
+          size: "245 KB",
+        }),
+    );
+
+    // Demo 11: Progress
+    console.log("\n" + formatter.formatHeader("Multi-step Process", "📈"));
+    for (let i = 1; i <= 5; i++) {
+      console.log(formatter.formatProgress(i, 5, `Processing step ${i}`));
+      await new Promise((resolve) => setTimeout(resolve, 300));
+    }
+
+    // Demo 12: Divider
+    console.log("\n" + formatter.formatDivider());
+
+    // Demo 13: Test summary
+    console.log(
+      formatter.formatSummary({
+        passed: 12,
+        failed: 2,
+        skipped: 1,
+        total: 15,
+        duration: "45.23s",
+      }),
+    );
+
+    // Demo 14: Error handling
+    console.log(formatter.formatHeader("Error Examples", "🚨"));
+    console.log(
+      formatter.formatError(
+        "Element not found",
+        new Error("Timeout after 5000ms"),
+      ),
+    );
+    console.log(
+      formatter.formatError(
+        "Connection failed",
+        new Error("Network unreachable"),
+      ),
+    );
+
+    // Demo 15: Test lifecycle
+    console.log(formatter.formatTestStart("Login Flow Test"));
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    console.log(formatter.formatTestEnd("Login Flow Test", true, 2345));
+
+    console.log(formatter.formatTestStart("Checkout Process"));
+    await new Promise((resolve) => setTimeout(resolve, 800));
+    console.log(formatter.formatTestEnd("Checkout Process", false, 5678));
+
+    // Final summary
+    console.log("\n" + formatter.formatHeader("Demo Complete!", "🎉"));
+    console.log("\n✨ Your test logs now look AWESOME! ✨\n");
+
+    await client.disconnect();
+    console.log("\n" + formatter.formatConnection("disconnect"));
+  } catch (error) {
+    console.error(formatter.formatError("Demo failed", error));
+    process.exit(1);
+  }
+})();