testdriverai 5.5.4 → 5.5.6

package/docs/overview/what-is-testdriver.mdx

@@ -1,21 +1,29 @@
 ---
 title: "What is TestDriver?"
+sidebarTitle: "What is TestDriver?"
 description: "TestDriver is a computer-use agent for QA testing of user interfaces."
 icon: "circle-info"
-mode: "wide"
 ---
 
 TestDriver uses AI vision and keyboard and mouse control to automate end-to-end testing. TestDriver is `selectorless` meaning it isn't aware of the underlying code structure.
 
-- **Easier set up:** No need to craft complex selectors
-- **Less Maintenance:** Tests don't break when code changes
-- **More Power:** TestDriver can test any application and control any OS setting
+<CardGroup cols={3}>
+  <Card title="Easier Setup" icon="wrench">
+    No need to craft complex selectors.
+  </Card>
+    <Card title="More Power" icon="bolt">
+    TestDriver can test anything a user can do.
+  </Card>
+  <Card title="Less Maintenance" icon="bandage">
+    Tests don't break when code changes.
+  </Card>
+</CardGroup>
 
 TestDriver is different from other computer-use agents in that it produces a `YAML` test script that increases the speed and repeatability of testing.
 
 ### Selectorless Testing
 
-Unlike traditional frameworks (e.g., Selenium, Playwright), TestDriver doesn’t rely on CSS selectors or static analysis. Instead, tests are described in plain English, such as:
+Unlike traditional frameworks (e.g., Selenium, Playwright), TestDriver doesn't rely on CSS selectors or static analysis. Instead, tests are described in plain English, such as:
 
 ```
 > Open Google Chrome and search for "testdriver"

package/docs/quickstart.mdx

@@ -26,7 +26,7 @@ Start TestDriver in interactive mode to create and refine tests dynamically:
 ```bash
 testdriverai
 ```  
-Once started, you can use commands like `/try`, `/undo`, and `/save` to interactively build and test your flows.
+Once started, you can use commands like `/explore`, `/undo`, and `/save` to interactively build and test your flows.
 </Step>
 
 <Step title="Define Your Test Steps">

package/docs/scenarios/ai-chatbot.mdx

@@ -0,0 +1,26 @@
+---
+title: "AI Chatbot"
+sidebarTitle: "AI Chatbot"
+description: "Integrate AI chatbots into your testing workflow."
+icon: "message-bot"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+TestDriver can be used to test AI chatbots, or any two way assertions like: 
+- Approval workflows
+- Chatbots
+- Email or SMS notifications
+- And more!
+
+
+This scenario is an example of how to set up and run tests for an AI chatbot using TestDriver.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Load a website with an AI chatbot. 
+2. Type a chat message or ask a question.
+3. Wait for the chatbot to respond.
+4. Verify that the chatbot's response is correct and meets the expected criteria.
+

package/docs/scenarios/cookie-banner.mdx

@@ -0,0 +1,30 @@
+---
+title: "Cookie Banner"
+sidebarTitle: "Cookie Banner"
+description: "Test cookie banners with TestDriver"
+icon: "cookie"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+With TestDriver, you can automate the testing of cookie banners on your web application. This scenario demonstrates how to check if a cookie banner appears when a user visits the site and how to interact with it.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. **Visit the Site**: The test will navigate to the target URL.
+2. **Check for Cookie Banner**: It will verify if the cookie banner is displayed.
+3. **Decline Cookies**: The test will simulate a user clicking the "Decline All" button on the cookie banner.
+
+
+
+## TestDriver in Action
+<iframe
+  width="560"
+  height="315"
+  src="https://www.youtube.com/embed/mhTffX0SG94"
+  title="Testing Cookie Banners with TestDriver"
+  frameborder="0"
+  allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+  allowfullscreen
+></iframe>

package/docs/scenarios/file-upload.mdx

@@ -0,0 +1,30 @@
+---
+title: "File Upload"
+sidebarTitle: "File Upload"
+description: "Test file upload functionality with TestDriver"
+icon: "upload"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+This scenario demonstrates how to automate testing the file upload functionality of a web application using TestDriver's capabilities. The test will check if the file upload process works correctly and if the uploaded file is processed as expected.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Load a website with a file upload feature. 
+2. Generate a file that will be used for the upload.
+3. Use TestDriver to perform the file upload process. 
+4. Check if the uploaded file is processed correctly by the application.
+
+
+## TestDriver in Action
+<iframe
+  width="560"
+  height="315"
+  src="https://www.youtube.com/embed/50AN5hDnwAg"
+  title="File Upload Testing with TestDriver"
+  frameborder="0"
+  allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+  allowfullscreen
+></iframe>

package/docs/scenarios/form-filling.mdx

@@ -0,0 +1,28 @@
+---
+title: "Form Filling"
+sidebarTitle: "Form Filling"
+description: "Test form filling functionality with TestDriver"
+icon: "list"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Open a webpage containing a form (Log in, Registration, etc.).
+2. Use TestDriver to fill in the form fields with test data.
+3. Submit the form and verify the expected outcome (e.g., successful login, registration confirmation, redirect etc.).
+
+
+
+## TestDriver in Action
+<iframe
+  width="560"
+  height="315"
+  src="https://www.youtube.com/embed/CqdRN_5_3I0"
+  title="Form Filling With TestDriver"
+  frameborder="0"
+  allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+  allowfullscreen
+></iframe>

package/docs/scenarios/log-in.mdx

@@ -0,0 +1,68 @@
+---
+title: "Login Scenario"
+sidebarTitle: "Login"
+description: "Test login functionality with TestDriver"
+icon: "lock"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+Test login functionality with TestDriver. This scenario demonstrates how to automate testing the login process for a web application using TestDriver.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Visit the login page of the web application.
+2. Enter the username and password into the respective fields (see [Reusable Snippets](/features/reusable-snippets#How-to Create-and-Use-Reusable-Snippets)).
+3. Click the "Login" button.
+4. Verify that the user is redirected to the dashboard or home page after a successful login.
+5. Optionally, check if the user is logged in by verifying the presence of a logout button or user profile information.
+
+
+### Setup your test environment with predefined credentials.
+In this example, we will use GitHub secret management to store our credentials.
+
+<Tip>You can also use the [`exec`](commands/exec) command to fetch a test user dynamically from a database or API.</Tip>
+
+To use GitHub secrets, create or modify an existing `.env` file in the root of your project and add the following lines:
+
+```bash
+TD_TEST_USERNAME=your_test_username
+TD_TEST_PASSWORD=your_test_password
+```
+
+2. Create a test file and use the credentials like this:
+
+```yaml
+version: 4.2.18
+steps:
+  - prompt: Log in to the application
+    commands:
+      - command: hover-text
+        text: Email address
+        description: email input field label
+        action: click
+      - command: type
+        text: ${TD_USERNAME} # Use environment variable for username
+      - command: hover-text
+        text: Password
+        description: password input field label
+        action: click
+      - command: type
+        text: ${TD_PASSWORD} # Use environment variable for password
+      - command: hover-text
+        text: Log In
+        description: log in button
+        action: click
+```
+
+3. Run the test using the command line:
+
+```bash
+testdriverai run login.yaml
+```
+
+4. Watch replays in [Your account](https://app.testdriver.ai)
+
+### Conclusion
+In this scenario, we demonstrated how to automate the login process for a web application using TestDriver. By leveraging reusable snippets and environment variables, you can create efficient and maintainable tests for your applications. This approach not only saves time but also ensures that your tests are easily adaptable to changes in the application or test data.

package/docs/scenarios/pdf-generation.mdx

@@ -0,0 +1,21 @@
+---
+title: "PDF Generation"
+sidebarTitle: "PDF Generation"
+description: "Test PDF generation functionality with TestDriver"
+icon: "file-pdf"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+Test PDF generation functionality with TestDriver. This scenario demonstrates how to automate testing the PDF generation process for a web application using TestDriver's capabilities.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Open a webpage or desktop app that will allow you to generate a PDF.
+2. Trigger the PDF generation process (e.g., by clicking a button).
+3. Verify that the PDF is generated successfully and contains the expected content.
+4. Optionally, check the file size and format of the generated PDF.
+
+## What Next?
+From here you can automate and add coverage to your PDF generation process, removing manual testing from the equation.

package/docs/scenarios/spell-check.mdx

@@ -0,0 +1,21 @@
+---
+title: "Spell Check"
+sidebarTitle: "Spell Check"
+description: "Test spell check functionality with TestDriver"
+icon: "spell-check"
+---
+
+import TestPrereqs from '/snippets/test-prereqs.mdx'
+
+This scenario demonstrates how to automate testing the spell check functionality of a web application using TestDriver's capabilities. The test will check if the spell checker correctly identifies and suggests corrections for misspelled words in a text input field.
+
+<TestPrereqs />
+
+## Scenario Overview
+1. Open up a webpage that has text input and a spell check function.
+2. Enter a misspelled word into the text input field.
+3. Trigger the spell check function (e.g., by clicking a button or losing focus on the input field).
+4. Verify that the spell checker identifies the misspelled word and provides suggestions for correction.
+5. Optionally, select a suggestion and apply it to the text input field.
+
+

package/docs/security/action.mdx

@@ -2,13 +2,15 @@
 title: "Security Features of the TestDriver Action"
 sidebarTitle: "GitHub Action"
 description: "Understand the ephemeral VM runners, secrets handling, and environment-specific security for the TestDriver Action."
-icon: "code-pull-request"
+icon: shield-check
 ---
 
+import GitignoreWarning from '/snippets/gitignore-warning.mdx';
+
 ## Open Source
 The TestDriver Action is open source, and its source code is available for review. You can find the repository here:
 
-[GitHub - testdriverai/action](https://github.com/testdriverai/action)
+<Card horizontal="true" icon="github"> [GitHub - testdriverai/action](https://github.com/testdriverai/action) </Card>
 
 ## Ephemeral Virtual Machine Runners
 TestDriver tests are executed on private virtual machines (VMs) managed by Amazon EC2. These VMs are ephemeral, meaning they only exist for the duration of the test execution. Once the test is complete:
@@ -20,7 +22,7 @@ To securely manage private information, we recommend storing sensitive data as s
 
 ### Handling Secrets
 - **Prerun Scripts**: Any secrets supplied within prerun scripts or prompts are transmitted over SSL to the TestDriver API. Prerun scripts are **not persisted**.
-- **Agent Prompts**: Secrets supplied to agent prompts are persisted (see [Agent Security](../agent)).
+- **Agent Prompts**: Secrets supplied to agent prompts are persisted (see [Agent Security](/security/agent)).
 - **Secure Workflows**: If your workflow requires secret sharing and you encounter issues, please contact us for assistance.
 
 ### Common Use Case
@@ -28,6 +30,8 @@ A common workflow involves using prerun scripts to securely access a private sta
 
 ## Environment-Specific Security
 
+<GitignoreeWarning/>
+
 ### Production
 Testing production environments is the simplest and most secure starting point. 
 - Production testing doesn't require any private information from your team.

package/docs/security/agent.mdx

@@ -2,7 +2,7 @@
 title: "Security Features of the TestDriver Agent"
 sidebarTitle: "TestDriver Agent "
 description: "Explore the data collected, persistence, and privacy practices of the TestDriver Agent."
-icon: "robot"
+icon: lock-keyhole
 ---
 
 ## Source

package/docs/security/platform.mdx

@@ -2,9 +2,11 @@
 title: "Security Features of the Dashboard"
 sidebarTitle: "TestDriver Platform" 
 description: "Learn about the security features of the TestDriver web dashboard, including SSL, OAuth, RBAC, and more."
-icon: "layer-group"
+icon: layer-group
 ---
 
+import GitignoreWarning from '/snippets/gitignore-warning.mdx';
+
 ## Overview
 The TestDriver web dashboard provides a secure interface for managing and reviewing your tests. Tests executed via the GitHub Action are recorded and reported through Dashcam, another application developed by TestDriver. For more details, refer to the [Dashcam documentation](https://dashcam.docs.testdriver.ai).
 
@@ -34,6 +36,12 @@ Dashcam and TestDriver share the same API and web application back end, which in
 - Teams can rotate their API key for enhanced security.
 - It is recommended to rotate the API key every 90 days to minimize risk.
 
+<Check>
+    For more details on **Team Management** see the [Team documentation](/account/team).
+</Check>
+
+<GitignoreWarning/>
+
 ### Secret Masking
 - Test replay logs and network requests are automatically scanned for sensitive information, such as:
   - Credit card numbers

package/docs/snippets/calendar-link.mdx

@@ -0,0 +1 @@
+export const calendar = 'https://calendly.com/d/cq23-qyn-3v6/testdriver-ai-demo';

package/docs/snippets/gitignore-warning.mdx

@@ -0,0 +1,4 @@
+<Warning>
+Always remember to add a `.gitignore` file to your repository including a `.env` line so you never accidentally commit you TestDriver API key.
+This is important for security and to prevent exposing sensitive information. For more info see [GitHub Docs](https://docs.github.com/en/get-started/git-basics/ignoring-files).
+</Warning>

package/docs/snippets/test-prereqs.mdx

@@ -0,0 +1,15 @@
+## Prerequisites
+Before running the tests, ensure you have performed the following steps:
+
+1. Install the [VS Code Extension](/getting-started/vscode) for TestDriver
+2. If you haven't already, signup for a Free Trial on the [TestDriver website](https://testdriver.ai/pricing)
+3. Install the [TestDriver CLI](/cli/overview) globally using npm or just follow the VS Code Extension Setup Walkthrough:
+```bash
+npm install -g @testdriverai@beta
+```
+4. Run the `init` command to set up the TestDriver configuration using the API key you got when you signed up for the trial:
+```bash
+testdriverai init
+```
+
+<Check>Now you are ready to run the tests!</Check>

package/docs/tutorials/advanced-test.mdx

@@ -1,6 +1,8 @@
 ---
 title: "Tutorial: Advanced Test"
 description: "Explore advanced testing techniques and features in TestDriver."
+sidebarTitle: "Advanced Test"
+icon: "flask-round"
 ---
 
 # Advanced Test Tutorial

package/docs/tutorials/basic-test.mdx

@@ -1,6 +1,8 @@
 ---
 title: "Tutorial: Basic Test"
 description: "Learn how to create and run a basic test using TestDriver."
+sidebarTitle: "Basic Test"
+icon: "test-tube"
 ---
 
 # Basic Test Tutorial
@@ -18,20 +20,19 @@ Before you begin, ensure you have the following:
 To create a test, follow these steps:
 1. Open your code editor.
 2. Write the test script using TestDriver's syntax.
-3. Save the test script with a `.td` extension.
+3. Save the test script with a `.yaml` extension.
 
 Example:
-```plaintext
-test("Example Test", () => {
-  expect(1 + 1).toBe(2);
-});
+```yaml
+
+
 ```
 
 ## Step 2: Run the Test
 
 To run the test, use the following command:
-```plaintext
-testdriver run example-test.td
+```bash
+testdriverai run example-test.yaml
 ```
 
 This will execute the test and display the results in the console.

package/lib/commands.js

@@ -336,7 +336,7 @@ let commands = {
     logger.info(chalk.dim("thinking..."), true);
     logger.info("");
 
-    const mdStream = createMarkdownStreamLogger();
+    // const mdStream = createMarkdownStreamLogger();
     let response = await sdk.req(
       "hover/text",
       {
@@ -349,13 +349,15 @@ let commands = {
       },
       (chunk) => {
         if (chunk.type === "data" && chunk.data) {
-          mdStream.log(chunk.data);
+          
+          // mdStream.log(chunk.data);
+
         } else if (chunk.type === "closeMatches") {
           emitter.emit(events.matches.show, chunk.data);
         }
       },
     );
-    mdStream.end();
+    // mdStream.end();
 
     if (!response.data) {
       throw new AiError("No text on screen matches description");
@@ -370,7 +372,7 @@ let commands = {
     logger.info(chalk.dim("thinking..."), true);
     logger.info("");
 
-    const mdStream = createMarkdownStreamLogger();
+    // const mdStream = createMarkdownStreamLogger();
     let response = await sdk.req(
       "hover/image",
       {
@@ -381,13 +383,13 @@ let commands = {
       },
       (chunk) => {
         if (chunk.type === "data") {
-          mdStream.log(chunk.data);
+          // mdStream.log(chunk.data);
         } else if (chunk.type === "closeMatches") {
           emitter.emit(events.matches.show, chunk.data);
         }
       },
     );
-    mdStream.end();
+    // mdStream.end();
 
     if (!response?.data) {
       throw new AiError("No image or icon on screen matches description");
@@ -739,7 +741,7 @@ let commands = {
 
     let plat = platform();
 
-    const scriptCode =
+    let scriptCode =
       plat == "linux"
         ? linux_code
         : plat == "windows"
@@ -793,7 +795,7 @@ let commands = {
         result = await exec(mac_code, { cwd: cwd() });
       }
 
-      if (result.out.exitCode !== 0) {
+      if (result.out && result.out.exitCode !== 0) {
         throw new AiError(
           `Command failed with exit code ${result.out.exitCode}: ${result.out.stderr}`,
           true,
@@ -802,10 +804,10 @@ let commands = {
         
         if (!silent) {
           logger.info(chalk.dim(`Command output:`), true);
-          logger.info(`${result.out.stdout}`, true)
+          logger.info(`${result.stdout}`, true)
         }
 
-        return result.out.stdout.trim();
+        return result.stdout.trim();
       }
 
     } else if (language == "js") {
@@ -816,29 +818,46 @@ let commands = {
         true,
       );
 
-      const context = vm.createContext({ require, console, fs, process });
+      console.log('')
+      console.log('------');
+      
+      const context = vm.createContext({ require, console, fs, process, fetch });
 
-      const script = new vm.Script(`
-          (async () => {
-            ${scriptCode}
-          })();
-        `);
+      scriptCode = '(async function() {\n' + scriptCode + '\n})();';
+      
+      const script = new vm.Script(scriptCode);
 
       try {
-        await script.runInContext(context);
+        await script.runInNewContext(context);
       } catch (e) {
+        console.error(e);
         throw new AiError(
           `Error running script: ${e.message}`,
           true,
         );
       }
 
-      if (!context.result) {
-        logger.info(`No result returned from script`, true);
+      // wait for context.result to resolve
+      let stepResult = await context.result;
+
+      // conver it to string
+      if (typeof stepResult === "object") {
+        stepResult = JSON.stringify(stepResult, null, 2);
+      } else if (typeof stepResult === "function") {
+        stepResult = stepResult.toString();
       }
 
-      // wait for context.result to resolve
-      const stepResult = await context.result;
+      console.log('------');
+      console.log('')
+
+      if (!stepResult) {
+        logger.info(`No result returned from script`, true);
+      } else {
+        if (!silent) {
+          logger.info(chalk.dim(`Result:`), true);
+          logger.info(stepResult, true);
+        }
+      }
 
       return stepResult;
       // }

package/lib/config.js

@@ -20,7 +20,6 @@ function parseValue(value) {
   return value;
 }
 
-// Object for TD related config, with defaults
 const config = {
   TD_SPEAK: false,
   TD_ANALYTICS: true,

package/lib/logger.js

@@ -5,7 +5,7 @@ const server = require("./ipc");
 
 // simple match for aws instance i-*
 const shouldLog =
-  os.hostname().indexOf("i-") == 0 || process.env["VERBOSE"] == "true";
+  os.hostname().indexOf("i-") == 0 || process.env["TD_DEV"] == "true";
 
 // responsible for rendering ai markdown output
 const { marked } = require("marked");
@@ -28,7 +28,7 @@ for (const [key, value] of Object.entries(process.env)) {
 
 const censorSensitiveData = (message) => {
   for (let value of Object.values(interpolationVars)) {
-    // Avoid replacing vars that are 0 or 1 character
+    // Avoid replacing vars that are 0 or 1 characters
     if (value.length >= 2) {
       message = message.replaceAll(value, "****");
     }

package/lib/parser.js

@@ -122,6 +122,7 @@ function interpolate (yaml, vars) {
   });
   // Replace \$ with $
   newyaml = newyaml.replace(/\\(\${[^}]+})/g, "$1");
+
   return newyaml;
 }