testdriverai 6.0.23 → 6.0.24-canary.fffd04a.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/acceptance-tests.yml +5 -1
- package/docs/action/ami.mdx +4 -3
- package/docs/action/performance.mdx +26 -8
- package/docs/action/secrets.mdx +11 -31
- package/docs/apps/chrome-extensions.mdx +1 -1
- package/docs/apps/desktop-apps.mdx +11 -8
- package/docs/apps/static-websites.mdx +7 -7
- package/docs/bugs/jira.mdx +12 -75
- package/docs/cli/overview.mdx +6 -6
- package/docs/commands/exec.mdx +32 -37
- package/docs/commands/focus-application.mdx +13 -2
- package/docs/commands/hover-image.mdx +29 -5
- package/docs/commands/hover-text.mdx +10 -16
- package/docs/commands/if.mdx +24 -21
- package/docs/commands/match-image.mdx +17 -9
- package/docs/commands/press-keys.mdx +55 -3
- package/docs/commands/remember.mdx +13 -9
- package/docs/commands/run.mdx +10 -4
- package/docs/commands/scroll-until-image.mdx +25 -11
- package/docs/commands/scroll-until-text.mdx +17 -10
- package/docs/commands/scroll.mdx +22 -8
- package/docs/commands/type.mdx +6 -6
- package/docs/commands/wait-for-image.mdx +20 -7
- package/docs/commands/wait-for-text.mdx +14 -12
- package/docs/commands/wait.mdx +8 -1
- package/docs/docs.json +4 -6
- package/docs/exporting/junit.mdx +3 -3
- package/docs/exporting/playwright.mdx +2 -2
- package/docs/features/auto-healing.mdx +3 -3
- package/docs/features/generation.mdx +2 -4
- package/docs/features/parallel-testing.mdx +3 -13
- package/docs/features/reusable-snippets.mdx +3 -3
- package/docs/features/selectorless.mdx +8 -2
- package/docs/features/visual-assertions.mdx +2 -2
- package/docs/getting-started/ci.mdx +42 -51
- package/docs/getting-started/cli.mdx +3 -9
- package/docs/getting-started/editing.mdx +13 -10
- package/docs/getting-started/running.mdx +7 -2
- package/docs/getting-started/vscode.mdx +4 -7
- package/docs/guide/assertions.mdx +2 -4
- package/docs/guide/authentication.mdx +8 -10
- package/docs/guide/code.mdx +14 -12
- package/docs/guide/environment-variables.mdx +13 -12
- package/docs/guide/lifecycle.mdx +39 -31
- package/docs/guide/variables.mdx +30 -20
- package/docs/guide/waiting.mdx +2 -2
- package/docs/importing/csv.mdx +3 -4
- package/docs/importing/jira.mdx +6 -12
- package/docs/importing/testrail.mdx +6 -6
- package/docs/integrations/electron.mdx +30 -56
- package/docs/integrations/netlify.mdx +9 -12
- package/docs/integrations/vercel.mdx +6 -42
- package/docs/interactive/explore.mdx +8 -10
- package/docs/interactive/run.mdx +9 -4
- package/docs/interactive/save.mdx +2 -2
- package/docs/overview/comparison.mdx +68 -60
- package/docs/overview/performance.mdx +0 -1
- package/docs/overview/what-is-testdriver.mdx +1 -1
- package/docs/scenarios/log-in.mdx +2 -2
- package/docs/security/platform.mdx +4 -2
- package/docs/snippets/test-prereqs.mdx +1 -1
- package/docs/snippets/tests/assert-replay.mdx +7 -5
- package/docs/snippets/tests/assert-yaml.mdx +2 -3
- package/docs/snippets/tests/exec-js-replay.mdx +7 -5
- package/docs/snippets/tests/exec-js-yaml.mdx +3 -4
- package/docs/snippets/tests/exec-shell-replay.mdx +7 -5
- package/docs/snippets/tests/exec-shell-yaml.mdx +4 -5
- package/docs/snippets/tests/hover-image-replay.mdx +7 -5
- package/docs/snippets/tests/hover-image-yaml.mdx +4 -6
- package/docs/snippets/tests/hover-text-replay.mdx +7 -5
- package/docs/snippets/tests/hover-text-with-description-replay.mdx +7 -5
- package/docs/snippets/tests/hover-text-with-description-yaml.mdx +4 -10
- package/docs/snippets/tests/hover-text-yaml.mdx +2 -3
- package/docs/snippets/tests/match-image-replay.mdx +7 -5
- package/docs/snippets/tests/match-image-yaml.mdx +6 -6
- package/docs/snippets/tests/press-keys-replay.mdx +7 -5
- package/docs/snippets/tests/press-keys-yaml.mdx +5 -6
- package/docs/snippets/tests/remember-replay.mdx +7 -5
- package/docs/snippets/tests/remember-yaml.mdx +25 -6
- package/docs/snippets/tests/scroll-replay.mdx +7 -5
- package/docs/snippets/tests/scroll-until-image-replay.mdx +7 -0
- package/docs/snippets/tests/scroll-until-image-yaml.mdx +14 -0
- package/docs/snippets/tests/scroll-until-text-replay.mdx +7 -5
- package/docs/snippets/tests/scroll-until-text-yaml.mdx +4 -8
- package/docs/snippets/tests/scroll-yaml.mdx +4 -13
- package/docs/snippets/tests/type-repeated-replay.mdx +7 -5
- package/docs/snippets/tests/type-repeated-yaml.mdx +2 -3
- package/docs/snippets/tests/type-replay.mdx +7 -5
- package/docs/snippets/tests/type-yaml.mdx +2 -5
- package/docs/snippets/tests/wait-for-image-replay.mdx +7 -0
- package/docs/snippets/tests/wait-for-image-yaml.mdx +18 -0
- package/docs/snippets/tests/wait-for-text-replay.mdx +7 -0
- package/docs/snippets/tests/wait-for-text-yaml.mdx +18 -0
- package/docs/snippets/tests/wait-replay.mdx +7 -0
- package/docs/snippets/tests/wait-yaml.mdx +13 -0
- package/docs/styles.css +46 -0
- package/package.json +3 -9
- package/testdriver/edge-cases/prompt-nested.yaml +2 -1
- package/docs/action/setup.mdx +0 -120
- package/docs/interactive/assert.mdx +0 -65
- package/docs/interactive/dry.mdx +0 -58
- package/docs/interactive/generate.mdx +0 -55
- package/docs/interactive/undo.mdx +0 -58
- package/docs/snippets/comments.mdx +0 -1
- package/docs/snippets/tests/prompt-replay.mdx +0 -5
- package/docs/snippets/tests/prompt-yaml.mdx +0 -10
- package/docs/snippets/yml-warning.mdx +0 -4
|
@@ -3,18 +3,25 @@ title: "wait-for-image"
|
|
|
3
3
|
sidebarTitle: "wait-for-image"
|
|
4
4
|
description: "Wait until an image matching the description is detected on the screen."
|
|
5
5
|
icon: "clock-three-thirty"
|
|
6
|
+
mode: "wide"
|
|
6
7
|
---
|
|
7
8
|
|
|
9
|
+
import Replay from "/snippets/tests/wait-for-image-replay.mdx";
|
|
10
|
+
import Example from "/snippets/tests/wait-for-image-yaml.mdx";
|
|
11
|
+
|
|
12
|
+
<Replay />
|
|
13
|
+
<Example />
|
|
14
|
+
|
|
8
15
|
## Description
|
|
9
16
|
|
|
10
17
|
The `wait-for-image` command waits until the specified image is detected on the screen. This is useful for ensuring that visual elements are present before proceeding with the next steps in a test.
|
|
11
18
|
|
|
12
19
|
## Arguments
|
|
13
20
|
|
|
14
|
-
|
|
|
15
|
-
|
|
|
16
|
-
| `description` | `string` | A description of the image.
|
|
17
|
-
|
|
|
21
|
+
| Argument | Type | Description |
|
|
22
|
+
| :-----------: | :------: | :-------------------------------------------------------------------------------------------------------- |
|
|
23
|
+
| `description` | `string` | A description of the image. |
|
|
24
|
+
| `timeout` | `number` | (Optional) The duration in milliseconds to wait for the image to appear. Default is `10000` (10 seconds). |
|
|
18
25
|
|
|
19
26
|
## Example usage
|
|
20
27
|
|
|
@@ -29,12 +36,18 @@ timeout: 5000
|
|
|
29
36
|
- Use clear and concise descriptions for the image to improve detection accuracy.
|
|
30
37
|
- Adjust the `timeout` value based on the expected load time of the image to avoid unnecessary delays.
|
|
31
38
|
|
|
39
|
+
<Tip>
|
|
40
|
+
If you are unable to land on an accurate description or experiencing flaky
|
|
41
|
+
results, try to upload the image to [chatgpt](https://chatgpt.com) and ask it
|
|
42
|
+
to describe the image and use the generated `description` to get better
|
|
43
|
+
accuracy and determinism.
|
|
44
|
+
</Tip>
|
|
45
|
+
|
|
32
46
|
## Gotchas
|
|
33
47
|
|
|
34
48
|
- If the image doesn't appear within the specified `timeout`, the command will fail.
|
|
35
49
|
- Ensure the description accurately represents the image to avoid detection issues.
|
|
36
50
|
|
|
37
|
-
|
|
51
|
+
---
|
|
38
52
|
|
|
39
|
-
|
|
40
|
-
- This command supports flexible timeouts to accommodate varying load times.
|
|
53
|
+
The `wait-for-image` command is ideal for synchronizing tests with visual elements that may take time to load.
|
|
@@ -3,19 +3,26 @@ title: "wait-for-text"
|
|
|
3
3
|
sidebarTitle: "wait-for-text"
|
|
4
4
|
description: "Wait until the specified text is detected on the screen."
|
|
5
5
|
icon: "clock-nine"
|
|
6
|
+
mode: "wide"
|
|
6
7
|
---
|
|
7
8
|
|
|
9
|
+
import Replay from "/snippets/tests/wait-for-text-replay.mdx";
|
|
10
|
+
import Example from "/snippets/tests/wait-for-text-yaml.mdx";
|
|
11
|
+
|
|
12
|
+
<Replay />
|
|
13
|
+
<Example />
|
|
14
|
+
|
|
8
15
|
## Description
|
|
9
16
|
|
|
10
17
|
The `wait-for-text` command waits until the specified text is detected on the screen. This is useful for ensuring that textual elements are present before proceeding with the next steps in a test.
|
|
11
18
|
|
|
12
19
|
## Arguments
|
|
13
20
|
|
|
14
|
-
| Argument |
|
|
15
|
-
|
|
|
16
|
-
|
|
|
17
|
-
| `timeout` | `number` | (Optional) The duration in milliseconds to wait for the text to appear. Default is `5000
|
|
18
|
-
| `method` |
|
|
21
|
+
| Argument | Type | Description |
|
|
22
|
+
| :-------: | :------: | :----------------------------------------------------------------------------------------------------- |
|
|
23
|
+
| `text` | `string` | The text to find on the screen. |
|
|
24
|
+
| `timeout` | `number` | (Optional) The duration in milliseconds to wait for the text to appear. Default is `5000` (5 seconds). |
|
|
25
|
+
| `method` | `enum` | (Optional) The matching algorithm to use. Possible values are `ai` and `turbo`. Default is `turbo` |
|
|
19
26
|
|
|
20
27
|
## Example usage
|
|
21
28
|
|
|
@@ -28,10 +35,6 @@ timeout: 5000
|
|
|
28
35
|
## Protips
|
|
29
36
|
|
|
30
37
|
- Use unique and specific text to improve detection accuracy.
|
|
31
|
-
- Choose the appropriate `method` for your use case:
|
|
32
|
-
- `ai`: More robust for complex or ambiguous text matching (default) using [Set-of-Mark](https://arxiv.org/abs/2310.11441).
|
|
33
|
-
- `levenshtein`: Compare string similarity using [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance).
|
|
34
|
-
- `turbo`: Ideal for simpler scenarios, combining `ai` & `levenshtein`
|
|
35
38
|
- Adjust the `timeout` value based on the expected load time of the text to avoid unnecessary delays.
|
|
36
39
|
|
|
37
40
|
## Gotchas
|
|
@@ -39,7 +42,6 @@ timeout: 5000
|
|
|
39
42
|
- If the text doesn't appear within the specified `timeout`, the command will fail.
|
|
40
43
|
- Ensure the text matches exactly, as variations in font size, style, or screen resolution may affect detection accuracy.
|
|
41
44
|
|
|
42
|
-
|
|
45
|
+
---
|
|
43
46
|
|
|
44
|
-
|
|
45
|
-
- This command supports flexible matching algorithms to accommodate different scenarios.
|
|
47
|
+
The `wait-for-text` command is ideal for synchronizing tests with textual elements that may take time to load.
|
package/docs/commands/wait.mdx
CHANGED
|
@@ -3,8 +3,15 @@ title: "wait"
|
|
|
3
3
|
sidebarTitle: "wait"
|
|
4
4
|
description: "Pause the execution of the script for a specified duration."
|
|
5
5
|
icon: "clock"
|
|
6
|
+
"mode": "wide"
|
|
6
7
|
---
|
|
7
8
|
|
|
9
|
+
import Replay from "/snippets/tests/wait-replay.mdx";
|
|
10
|
+
import Example from "/snippets/tests/wait-yaml.mdx";
|
|
11
|
+
|
|
12
|
+
<Replay />
|
|
13
|
+
<Example />
|
|
14
|
+
|
|
8
15
|
## Description
|
|
9
16
|
|
|
10
17
|
The `wait` command pauses the execution of the script for a specified number of milliseconds before continuing. This is useful for adding delays between commands or waiting for certain conditions to stabilize.
|
|
@@ -35,4 +42,4 @@ timeout: 5000
|
|
|
35
42
|
## Notes
|
|
36
43
|
|
|
37
44
|
- The `wait` command is ideal for introducing controlled pauses in your test scripts.
|
|
38
|
-
-
|
|
45
|
+
- Whenever you are waiting for something to load, use the [wait-for-text](/commands/wait-for-text) or [wait-for-image](/commands/wait-for-image) commands instead to make the test more efficient.
|
package/docs/docs.json
CHANGED
|
@@ -44,13 +44,9 @@
|
|
|
44
44
|
"group": "Commands",
|
|
45
45
|
"icon": "command",
|
|
46
46
|
"pages": [
|
|
47
|
-
"/interactive/assert",
|
|
48
|
-
"/interactive/dry",
|
|
49
47
|
"/interactive/explore",
|
|
50
|
-
"/interactive/generate",
|
|
51
48
|
"/interactive/run",
|
|
52
|
-
"/interactive/save"
|
|
53
|
-
"/interactive/undo"
|
|
49
|
+
"/interactive/save"
|
|
54
50
|
]
|
|
55
51
|
}
|
|
56
52
|
]
|
|
@@ -101,6 +97,7 @@
|
|
|
101
97
|
"group": "Configuration",
|
|
102
98
|
"icon": "desktop",
|
|
103
99
|
"pages": [
|
|
100
|
+
"/guide/authentication",
|
|
104
101
|
"/guide/variables",
|
|
105
102
|
"/guide/lifecycle",
|
|
106
103
|
"/guide/environment-variables",
|
|
@@ -135,7 +132,7 @@
|
|
|
135
132
|
"icon": "play",
|
|
136
133
|
"pages": [
|
|
137
134
|
"/getting-started/running",
|
|
138
|
-
"/
|
|
135
|
+
"/getting-started/ci",
|
|
139
136
|
"/features/parallel-testing",
|
|
140
137
|
"/action/performance",
|
|
141
138
|
"/action/secrets"
|
|
@@ -199,6 +196,7 @@
|
|
|
199
196
|
"links": [
|
|
200
197
|
{
|
|
201
198
|
"label": "Discord",
|
|
199
|
+
"icon": "discord",
|
|
202
200
|
"href": "https://discord.com/invite/cWDFW8DzPm"
|
|
203
201
|
}
|
|
204
202
|
],
|
package/docs/exporting/junit.mdx
CHANGED
|
@@ -104,7 +104,7 @@ jobs:
|
|
|
104
104
|
- uses: actions/checkout@v4
|
|
105
105
|
|
|
106
106
|
- name: Run TestDriver Tests
|
|
107
|
-
run: npx testdriverai@latest run testdriver/test.yaml --
|
|
107
|
+
run: npx testdriverai@latest run testdriver/test.yaml --junit=test-results.xml
|
|
108
108
|
env:
|
|
109
109
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
110
110
|
|
|
@@ -127,7 +127,7 @@ pipeline {
|
|
|
127
127
|
stage('Test') {
|
|
128
128
|
steps {
|
|
129
129
|
script {
|
|
130
|
-
sh 'npx testdriverai@latest run testdriver/test.yaml --
|
|
130
|
+
sh 'npx testdriverai@latest run testdriver/test.yaml --junit=junit-results.xml'
|
|
131
131
|
}
|
|
132
132
|
}
|
|
133
133
|
post {
|
|
@@ -145,7 +145,7 @@ pipeline {
|
|
|
145
145
|
```yaml
|
|
146
146
|
test:
|
|
147
147
|
script:
|
|
148
|
-
- npx testdriverai@latest run testdriver/test.yaml --
|
|
148
|
+
- npx testdriverai@latest run testdriver/test.yaml --junit=test-results.xml
|
|
149
149
|
artifacts:
|
|
150
150
|
when: always
|
|
151
151
|
reports:
|
|
@@ -57,7 +57,7 @@ jobs:
|
|
|
57
57
|
ref: ${{ github.event.ref }}
|
|
58
58
|
|
|
59
59
|
- name: Run TestDriver
|
|
60
|
-
run: npx testdriverai@latest run testdriver/test.yaml
|
|
60
|
+
run: npx testdriverai@latest run testdriver/test.yaml
|
|
61
61
|
env:
|
|
62
62
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
63
63
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -87,7 +87,7 @@ TestDriver CLI executes the test file in headless mode. The test file should con
|
|
|
87
87
|
|
|
88
88
|
```yaml
|
|
89
89
|
- name: Run TestDriver
|
|
90
|
-
run: npx testdriverai@latest run testdriver/test.yaml
|
|
90
|
+
run: npx testdriverai@latest run testdriver/test.yaml
|
|
91
91
|
env:
|
|
92
92
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
93
93
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -68,7 +68,7 @@ When running with `--heal` and `--write`, TestDriver will update the test files
|
|
|
68
68
|
</Step>
|
|
69
69
|
|
|
70
70
|
<Step title="Self-Healing">
|
|
71
|
-
- The AI detects the failure and searches for a similar element
|
|
71
|
+
- The AI detects the failure and searches for a similar element based on the `description`.
|
|
72
72
|
- It identifies the "Send" button as the updated element and retries the action.
|
|
73
73
|
</Step>
|
|
74
74
|
<Step title="Test Update">
|
|
@@ -109,8 +109,8 @@ jobs:
|
|
|
109
109
|
- name: Check out repository
|
|
110
110
|
uses: actions/checkout@v4
|
|
111
111
|
|
|
112
|
-
- name: Run TestDriver with
|
|
113
|
-
run: npx testdriverai@latest run testdriver/onboarding.yaml --heal --write
|
|
112
|
+
- name: Run TestDriver with Auto-Healing
|
|
113
|
+
run: npx testdriverai@latest run testdriver/onboarding.yaml --heal --write
|
|
114
114
|
env:
|
|
115
115
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
116
116
|
|
|
@@ -5,9 +5,7 @@ description: "Simplify test creation with AI-driven exploratory testing."
|
|
|
5
5
|
icon: "wand-sparkles"
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Writing tests can be a tedious and time-consuming task. TestDriver can generate tests just by exploring your app\! This guide will show you how to generate tests using TestDriver.
|
|
8
|
+
Writing tests can be a tedious and time-consuming task. TestDriver can generate tests just by exploring your app. This guide will show you how to generate tests using TestDriver.
|
|
11
9
|
|
|
12
10
|
# Generate exploratory tests
|
|
13
11
|
|
|
@@ -21,7 +19,7 @@ TestDriver will boot up.
|
|
|
21
19
|
|
|
22
20
|
```bash
|
|
23
21
|
❯ testdriverai
|
|
24
|
-
Howdy! I'm TestDriver
|
|
22
|
+
Howdy! I'm TestDriver v6.0.0
|
|
25
23
|
```
|
|
26
24
|
|
|
27
25
|
Ensure your website or app is visible on your test runner's display.
|
|
@@ -75,17 +75,8 @@ jobs:
|
|
|
75
75
|
with:
|
|
76
76
|
fetch-depth: 0
|
|
77
77
|
|
|
78
|
-
- name: Set up Node.js
|
|
79
|
-
uses: actions/setup-node@v4
|
|
80
|
-
with:
|
|
81
|
-
node-version: "20"
|
|
82
|
-
cache: "npm"
|
|
83
|
-
|
|
84
|
-
- name: Install dependencies
|
|
85
|
-
run: NODE_ENV=production npm ci
|
|
86
|
-
|
|
87
78
|
- name: Run test in headless mode
|
|
88
|
-
run: npx testdriverai@latest run testdriver/acceptance/${{ matrix.test }}
|
|
79
|
+
run: npx testdriverai@latest run testdriver/acceptance/${{ matrix.test }}
|
|
89
80
|
env:
|
|
90
81
|
FORCE_COLOR: 3
|
|
91
82
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
@@ -108,7 +99,6 @@ jobs:
|
|
|
108
99
|
2. **`run-tests` Job**:
|
|
109
100
|
|
|
110
101
|
- Uses the matrix strategy to dynamically create a job for each test file.
|
|
111
|
-
- Sets up Node.js and installs dependencies for TestDriver CLI.
|
|
112
102
|
- Runs each test file in parallel using `npx testdriverai@latest` with the `--headless` flag.
|
|
113
103
|
- The `max-parallel` setting limits the number of tests that can run in parallel to `8` (the max instances for the [Medium Team Plan](https://testdriver.ai/pricing)).
|
|
114
104
|
|
|
@@ -120,8 +110,8 @@ jobs:
|
|
|
120
110
|
4. **Environment Variables**:
|
|
121
111
|
|
|
122
112
|
- `TD_API_KEY`: Your TestDriver API key for authentication.
|
|
123
|
-
- `TD_WEBSITE`: The target website URL for testing.
|
|
124
|
-
- `TD_THIS_FILE`: The current test file being executed.
|
|
113
|
+
- `TD_WEBSITE`: The target website URL for testing if its a web application.
|
|
114
|
+
- `TD_THIS_FILE`: The current test file being executed, this will also be the title of the test recording.
|
|
125
115
|
|
|
126
116
|
5. **Fail-Fast**:
|
|
127
117
|
|
|
@@ -34,7 +34,7 @@ Reusable snippets are YAML files containing a set of test steps that perform a s
|
|
|
34
34
|
Save a YAML file containing the reusable steps. For example, create a `snippets/login.yaml` file for logging into an application:
|
|
35
35
|
|
|
36
36
|
```yaml
|
|
37
|
-
version:
|
|
37
|
+
version: 6.0.0
|
|
38
38
|
steps:
|
|
39
39
|
- prompt: Log in to the application
|
|
40
40
|
commands:
|
|
@@ -63,7 +63,7 @@ steps:
|
|
|
63
63
|
Use the [`run`](/commands/run) command to include the snippet in your main test file. For example:
|
|
64
64
|
|
|
65
65
|
```yaml
|
|
66
|
-
version:
|
|
66
|
+
version: 6.0.0
|
|
67
67
|
steps:
|
|
68
68
|
- prompt: Log in and navigate to the dashboard
|
|
69
69
|
commands:
|
|
@@ -95,7 +95,7 @@ TD_PASSWORD=your_password
|
|
|
95
95
|
You can chain multiple snippets together to create complex workflows. For example:
|
|
96
96
|
|
|
97
97
|
```yaml
|
|
98
|
-
version:
|
|
98
|
+
version: 6.0.0
|
|
99
99
|
steps:
|
|
100
100
|
- prompt: Log in, search for a product, and add it to the cart
|
|
101
101
|
commands:
|
|
@@ -18,7 +18,7 @@ Instead, TestDriver uses natural language prompts and AI-powered vision to inter
|
|
|
18
18
|
The following is an example of a TestDriver test.
|
|
19
19
|
|
|
20
20
|
```yaml
|
|
21
|
-
version:
|
|
21
|
+
version: 6.0.0
|
|
22
22
|
steps:
|
|
23
23
|
- prompt: Click the "Sign Up" button
|
|
24
24
|
commands:
|
|
@@ -44,7 +44,7 @@ If the button text changes to "Register," TestDriver's AI vision will still loca
|
|
|
44
44
|
TestDriver will then update the test to reflect the new UI by modifying the `text` field. Then, it will open a new pull request with the changes.
|
|
45
45
|
|
|
46
46
|
```yaml
|
|
47
|
-
version:
|
|
47
|
+
version: 6.0.0
|
|
48
48
|
steps:
|
|
49
49
|
- prompt: Click the "Register" button
|
|
50
50
|
commands:
|
|
@@ -54,6 +54,12 @@ steps:
|
|
|
54
54
|
action: click
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
+
<Note>
|
|
58
|
+
Need to add `--heal` flag during the run command to enable auto-healing of the
|
|
59
|
+
tests. Refer the [Auto-Healing](/getting-started/running#auto-healing) section
|
|
60
|
+
to know more.
|
|
61
|
+
</Note>
|
|
62
|
+
|
|
57
63
|
## Why selectorless testing?
|
|
58
64
|
|
|
59
65
|
Traditional testing frameworks rely on selectors tightly coupled to the codebase.
|
|
@@ -99,7 +99,7 @@ Visual assertions in TestDriver allow you to validate that your application beha
|
|
|
99
99
|
|
|
100
100
|
```yaml
|
|
101
101
|
- command: assert
|
|
102
|
-
expect:
|
|
102
|
+
expect: The chatbot response is displayed correctly
|
|
103
103
|
```
|
|
104
104
|
|
|
105
105
|
### Example: Asserting an image is present
|
|
@@ -113,7 +113,7 @@ Visual assertions in TestDriver allow you to validate that your application beha
|
|
|
113
113
|
|
|
114
114
|
```yaml
|
|
115
115
|
- command: assert
|
|
116
|
-
expect:
|
|
116
|
+
expect: The video is playing
|
|
117
117
|
```
|
|
118
118
|
|
|
119
119
|
---
|
|
@@ -9,7 +9,8 @@ This guide explains how to set up TestDriver in CI/CD pipelines, using **GitHub
|
|
|
9
9
|
|
|
10
10
|
## Prerequisites
|
|
11
11
|
|
|
12
|
-
1. **
|
|
12
|
+
1. **Add API key to secrets**: Obtain your API key from TestDriver and store it as a [CI/CD secret](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets) as `TD_API_KEY`.
|
|
13
|
+
Also make sure to add the `TD_WEBSITE` secret to the repository with the value of the website you are testing if its a web based application.
|
|
13
14
|
2. **Test Files**: Ensure your test files are saved in the `testdriver/` directory of your repository.
|
|
14
15
|
|
|
15
16
|
## Create a CI/CD workflow
|
|
@@ -17,6 +18,12 @@ This guide explains how to set up TestDriver in CI/CD pipelines, using **GitHub
|
|
|
17
18
|
This example uses GitHub Actions, but the same approach can be adapted for other CI/CD platforms:
|
|
18
19
|
|
|
19
20
|
1. Create a new file in your repository: `.github/workflows/testdriver.yaml`.
|
|
21
|
+
|
|
22
|
+
<Note>
|
|
23
|
+
If you have already [initialized the project](/getting-started/setup) with
|
|
24
|
+
`npx testdriverai@latest init`, the workflow file will already be created.
|
|
25
|
+
</Note>
|
|
26
|
+
|
|
20
27
|
2. Add the following workflow configuration:
|
|
21
28
|
|
|
22
29
|
### Example GitHub Actions workflow
|
|
@@ -30,67 +37,25 @@ on:
|
|
|
30
37
|
pull_request:
|
|
31
38
|
workflow_dispatch:
|
|
32
39
|
schedule:
|
|
33
|
-
- cron:
|
|
40
|
+
- cron: "0 13 * * *" # Every day at 1 PM UTC (adjust if needed for timezone)
|
|
34
41
|
|
|
35
42
|
permissions:
|
|
36
|
-
actions: read
|
|
37
43
|
contents: read
|
|
38
|
-
|
|
39
|
-
pull-requests: write
|
|
44
|
+
pull-requests: read
|
|
40
45
|
|
|
41
46
|
jobs:
|
|
42
|
-
gather-test-files:
|
|
43
|
-
name: Setup Test Matrix (./testdriver/*.yaml)
|
|
44
|
-
runs-on: ubuntu-latest
|
|
45
|
-
outputs:
|
|
46
|
-
test_files: ${{ steps.test_list.outputs.files }}
|
|
47
|
-
steps:
|
|
48
|
-
- name: Check out repository
|
|
49
|
-
uses: actions/checkout@v4
|
|
50
|
-
with:
|
|
51
|
-
ref: ${{ github.event.ref }}
|
|
52
|
-
- name: Find all test files and extract filenames
|
|
53
|
-
id: test_list
|
|
54
|
-
run: |
|
|
55
|
-
FILES=$(ls ./testdriver/*.yaml)
|
|
56
|
-
FILENAMES=$(basename -a $FILES)
|
|
57
|
-
FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]')
|
|
58
|
-
echo "files=$FILES_JSON" >> $GITHUB_OUTPUT
|
|
59
|
-
|
|
60
47
|
test:
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
matrix:
|
|
64
|
-
test: ${{ fromJson(needs.gather-test-files.outputs.test_files) }}
|
|
65
|
-
fail-fast: false
|
|
66
|
-
name: ${{ matrix.test }}
|
|
48
|
+
name: TestDriver
|
|
49
|
+
runs-on: ubuntu-latest
|
|
67
50
|
steps:
|
|
68
|
-
- name:
|
|
51
|
+
- name: Checkout repository
|
|
69
52
|
uses: actions/checkout@v4
|
|
70
|
-
with:
|
|
71
|
-
ref: ${{ github.event.ref }}
|
|
72
|
-
fetch-depth: 0
|
|
73
|
-
|
|
74
|
-
- name: Set up Node.js
|
|
75
|
-
uses: actions/setup-node@v4
|
|
76
|
-
with:
|
|
77
|
-
node-version: "20"
|
|
78
|
-
cache: "npm"
|
|
79
|
-
|
|
80
|
-
- name: Install dependencies
|
|
81
|
-
run: NODE_ENV=production npm ci
|
|
82
|
-
|
|
83
|
-
- name: Display filename being tested
|
|
84
|
-
run: echo "Running job for file: ${{ matrix.test }}"
|
|
85
53
|
|
|
86
54
|
- name: Run TestDriver
|
|
87
|
-
run: npx testdriverai@latest run testdriver
|
|
55
|
+
run: npx testdriverai@latest run testdriver/testdriver.yaml
|
|
88
56
|
env:
|
|
89
57
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
90
|
-
|
|
91
|
-
FORCE_COLOR: "3"
|
|
92
|
-
TD_WEBSITE: ${{ vars.TD_WEBSITE || 'https://your-app.com' }}
|
|
93
|
-
TD_THIS_FILE: ${{ matrix.test }}
|
|
58
|
+
TD_WEBSITE: ${{ secrets.TD_WEBSITE }}
|
|
94
59
|
```
|
|
95
60
|
|
|
96
61
|
## Trigger the workflow
|
|
@@ -102,7 +67,12 @@ jobs:
|
|
|
102
67
|
## View results
|
|
103
68
|
|
|
104
69
|
1. Navigate to the **Actions** tab in your GitHub repository (or equivalent CI/CD dashboard).
|
|
105
|
-
2. Select the workflow run to view the test results.
|
|
70
|
+
2. Select the workflow run to view the test results. You will also find the link to the test recording at the end of the run, the url is of the format: `https://app.dashcam.io/replay/1234567890`
|
|
71
|
+
|
|
72
|
+
<Tip>
|
|
73
|
+
You can also intregate our supported [reporting platform](/exporting/junit) to
|
|
74
|
+
get more currated results.
|
|
75
|
+
</Tip>
|
|
106
76
|
|
|
107
77
|
<Note>
|
|
108
78
|
For other CI/CD platforms, adapt the workflow syntax to match your platform's
|
|
@@ -153,3 +123,24 @@ TestDriver provides a powerful debugging interface through its platform. This in
|
|
|
153
123
|
|
|
154
124
|
- Share test results with your team by generating a shareable link.
|
|
155
125
|
- Add comments or annotations to specific steps to facilitate discussions and debugging.
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## How it works
|
|
130
|
+
|
|
131
|
+
1. **Trigger**: The GitHub workflow is triggered based on the conditions defined in the `on` section.
|
|
132
|
+
2. **Fetch the test files**: The test files are fetched from the repository based off the current branch.
|
|
133
|
+
3. **Running the tests**: The TestDriver CLI is installed and the tests are run via the [`run`](/commands/run) command.
|
|
134
|
+
4. **Authentication**: The `TD_API_KEY` environment variable authenticates your account.
|
|
135
|
+
5. **Dashcam Recording**: Dashcam begins recording the test execution.
|
|
136
|
+
6. **Test Execution**: The TestDriver CLI executes your commands on the cloud VM.
|
|
137
|
+
7. **Test Summary**: TestDriver summarizes the test and sets the exit code based on the pass or fail state.
|
|
138
|
+
8. **Cleanup**: The recording is uploaded to the TestDriver platform and the VM is destroyed, and all data is wiped.
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Additional features
|
|
143
|
+
|
|
144
|
+
- **Parallel testing**: You can run multiple tests in [parallel](/features/parallel-testing#setting-up-parallel-testing-with-a-matrix-strategy). The number of parallel instances depends on the plan you are on.
|
|
145
|
+
- **Environment Variables**: Pass [custom environment variables](guide/authentication#step-2%3A-pass-credentials-to-the-workflow) to configure credentials that are required for the tests.
|
|
146
|
+
- **Chaining Workflows**: A common workflow involves waiting for staging to deploy and testing against the staging environment by passing in the deployed url as `TD_WEBSITE` or the staging build's installation url depending on the kind of application you are testing.
|
|
@@ -6,8 +6,6 @@ icon: "download"
|
|
|
6
6
|
mode: "wide"
|
|
7
7
|
---
|
|
8
8
|
|
|
9
|
-
import Comments from "/snippets/comments.mdx";
|
|
10
|
-
|
|
11
9
|
<Steps>
|
|
12
10
|
<Step title="Set up your environment">
|
|
13
11
|
|
|
@@ -46,7 +44,7 @@ import Comments from "/snippets/comments.mdx";
|
|
|
46
44
|
- prompt: complete checkout
|
|
47
45
|
```
|
|
48
46
|
|
|
49
|
-
Each step has a `prompt` that describes what the agent should do. The agent will use the prompt to generate [commands](/commands) that make the tests faster and more reliable the next time you run the test.
|
|
47
|
+
Each step has a `prompt` that describes what the agent should do. The agent will use the prompt to generate [commands](/commands/assert) that make the tests faster and more reliable the next time you run the test.
|
|
50
48
|
|
|
51
49
|
</Step>
|
|
52
50
|
<Step title="Generate test steps from prompts">
|
|
@@ -54,7 +52,7 @@ import Comments from "/snippets/comments.mdx";
|
|
|
54
52
|
Run the following command to run the test file. TestDriver will spawn a virtual machine, launch the sandbox test page, and execute the steps defined in the `example.yaml` file.
|
|
55
53
|
|
|
56
54
|
```bash
|
|
57
|
-
npx testdriverai@
|
|
55
|
+
npx testdriverai@latest run example.yaml --write --heal
|
|
58
56
|
```
|
|
59
57
|
|
|
60
58
|
The `--write` flag tells TestDriver to save any generated commands to the test file, and the `--heal` flag allows TestDriver to recover from unexpected issues during the test run.
|
|
@@ -63,8 +61,7 @@ import Comments from "/snippets/comments.mdx";
|
|
|
63
61
|
<Step title="Run a regression test">
|
|
64
62
|
|
|
65
63
|
```yaml
|
|
66
|
-
version:
|
|
67
|
-
session: 682f6071811bd5a322c0e6dd
|
|
64
|
+
version: 6.0.0
|
|
68
65
|
steps:
|
|
69
66
|
- prompt: focus chrome
|
|
70
67
|
commands:
|
|
@@ -92,6 +89,3 @@ import Comments from "/snippets/comments.mdx";
|
|
|
92
89
|
|
|
93
90
|
</Step>
|
|
94
91
|
</Steps>
|
|
95
|
-
|
|
96
|
-
<Comments />
|
|
97
|
-
```
|
|
@@ -5,12 +5,8 @@ description: "Edit previously generated tests."
|
|
|
5
5
|
icon: "file-pen"
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
import YmlWarning from "/snippets/yml-warning.mdx";
|
|
9
|
-
|
|
10
8
|
Test scripts are written in YAML format, making them easy to read and modify manually if you need to.
|
|
11
9
|
|
|
12
|
-
<YmlWarning />
|
|
13
|
-
|
|
14
10
|
<Card title="Commands Reference" icon="link" href="/commands/assert">
|
|
15
11
|
Explore the available commands and their parameters.
|
|
16
12
|
</Card>
|
|
@@ -27,12 +23,12 @@ Use your preferred text editor or IDE (for example, Visual Studio Code) to open
|
|
|
27
23
|
|
|
28
24
|
Each test file consists of:
|
|
29
25
|
|
|
30
|
-
- **Version**: Specifies the TestDriver version.
|
|
26
|
+
- **Version**: Specifies the TestDriver version that was used to generate the test file. This is used to lock the version of TestDriver that is used to run the test.
|
|
31
27
|
- **Session**: A unique identifier for the test session.
|
|
32
28
|
- **Steps**: A list of prompts and commands to execute.
|
|
33
29
|
|
|
34
|
-
```yaml
|
|
35
|
-
version:
|
|
30
|
+
```yaml test-file.yaml
|
|
31
|
+
version: 6.0.0
|
|
36
32
|
session: abc1234
|
|
37
33
|
steps:
|
|
38
34
|
- prompt: Open Google Chrome and navigate to Airbnb
|
|
@@ -46,7 +42,8 @@ steps:
|
|
|
46
42
|
- command: type
|
|
47
43
|
text: airbnb.com
|
|
48
44
|
- command: press-keys
|
|
49
|
-
keys:
|
|
45
|
+
keys:
|
|
46
|
+
- enter
|
|
50
47
|
```
|
|
51
48
|
|
|
52
49
|
<Tip>
|
|
@@ -61,7 +58,7 @@ steps:
|
|
|
61
58
|
You can add, edit, or remove steps as needed. Each step contains:
|
|
62
59
|
|
|
63
60
|
- **Prompt**: A description of the action.
|
|
64
|
-
- **Commands**: The specific actions to perform.
|
|
61
|
+
- **Commands**: The specific actions to perform. It is an **optional** field, meaning, if there are no `commands` the AI considers the `prompt` and generates and executes the yaml on-the-go using the [exploratory mode](/interactive/explore)
|
|
65
62
|
|
|
66
63
|
#### Example edits:
|
|
67
64
|
|
|
@@ -86,7 +83,13 @@ Ensure the YAML file is properly formatted:
|
|
|
86
83
|
|
|
87
84
|
- Use consistent indentation (2 spaces recommended).
|
|
88
85
|
- Avoid trailing spaces or tabs.
|
|
89
|
-
-
|
|
86
|
+
- TestDriver will validate the yaml syntax and provide suggestions to fix any errors when you run the test.
|
|
87
|
+
|
|
88
|
+
<Tip>
|
|
89
|
+
It is recommended to use our [VS Code extension](/getting-started/vscode)
|
|
90
|
+
while you are editing the yaml. It will provide you with a rich experience
|
|
91
|
+
with auto-completion, syntax highlighting, and validation.
|
|
92
|
+
</Tip>
|
|
90
93
|
|
|
91
94
|
### Test the changes
|
|
92
95
|
|
|
@@ -17,6 +17,11 @@ npx testdriverai@latest run path/to/test.yaml
|
|
|
17
17
|
|
|
18
18
|
This will execute the test script located at `path/to/test.yaml`. If the test passes, the command will exit with a status code of `0`. If it fails, the command will exit with a status code of `1`.
|
|
19
19
|
|
|
20
|
+
<Note>
|
|
21
|
+
The path to the test file provided should be relative to the current working
|
|
22
|
+
directory.
|
|
23
|
+
</Note>
|
|
24
|
+
|
|
20
25
|
### Interactive mode
|
|
21
26
|
|
|
22
27
|
To run a test interactively:
|
|
@@ -24,13 +29,13 @@ To run a test interactively:
|
|
|
24
29
|
1. Launch the TestDriver CLI:
|
|
25
30
|
|
|
26
31
|
```bash
|
|
27
|
-
npx testdriverai@latest
|
|
32
|
+
npx testdriverai@latest edit
|
|
28
33
|
```
|
|
29
34
|
|
|
30
35
|
2. At the `>` prompt, type:
|
|
31
36
|
|
|
32
37
|
```bash
|
|
33
|
-
/run path/to/test.yaml
|
|
38
|
+
> /run path/to/test.yaml
|
|
34
39
|
```
|
|
35
40
|
|
|
36
41
|
## Notes
|