testdriverai 7.2.92 โ 7.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/index.js +53 -13
- package/agent/lib/analytics.js +4 -1
- package/agent/lib/commands.js +252 -252
- package/agent/lib/sandbox.js +1 -1
- package/ai/skills/testdriver:aws-setup/SKILL.md +4 -4
- package/ai/skills/testdriver:captcha/SKILL.md +1 -1
- package/ai/skills/testdriver:ci-cd/SKILL.md +23 -23
- package/ai/skills/testdriver:cloud/SKILL.md +1 -1
- package/ai/skills/testdriver:customizing-devices/SKILL.md +5 -5
- package/ai/skills/testdriver:running-tests/SKILL.md +11 -11
- package/ai/skills/testdriver:secrets/SKILL.md +1 -1
- package/ai/skills/testdriver:testdriver/SKILL.md +5 -5
- package/ai/skills/testdriver:variables/SKILL.md +3 -3
- package/debugger/index.html +0 -36
- package/lib/vitest/hooks.mjs +3 -0
- package/package.json +1 -1
- package/sdk-log-formatter.js +8 -1
- package/sdk.d.ts +86 -2
- package/sdk.js +126 -24
package/agent/lib/sandbox.js
CHANGED
|
@@ -144,7 +144,7 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
|
|
|
144
144
|
if (reply.traceId) {
|
|
145
145
|
this.traceId = reply.traceId;
|
|
146
146
|
logger.log('');
|
|
147
|
-
logger.log(`๐
|
|
147
|
+
logger.log(`๐ Trace Report (Share When Reporting Bugs):`);
|
|
148
148
|
logger.log(`https://testdriver.sentry.io/explore/traces/trace/${reply.traceId}`);
|
|
149
149
|
}
|
|
150
150
|
|
|
@@ -49,7 +49,7 @@ That's it! No manual instance management needed.
|
|
|
49
49
|
```bash
|
|
50
50
|
TD_OS=windows AWS_REGION=us-east-2 \
|
|
51
51
|
AWS_LAUNCH_TEMPLATE_ID=lt-xxx AMI_ID=ami-xxx \
|
|
52
|
-
|
|
52
|
+
vitest run
|
|
53
53
|
```
|
|
54
54
|
</Step>
|
|
55
55
|
</Steps>
|
|
@@ -219,7 +219,7 @@ TD_OS=windows \
|
|
|
219
219
|
AWS_REGION=us-east-2 \
|
|
220
220
|
AWS_LAUNCH_TEMPLATE_ID=lt-xxx \
|
|
221
221
|
AMI_ID=ami-0504bf50fad62f312 \
|
|
222
|
-
|
|
222
|
+
vitest run
|
|
223
223
|
```
|
|
224
224
|
|
|
225
225
|
<Note>
|
|
@@ -256,7 +256,7 @@ jobs:
|
|
|
256
256
|
run: npm ci
|
|
257
257
|
|
|
258
258
|
- name: Run Windows tests with self-hosted instances
|
|
259
|
-
run:
|
|
259
|
+
run: vitest run examples/*.test.mjs
|
|
260
260
|
env:
|
|
261
261
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
262
262
|
TD_OS: windows
|
|
@@ -306,7 +306,7 @@ For complete production examples, see:
|
|
|
306
306
|
If you already have a running instance, you can skip automatic spawning by providing `TD_IP`:
|
|
307
307
|
|
|
308
308
|
```bash
|
|
309
|
-
TD_OS=windows TD_IP=1.2.3.4
|
|
309
|
+
TD_OS=windows TD_IP=1.2.3.4 vitest run
|
|
310
310
|
```
|
|
311
311
|
|
|
312
312
|
The `setup-aws` hook will detect `TD_IP` is already set and skip spawning a new instance.
|
|
@@ -65,7 +65,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
65
65
|
- name: Run TestDriver tests
|
|
66
66
|
env:
|
|
67
67
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
68
|
-
run:
|
|
68
|
+
run: vitest --run
|
|
69
69
|
```
|
|
70
70
|
|
|
71
71
|
### Parallel Execution
|
|
@@ -95,7 +95,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
95
95
|
- name: Run tests (shard ${{ matrix.shard }}/4)
|
|
96
96
|
env:
|
|
97
97
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
98
|
-
run:
|
|
98
|
+
run: vitest --run --shard=${{ matrix.shard }}/4
|
|
99
99
|
```
|
|
100
100
|
|
|
101
101
|
### Multi-Platform Testing
|
|
@@ -124,7 +124,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
124
124
|
env:
|
|
125
125
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
126
126
|
TD_OS: ${{ matrix.td-os }}
|
|
127
|
-
run:
|
|
127
|
+
run: vitest --run
|
|
128
128
|
```
|
|
129
129
|
</Tab>
|
|
130
130
|
|
|
@@ -153,7 +153,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
153
153
|
- node_modules/
|
|
154
154
|
script:
|
|
155
155
|
- npm ci
|
|
156
|
-
-
|
|
156
|
+
- vitest --run
|
|
157
157
|
variables:
|
|
158
158
|
TD_API_KEY: $TD_API_KEY
|
|
159
159
|
```
|
|
@@ -178,22 +178,22 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
178
178
|
testdriver-shard-1:
|
|
179
179
|
extends: .testdriver-base
|
|
180
180
|
script:
|
|
181
|
-
-
|
|
181
|
+
- vitest --run --shard=1/4
|
|
182
182
|
|
|
183
183
|
testdriver-shard-2:
|
|
184
184
|
extends: .testdriver-base
|
|
185
185
|
script:
|
|
186
|
-
-
|
|
186
|
+
- vitest --run --shard=2/4
|
|
187
187
|
|
|
188
188
|
testdriver-shard-3:
|
|
189
189
|
extends: .testdriver-base
|
|
190
190
|
script:
|
|
191
|
-
-
|
|
191
|
+
- vitest --run --shard=3/4
|
|
192
192
|
|
|
193
193
|
testdriver-shard-4:
|
|
194
194
|
extends: .testdriver-base
|
|
195
195
|
script:
|
|
196
|
-
-
|
|
196
|
+
- vitest --run --shard=4/4
|
|
197
197
|
```
|
|
198
198
|
|
|
199
199
|
### Multi-Platform Testing
|
|
@@ -218,14 +218,14 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
218
218
|
variables:
|
|
219
219
|
TD_OS: linux
|
|
220
220
|
script:
|
|
221
|
-
-
|
|
221
|
+
- vitest --run
|
|
222
222
|
|
|
223
223
|
testdriver-windows:
|
|
224
224
|
extends: .testdriver-base
|
|
225
225
|
variables:
|
|
226
226
|
TD_OS: windows
|
|
227
227
|
script:
|
|
228
|
-
-
|
|
228
|
+
- vitest --run
|
|
229
229
|
```
|
|
230
230
|
</Tab>
|
|
231
231
|
|
|
@@ -260,7 +260,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
260
260
|
- node_modules
|
|
261
261
|
- run:
|
|
262
262
|
name: Run TestDriver tests
|
|
263
|
-
command:
|
|
263
|
+
command: vitest --run
|
|
264
264
|
environment:
|
|
265
265
|
TD_API_KEY: ${TD_API_KEY}
|
|
266
266
|
|
|
@@ -293,7 +293,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
293
293
|
- run:
|
|
294
294
|
name: Run TestDriver tests
|
|
295
295
|
command: |
|
|
296
|
-
|
|
296
|
+
vitest --run --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
|
|
297
297
|
environment:
|
|
298
298
|
TD_API_KEY: ${TD_API_KEY}
|
|
299
299
|
|
|
@@ -320,7 +320,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
320
320
|
- run: npm ci
|
|
321
321
|
- run:
|
|
322
322
|
name: Run TestDriver tests on << parameters.td-os >>
|
|
323
|
-
command:
|
|
323
|
+
command: vitest --run
|
|
324
324
|
environment:
|
|
325
325
|
TD_API_KEY: ${TD_API_KEY}
|
|
326
326
|
TD_OS: << parameters.td-os >>
|
|
@@ -364,7 +364,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
364
364
|
- script: npm ci
|
|
365
365
|
displayName: 'Install dependencies'
|
|
366
366
|
|
|
367
|
-
- script:
|
|
367
|
+
- script: vitest --run
|
|
368
368
|
displayName: 'Run TestDriver tests'
|
|
369
369
|
env:
|
|
370
370
|
TD_API_KEY: $(TD_API_KEY)
|
|
@@ -398,7 +398,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
398
398
|
- script: npm ci
|
|
399
399
|
displayName: 'Install dependencies'
|
|
400
400
|
|
|
401
|
-
- script:
|
|
401
|
+
- script: vitest --run --shard=$(SHARD)
|
|
402
402
|
displayName: 'Run TestDriver tests'
|
|
403
403
|
env:
|
|
404
404
|
TD_API_KEY: $(TD_API_KEY)
|
|
@@ -428,7 +428,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
428
428
|
- script: npm ci
|
|
429
429
|
displayName: 'Install dependencies'
|
|
430
430
|
|
|
431
|
-
- script:
|
|
431
|
+
- script: vitest --run
|
|
432
432
|
displayName: 'Run TestDriver tests on $(TD_OS)'
|
|
433
433
|
env:
|
|
434
434
|
TD_API_KEY: $(TD_API_KEY)
|
|
@@ -470,7 +470,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
470
470
|
|
|
471
471
|
stage('Test') {
|
|
472
472
|
steps {
|
|
473
|
-
sh '
|
|
473
|
+
sh 'vitest --run'
|
|
474
474
|
}
|
|
475
475
|
}
|
|
476
476
|
}
|
|
@@ -494,28 +494,28 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
494
494
|
agent { docker { image 'node:20' } }
|
|
495
495
|
steps {
|
|
496
496
|
sh 'npm ci'
|
|
497
|
-
sh '
|
|
497
|
+
sh 'vitest --run --shard=1/4'
|
|
498
498
|
}
|
|
499
499
|
}
|
|
500
500
|
stage('Shard 2') {
|
|
501
501
|
agent { docker { image 'node:20' } }
|
|
502
502
|
steps {
|
|
503
503
|
sh 'npm ci'
|
|
504
|
-
sh '
|
|
504
|
+
sh 'vitest --run --shard=2/4'
|
|
505
505
|
}
|
|
506
506
|
}
|
|
507
507
|
stage('Shard 3') {
|
|
508
508
|
agent { docker { image 'node:20' } }
|
|
509
509
|
steps {
|
|
510
510
|
sh 'npm ci'
|
|
511
|
-
sh '
|
|
511
|
+
sh 'vitest --run --shard=3/4'
|
|
512
512
|
}
|
|
513
513
|
}
|
|
514
514
|
stage('Shard 4') {
|
|
515
515
|
agent { docker { image 'node:20' } }
|
|
516
516
|
steps {
|
|
517
517
|
sh 'npm ci'
|
|
518
|
-
sh '
|
|
518
|
+
sh 'vitest --run --shard=4/4'
|
|
519
519
|
}
|
|
520
520
|
}
|
|
521
521
|
}
|
|
@@ -544,7 +544,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
544
544
|
}
|
|
545
545
|
steps {
|
|
546
546
|
sh 'npm ci'
|
|
547
|
-
sh '
|
|
547
|
+
sh 'vitest --run'
|
|
548
548
|
}
|
|
549
549
|
}
|
|
550
550
|
stage('Windows') {
|
|
@@ -554,7 +554,7 @@ TestDriver requires an API key to authenticate with the TestDriver cloud. Store
|
|
|
554
554
|
}
|
|
555
555
|
steps {
|
|
556
556
|
sh 'npm ci'
|
|
557
|
-
sh '
|
|
557
|
+
sh 'vitest --run'
|
|
558
558
|
}
|
|
559
559
|
}
|
|
560
560
|
}
|
|
@@ -88,10 +88,10 @@ Then pass the variable when running tests:
|
|
|
88
88
|
|
|
89
89
|
```bash
|
|
90
90
|
# Run tests on Windows
|
|
91
|
-
TD_OS=windows
|
|
91
|
+
TD_OS=windows vitest run
|
|
92
92
|
|
|
93
93
|
# Run tests on Linux (default)
|
|
94
|
-
TD_OS=linux
|
|
94
|
+
TD_OS=linux vitest run
|
|
95
95
|
```
|
|
96
96
|
|
|
97
97
|
This pattern is useful for running the same test suite across multiple operating systems in CI/CD:
|
|
@@ -102,7 +102,7 @@ strategy:
|
|
|
102
102
|
matrix:
|
|
103
103
|
os: [linux, windows]
|
|
104
104
|
steps:
|
|
105
|
-
- run: TD_OS=${{ matrix.os }}
|
|
105
|
+
- run: TD_OS=${{ matrix.os }} vitest run
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
## Keepalive
|
|
@@ -139,13 +139,13 @@ const testdriver = TestDriver(context, {
|
|
|
139
139
|
Then, you can run both tests in sequence:
|
|
140
140
|
|
|
141
141
|
```bash
|
|
142
|
-
|
|
142
|
+
vitest run -t known-good.test.mjs -t work-in-progress.test.mjs
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
And as you make changes to `work-in-progress.test.mjs`, you can re-run just that file to quickly iterate on the failing steps.
|
|
146
146
|
|
|
147
147
|
```bash
|
|
148
|
-
|
|
148
|
+
vitest run work-in-progress.test.mjs
|
|
149
149
|
```
|
|
150
150
|
|
|
151
151
|
<Warning>
|
|
@@ -13,7 +13,7 @@ TestDriver works with Vitest's powerful test runner.
|
|
|
13
13
|
### Run All Tests
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
|
|
16
|
+
vitest run
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
Executes all test files in your project once and exits. Vitest automatically discovers files matching patterns like `*.test.js`, `*.test.mjs`, or `*.spec.js`.
|
|
@@ -21,7 +21,7 @@ Executes all test files in your project once and exits. Vitest automatically dis
|
|
|
21
21
|
### Run with Coverage
|
|
22
22
|
|
|
23
23
|
```bash
|
|
24
|
-
|
|
24
|
+
vitest run --coverage
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
Generates a code coverage report showing which lines of your source code were executed during tests. Coverage helps identify untested code paths. Results are displayed in the terminal and saved to a `coverage/` directory.
|
|
@@ -33,13 +33,13 @@ Generates a code coverage report showing which lines of your source code were ex
|
|
|
33
33
|
### Run Specific Tests
|
|
34
34
|
|
|
35
35
|
```bash
|
|
36
|
-
|
|
36
|
+
vitest run login.test.js
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
Runs only the specified test file. Useful when debugging a single test or working on a specific feature.
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
|
-
|
|
42
|
+
vitest run login.test.js checkout.test.js
|
|
43
43
|
```
|
|
44
44
|
|
|
45
45
|
Runs multiple specific test files. List as many files as needed, separated by spaces.
|
|
@@ -47,7 +47,7 @@ Runs multiple specific test files. List as many files as needed, separated by sp
|
|
|
47
47
|
### Filter Tests by Name
|
|
48
48
|
|
|
49
49
|
```bash
|
|
50
|
-
|
|
50
|
+
vitest run --grep "login"
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
The `--grep` flag filters tests by their name (the string passed to `it()` or `test()`). Only tests whose names match the pattern will run. Supports regex patterns for complex matching.
|
|
@@ -55,7 +55,7 @@ The `--grep` flag filters tests by their name (the string passed to `it()` or `t
|
|
|
55
55
|
### Run Tests in a Folder
|
|
56
56
|
|
|
57
57
|
```bash
|
|
58
|
-
|
|
58
|
+
vitest run tests/e2e/
|
|
59
59
|
```
|
|
60
60
|
|
|
61
61
|
Runs all test files within a specific directory. Great for organizing tests by type (unit, integration, e2e) and running them separately.
|
|
@@ -67,7 +67,7 @@ TestDriver runs each test in its own cloud sandbox, enabling true parallel execu
|
|
|
67
67
|
### Control Concurrency
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
|
|
70
|
+
vitest run --maxConcurrency=5
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
The `--maxConcurrency` flag limits how many tests run simultaneously. This should match your TestDriver license slots to avoid failures from exhausted slots.
|
|
@@ -75,7 +75,7 @@ The `--maxConcurrency` flag limits how many tests run simultaneously. This shoul
|
|
|
75
75
|
### Thread Configuration
|
|
76
76
|
|
|
77
77
|
```bash
|
|
78
|
-
|
|
78
|
+
vitest run --pool=threads --minThreads=2 --maxThreads=8
|
|
79
79
|
```
|
|
80
80
|
|
|
81
81
|
Fine-tune thread allocation for optimal performance:
|
|
@@ -139,7 +139,7 @@ export default defineConfig({
|
|
|
139
139
|
Use Vitest UI for interactive debugging:
|
|
140
140
|
|
|
141
141
|
```bash
|
|
142
|
-
|
|
142
|
+
vitest --ui
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
The `--ui` flag launches a web-based interface for managing your test suite. Unlike `vitest run`, this starts in watch mode by default.
|
|
@@ -152,7 +152,7 @@ Open http://localhost:51204 to see:
|
|
|
152
152
|
- **Filter and search** โ Quickly find tests by name or status
|
|
153
153
|
|
|
154
154
|
<Tip>
|
|
155
|
-
Combine with `--open` to automatically open the UI in your browser: `
|
|
155
|
+
Combine with `--open` to automatically open the UI in your browser: `vitest --ui --open`
|
|
156
156
|
</Tip>
|
|
157
157
|
|
|
158
158
|
|
|
@@ -167,7 +167,7 @@ Reports include:
|
|
|
167
167
|
- **Error details** - Debug failures with full context
|
|
168
168
|
|
|
169
169
|
```bash
|
|
170
|
-
$
|
|
170
|
+
$ vitest run
|
|
171
171
|
|
|
172
172
|
โ login.test.js (2) 18.4s
|
|
173
173
|
โ user can login 12.3s
|
|
@@ -57,7 +57,7 @@ Store sensitive credentials as GitHub repository secrets so they're never expose
|
|
|
57
57
|
TD_API_KEY: ${{ secrets.TD_API_KEY }}
|
|
58
58
|
TD_USERNAME: ${{ secrets.TD_USERNAME }}
|
|
59
59
|
TD_PASSWORD: ${{ secrets.TD_PASSWORD }}
|
|
60
|
-
run:
|
|
60
|
+
run: vitest run
|
|
61
61
|
```
|
|
62
62
|
</Step>
|
|
63
63
|
</Steps>
|
|
@@ -34,7 +34,7 @@ Use this agent when the user asks to:
|
|
|
34
34
|
4. **โ ๏ธ WRITE CODE IMMEDIATELY**: After EVERY successful action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the end.
|
|
35
35
|
5. **Verify Actions**: Use `check` after actions to verify they succeeded (for YOUR understanding only).
|
|
36
36
|
6. **Add Assertions**: Use `assert` for test conditions that should be in the final test file.
|
|
37
|
-
7. **โ ๏ธ RUN THE TEST YOURSELF**: Use `
|
|
37
|
+
7. **โ ๏ธ RUN THE TEST YOURSELF**: Use `vitest run <testFile> --reporter=dot` to run the test - do NOT tell the user to run it. Iterate until it passes.
|
|
38
38
|
|
|
39
39
|
## Prerequisites
|
|
40
40
|
|
|
@@ -224,7 +224,7 @@ await testdriver.screenshot(1, false, true);
|
|
|
224
224
|
|
|
225
225
|
**Every MCP tool response includes "ACTION REQUIRED: Append this code..." - you MUST write that code to the test file IMMEDIATELY before proceeding to the next action.**
|
|
226
226
|
|
|
227
|
-
**When ready to validate, RUN THE TEST YOURSELF using `
|
|
227
|
+
**When ready to validate, RUN THE TEST YOURSELF using `vitest run`. Do NOT tell the user to run it.**
|
|
228
228
|
|
|
229
229
|
### Step 1: Start a Session
|
|
230
230
|
|
|
@@ -284,7 +284,7 @@ assert({ assertion: "the dashboard is visible" })
|
|
|
284
284
|
**โ ๏ธ YOU must run the test - do NOT tell the user to run it:**
|
|
285
285
|
|
|
286
286
|
```bash
|
|
287
|
-
|
|
287
|
+
vitest run tests/login.test.mjs --reporter=dot
|
|
288
288
|
```
|
|
289
289
|
|
|
290
290
|
**Always use `--reporter=dot`** for cleaner, more concise output that's easier to parse.
|
|
@@ -356,7 +356,7 @@ view_local_screenshot({ path: ".testdriver/screenshots/checkout.test/before-asse
|
|
|
356
356
|
### Tips for MCP Workflow
|
|
357
357
|
|
|
358
358
|
1. **โ ๏ธ Write code IMMEDIATELY** - After EVERY action, append generated code to test file RIGHT AWAY
|
|
359
|
-
2. **โ ๏ธ Run tests YOURSELF** - Use `
|
|
359
|
+
2. **โ ๏ธ Run tests YOURSELF** - Use `vitest run` - do NOT tell user to run tests
|
|
360
360
|
3. **โ ๏ธ Add screenshots liberally** - Include `await testdriver.screenshot()` after every significant action for debugging
|
|
361
361
|
4. **โ ๏ธ Use screenshot viewing for debugging** - When tests fail, use `list_local_screenshots` and `view_local_screenshot` to understand what went wrong
|
|
362
362
|
5. **Work incrementally** - Don't try to build the entire test at once
|
|
@@ -500,7 +500,7 @@ const result = await testdriver.assert("dashboard is visible");
|
|
|
500
500
|
## Tips for Agents
|
|
501
501
|
|
|
502
502
|
1. **โ ๏ธ WRITE CODE IMMEDIATELY** - After EVERY successful MCP action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the session ends.
|
|
503
|
-
2. **โ ๏ธ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `
|
|
503
|
+
2. **โ ๏ธ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `vitest run <testFile> --reporter=dot`. Always use `--reporter=dot` for cleaner output. Analyze the output and iterate until the test passes. **Always share the test report link** (e.g., `https://app.testdriver.ai/projects/.../reports/...`) with the user after each run.
|
|
504
504
|
3. **โ ๏ธ ADD SCREENSHOTS LIBERALLY** - Include `await testdriver.screenshot()` throughout your tests: after provision, before/after clicks, after typing, and before assertions. This creates a visual trail that makes debugging failures much easier.
|
|
505
505
|
4. **โ ๏ธ USE SCREENSHOT VIEWING FOR DEBUGGING** - When tests fail, use `list_local_screenshots` and `view_local_screenshot` MCP commands to see exactly what the UI looked like. This is often faster than re-running the test.
|
|
506
506
|
5. **โ ๏ธ NEVER USE `.wait()`** - Do NOT use any `.wait()` method. Instead, use `find()` with a `timeout` option to poll for elements, or use `assert()` / `check()` to verify state. Explicit waits are flaky and slow.
|
|
@@ -32,9 +32,9 @@ test('multi-environment testing', async (context) => {
|
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
# Run against different environments
|
|
35
|
-
TEST_ENV=dev
|
|
36
|
-
TEST_ENV=staging
|
|
37
|
-
TEST_ENV=production
|
|
35
|
+
TEST_ENV=dev vitest run
|
|
36
|
+
TEST_ENV=staging vitest run
|
|
37
|
+
TEST_ENV=production vitest run
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
## Test Fixtures
|
package/debugger/index.html
CHANGED
|
@@ -309,45 +309,9 @@
|
|
|
309
309
|
user-select: none;
|
|
310
310
|
}
|
|
311
311
|
|
|
312
|
-
.close-button {
|
|
313
|
-
position: fixed;
|
|
314
|
-
top: 12px;
|
|
315
|
-
right: 12px;
|
|
316
|
-
z-index: 100;
|
|
317
|
-
background: rgba(0, 0, 0, 0.8);
|
|
318
|
-
border: 1px solid #444;
|
|
319
|
-
color: #fff;
|
|
320
|
-
padding: 8px 16px;
|
|
321
|
-
border-radius: 6px;
|
|
322
|
-
cursor: pointer;
|
|
323
|
-
font-size: 13px;
|
|
324
|
-
font-weight: 500;
|
|
325
|
-
pointer-events: auto;
|
|
326
|
-
transition: all 0.2s ease;
|
|
327
|
-
display: flex;
|
|
328
|
-
align-items: center;
|
|
329
|
-
gap: 6px;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
.close-button:hover {
|
|
333
|
-
background: rgba(220, 53, 69, 0.9);
|
|
334
|
-
border-color: #dc3545;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
.close-button svg {
|
|
338
|
-
width: 14px;
|
|
339
|
-
height: 14px;
|
|
340
|
-
fill: currentColor;
|
|
341
|
-
}
|
|
342
312
|
</style>
|
|
343
313
|
</head>
|
|
344
314
|
<body>
|
|
345
|
-
<!-- Close window button -->
|
|
346
|
-
<button class="close-button" onclick="window.close()" title="Close this window">
|
|
347
|
-
<svg viewBox="0 0 24 24"><path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
|
|
348
|
-
Close
|
|
349
|
-
</button>
|
|
350
|
-
|
|
351
315
|
<!-- Loading screen -->
|
|
352
316
|
<div class="loading-screen" id="loading-screen">
|
|
353
317
|
<div class="testdriver-logo">
|
package/lib/vitest/hooks.mjs
CHANGED
|
@@ -191,6 +191,7 @@ const lifecycleHandlers = new WeakMap();
|
|
|
191
191
|
* });
|
|
192
192
|
*/
|
|
193
193
|
export function TestDriver(context, options = {}) {
|
|
194
|
+
console.log("[DEBUG hooks entry] options:", JSON.stringify(options));
|
|
194
195
|
if (!context || !context.task) {
|
|
195
196
|
throw new Error(
|
|
196
197
|
'TestDriver() requires Vitest context. Pass the context parameter from your test function: test("name", async (context) => { ... })',
|
|
@@ -246,6 +247,8 @@ export function TestDriver(context, options = {}) {
|
|
|
246
247
|
config.apiRoot = process.env.TD_API_ROOT;
|
|
247
248
|
}
|
|
248
249
|
|
|
250
|
+
console.log("[DEBUG hooks] options.preview:", options.preview, "config.preview:", config.preview);
|
|
251
|
+
|
|
249
252
|
const testdriver = new TestDriverSDK(apiKey, config);
|
|
250
253
|
testdriver.__vitestContext = context.task;
|
|
251
254
|
testdriver._debugOnFailure = mergedOptions.debugOnFailure || false;
|
package/package.json
CHANGED
package/sdk-log-formatter.js
CHANGED
|
@@ -477,9 +477,10 @@ class SDKLogFormatter {
|
|
|
477
477
|
* @param {boolean} passed - Whether assertion passed
|
|
478
478
|
* @param {string} response - The AI response message
|
|
479
479
|
* @param {number} durationMs - Duration in milliseconds
|
|
480
|
+
* @param {boolean} cacheHit - Whether the result was from cache
|
|
480
481
|
* @returns {string} Formatted result line
|
|
481
482
|
*/
|
|
482
|
-
formatAssertResult(passed, response, durationMs) {
|
|
483
|
+
formatAssertResult(passed, response, durationMs, cacheHit = false) {
|
|
483
484
|
const parts = [];
|
|
484
485
|
this.addTimestamp(parts);
|
|
485
486
|
parts.push(this.getResultPrefix());
|
|
@@ -490,6 +491,12 @@ class SDKLogFormatter {
|
|
|
490
491
|
parts.push(chalk.red("failed"));
|
|
491
492
|
}
|
|
492
493
|
|
|
494
|
+
// Add cache hit indicator (like find does)
|
|
495
|
+
if (cacheHit) {
|
|
496
|
+
parts.push(chalk.dim("ยท"));
|
|
497
|
+
parts.push(chalk.bold.yellow("โก cached"));
|
|
498
|
+
}
|
|
499
|
+
|
|
493
500
|
// Add the response message (trimmed)
|
|
494
501
|
if (response) {
|
|
495
502
|
const trimmedResponse = response.trim().split('\n')[0]; // First line only
|
package/sdk.d.ts
CHANGED
|
@@ -364,6 +364,42 @@ export interface HoverResult {
|
|
|
364
364
|
[key: string]: any;
|
|
365
365
|
}
|
|
366
366
|
|
|
367
|
+
/** Bounding box for an OCR word */
|
|
368
|
+
export interface OCRBoundingBox {
|
|
369
|
+
/** Left edge X coordinate */
|
|
370
|
+
x0: number;
|
|
371
|
+
/** Top edge Y coordinate */
|
|
372
|
+
y0: number;
|
|
373
|
+
/** Right edge X coordinate */
|
|
374
|
+
x1: number;
|
|
375
|
+
/** Bottom edge Y coordinate */
|
|
376
|
+
y1: number;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Individual word extracted by OCR */
|
|
380
|
+
export interface OCRWord {
|
|
381
|
+
/** The text content of the word */
|
|
382
|
+
content: string;
|
|
383
|
+
/** Confidence score for this word (0-100) */
|
|
384
|
+
confidence: number;
|
|
385
|
+
/** Bounding box coordinates */
|
|
386
|
+
bbox: OCRBoundingBox;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** Result from OCR text extraction */
|
|
390
|
+
export interface OCRResult {
|
|
391
|
+
/** Array of extracted words with positions */
|
|
392
|
+
words: OCRWord[];
|
|
393
|
+
/** All text concatenated with spaces */
|
|
394
|
+
fullText: string;
|
|
395
|
+
/** Overall OCR confidence (0-100) */
|
|
396
|
+
confidence: number;
|
|
397
|
+
/** Width of the analyzed screenshot */
|
|
398
|
+
imageWidth: number;
|
|
399
|
+
/** Height of the analyzed screenshot */
|
|
400
|
+
imageHeight: number;
|
|
401
|
+
}
|
|
402
|
+
|
|
367
403
|
// ====================================
|
|
368
404
|
// Command Options Interfaces
|
|
369
405
|
// ====================================
|
|
@@ -520,6 +556,14 @@ export interface ExtractOptions {
|
|
|
520
556
|
export interface AssertOptions {
|
|
521
557
|
/** Assertion to check */
|
|
522
558
|
assertion: string;
|
|
559
|
+
/** Cache threshold (0-1). Lower values require closer matches. Set to -1 to disable cache. */
|
|
560
|
+
threshold?: number;
|
|
561
|
+
/** Cache key for grouping cached assertions (enables caching when provided) */
|
|
562
|
+
cacheKey?: string;
|
|
563
|
+
/** Operating system identifier for cache partitioning */
|
|
564
|
+
os?: string;
|
|
565
|
+
/** Screen resolution for cache partitioning */
|
|
566
|
+
resolution?: string;
|
|
523
567
|
}
|
|
524
568
|
|
|
525
569
|
/** Options for exec command */
|
|
@@ -1209,9 +1253,21 @@ export default class TestDriverSDK {
|
|
|
1209
1253
|
/**
|
|
1210
1254
|
* Make an AI-powered assertion
|
|
1211
1255
|
* @param assertion - Assertion to check
|
|
1212
|
-
* @param options -
|
|
1256
|
+
* @param options - Cache options for the assertion
|
|
1257
|
+
*
|
|
1258
|
+
* @example
|
|
1259
|
+
* // Simple assertion
|
|
1260
|
+
* await client.assert('the login form is visible');
|
|
1261
|
+
*
|
|
1262
|
+
* @example
|
|
1263
|
+
* // With caching enabled via cacheKey
|
|
1264
|
+
* await client.assert('the submit button is enabled', { cacheKey: 'my-test-run' });
|
|
1265
|
+
*
|
|
1266
|
+
* @example
|
|
1267
|
+
* // With custom threshold
|
|
1268
|
+
* await client.assert('the page loaded', { threshold: 0.01, cacheKey: 'login-test' });
|
|
1213
1269
|
*/
|
|
1214
|
-
assert(assertion: string, options?:
|
|
1270
|
+
assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string }): Promise<boolean>;
|
|
1215
1271
|
|
|
1216
1272
|
/**
|
|
1217
1273
|
* Extract information from the screen using AI
|
|
@@ -1269,6 +1325,34 @@ export default class TestDriverSDK {
|
|
|
1269
1325
|
*/
|
|
1270
1326
|
screenshot(filename?: string): Promise<string>;
|
|
1271
1327
|
|
|
1328
|
+
/**
|
|
1329
|
+
* Extract all visible text from the current screen using OCR (Tesseract)
|
|
1330
|
+
* Returns structured data with text content, bounding boxes, and confidence scores
|
|
1331
|
+
*
|
|
1332
|
+
* @returns OCR extraction result with words, positions, and confidence
|
|
1333
|
+
*
|
|
1334
|
+
* @example
|
|
1335
|
+
* // Get all text on screen
|
|
1336
|
+
* const result = await testdriver.ocr();
|
|
1337
|
+
* console.log(result.fullText);
|
|
1338
|
+
*
|
|
1339
|
+
* @example
|
|
1340
|
+
* // Find and click text
|
|
1341
|
+
* const result = await testdriver.ocr();
|
|
1342
|
+
* const submit = result.words.find(w => w.content === 'Submit');
|
|
1343
|
+
* if (submit) {
|
|
1344
|
+
* const x = (submit.bbox.x0 + submit.bbox.x1) / 2;
|
|
1345
|
+
* const y = (submit.bbox.y0 + submit.bbox.y1) / 2;
|
|
1346
|
+
* await testdriver.click({ x, y });
|
|
1347
|
+
* }
|
|
1348
|
+
*
|
|
1349
|
+
* @example
|
|
1350
|
+
* // Check if text exists
|
|
1351
|
+
* const result = await testdriver.ocr();
|
|
1352
|
+
* const hasError = result.words.some(w => w.content.toLowerCase().includes('error'));
|
|
1353
|
+
*/
|
|
1354
|
+
ocr(): Promise<OCRResult>;
|
|
1355
|
+
|
|
1272
1356
|
/**
|
|
1273
1357
|
* Wait for specified time
|
|
1274
1358
|
* @deprecated Consider using element polling with find() instead of arbitrary waits
|