testdriverai 7.9.8-test → 7.9.10-test
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/lib/sandbox.js +6 -6
- package/agent/lib/sdk.js +1 -4
- package/ai/agents/testdriver.md +3 -5
- package/ai/skills/testdriver-testdriver/SKILL.md +3 -5
- package/docs/_data/examples-manifest.json +42 -42
- package/docs/docs.json +0 -1
- package/docs/v7/examples/ai.mdx +1 -1
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +1 -1
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/exec-output.mdx +1 -1
- package/docs/v7/examples/exec-pwsh.mdx +1 -1
- package/docs/v7/examples/findall-coffee-icons.mdx +1 -1
- package/docs/v7/examples/focus-window.mdx +1 -1
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text-with-description.mdx +1 -1
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +1 -1
- package/docs/v7/examples/parse.mdx +1 -1
- package/docs/v7/examples/press-keys.mdx +1 -1
- package/docs/v7/examples/prompt.mdx +1 -1
- package/docs/v7/examples/scroll-keyboard.mdx +1 -1
- package/docs/v7/examples/scroll-until-image.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/docs/v7/examples/windows-installer.mdx +1 -1
- package/interfaces/vitest-plugin.mjs +6 -5
- package/lib/core/Dashcam.js +1 -1
- package/lib/github-comment.mjs +11 -5
- package/package.json +1 -1
- package/sdk.d.ts +1 -1
- package/sdk.js +7 -3
package/agent/lib/sandbox.js
CHANGED
|
@@ -1283,12 +1283,12 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
|
|
|
1283
1283
|
var rate = (this._publishCount / windowElapsed) * 1000;
|
|
1284
1284
|
var rateStr = rate.toFixed(1);
|
|
1285
1285
|
|
|
1286
|
-
// Log rate - warning if approaching limit, debug otherwise
|
|
1287
|
-
if (rate > 45) {
|
|
1288
|
-
|
|
1289
|
-
} else if (process.env.VERBOSE || process.env.TD_DEBUG) {
|
|
1290
|
-
|
|
1291
|
-
}
|
|
1286
|
+
// // Log rate - warning if approaching limit, debug otherwise
|
|
1287
|
+
// if (rate > 45) {
|
|
1288
|
+
// logger.warn("Ably publish rate: " + rateStr + " msg/sec (approaching 50/sec limit)");
|
|
1289
|
+
// } else if (process.env.VERBOSE || process.env.TD_DEBUG) {
|
|
1290
|
+
// logger.log("Ably publish rate: " + rateStr + " msg/sec");
|
|
1291
|
+
// }
|
|
1292
1292
|
|
|
1293
1293
|
// Reset window
|
|
1294
1294
|
this._publishCount = 0;
|
package/agent/lib/sdk.js
CHANGED
|
@@ -395,10 +395,7 @@ const createSDK = (emitter, config, sessionInstance) => {
|
|
|
395
395
|
const savedKB = (data.image.length / 1024).toFixed(0);
|
|
396
396
|
delete data.image;
|
|
397
397
|
data.imageKey = imageKey;
|
|
398
|
-
|
|
399
|
-
path,
|
|
400
|
-
message: `[sdk] uploaded screenshot to S3 (saved ${savedKB}KB inline), imageKey=${imageKey}`,
|
|
401
|
-
});
|
|
398
|
+
|
|
402
399
|
}
|
|
403
400
|
} catch (uploadErr) {
|
|
404
401
|
// Non-fatal: fall back to sending base64 inline
|
package/ai/agents/testdriver.md
CHANGED
|
@@ -44,7 +44,7 @@ Use this agent when the user asks to:
|
|
|
44
44
|
4. **⚠️ WRITE CODE IMMEDIATELY**: After EVERY successful action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the end.
|
|
45
45
|
5. **Verify Actions**: Use `check` after actions to verify they succeeded (for YOUR understanding only).
|
|
46
46
|
6. **Add Assertions**: Use `assert` for test conditions that should be in the final test file.
|
|
47
|
-
7. **⚠️ RUN THE TEST YOURSELF**: Use `vitest run <testFile
|
|
47
|
+
7. **⚠️ RUN THE TEST YOURSELF**: Use `vitest run <testFile>` to run the test - do NOT tell the user to run it. Iterate until it passes. **NEVER use `npx vitest`** - always use `vitest` directly.
|
|
48
48
|
8. **⚠️ SHARE THE TEST REPORT**: After EVERY test run, find the `TESTDRIVER_RUN_URL` in the output (e.g., `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...`) and share it with the user so they can view the recording and results.
|
|
49
49
|
|
|
50
50
|
## Prerequisites
|
|
@@ -315,11 +315,9 @@ assert({ assertion: "the dashboard is visible" })
|
|
|
315
315
|
**⚠️ YOU must run the test - do NOT tell the user to run it. NEVER use `npx vitest` - always use `vitest` directly:**
|
|
316
316
|
|
|
317
317
|
```bash
|
|
318
|
-
vitest run tests/login.test.mjs
|
|
318
|
+
vitest run tests/login.test.mjs
|
|
319
319
|
```
|
|
320
320
|
|
|
321
|
-
**Always use `--reporter=dot`** for cleaner, more concise output that's easier to parse.
|
|
322
|
-
|
|
323
321
|
Analyze the output, fix any issues, and iterate until the test passes.
|
|
324
322
|
|
|
325
323
|
**⚠️ ALWAYS share the test report link with the user.** After each test run, look for `TESTDRIVER_RUN_URL` in the test output (e.g., `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...`) and share it with the user so they can view the recording and results. This is CRITICAL - users need to see the visual recording to understand test behavior.
|
|
@@ -614,7 +612,7 @@ await testdriver.screenshot(1, false, true);
|
|
|
614
612
|
## Tips for Agents
|
|
615
613
|
|
|
616
614
|
1. **⚠️ WRITE CODE IMMEDIATELY** - After EVERY successful MCP action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the session ends.
|
|
617
|
-
2. **⚠️ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `vitest run <testFile
|
|
615
|
+
2. **⚠️ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `vitest run <testFile>` (NEVER use `npx vitest`). Analyze the output and iterate until the test passes.
|
|
618
616
|
3. **⚠️ SHARE THE TEST REPORT URL** - After EVERY test run, find `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...` in the output and share it with the user. This is CRITICAL - users need to view the recording to understand what happened.
|
|
619
617
|
3. **Screenshots are automatic** - TestDriver captures screenshots before/after every command by default. Each screenshot filename includes the line number (e.g., `001-click-before-L42-submit-button.png`) making it easy to trace issues.
|
|
620
618
|
4. **⚠️ USE SCREENSHOT VIEWING FOR DEBUGGING** - When tests fail, use `list_local_screenshots` and `view_local_screenshot` MCP commands to see exactly what the UI looked like. The filenames tell you which line of code triggered each screenshot.
|
|
@@ -34,7 +34,7 @@ Use this agent when the user asks to:
|
|
|
34
34
|
4. **⚠️ WRITE CODE IMMEDIATELY**: After EVERY successful action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the end.
|
|
35
35
|
5. **Verify Actions**: Use `check` after actions to verify they succeeded (for YOUR understanding only).
|
|
36
36
|
6. **Add Assertions**: Use `assert` for test conditions that should be in the final test file.
|
|
37
|
-
7. **⚠️ RUN THE TEST YOURSELF**: Use `vitest run <testFile
|
|
37
|
+
7. **⚠️ RUN THE TEST YOURSELF**: Use `vitest run <testFile>` to run the test - do NOT tell the user to run it. Iterate until it passes. **NEVER use `npx vitest`** - always use `vitest` directly.
|
|
38
38
|
8. **⚠️ SHARE THE TEST REPORT**: After EVERY test run, find the `TESTDRIVER_RUN_URL` in the output (e.g., `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...`) and share it with the user so they can view the recording and results.
|
|
39
39
|
|
|
40
40
|
## Prerequisites
|
|
@@ -305,11 +305,9 @@ assert({ assertion: "the dashboard is visible" })
|
|
|
305
305
|
**⚠️ YOU must run the test - do NOT tell the user to run it. NEVER use `npx vitest` - always use `vitest` directly:**
|
|
306
306
|
|
|
307
307
|
```bash
|
|
308
|
-
vitest run tests/login.test.mjs
|
|
308
|
+
vitest run tests/login.test.mjs
|
|
309
309
|
```
|
|
310
310
|
|
|
311
|
-
**Always use `--reporter=dot`** for cleaner, more concise output that's easier to parse.
|
|
312
|
-
|
|
313
311
|
Analyze the output, fix any issues, and iterate until the test passes.
|
|
314
312
|
|
|
315
313
|
**⚠️ ALWAYS share the test report link with the user.** After each test run, look for `TESTDRIVER_RUN_URL` in the test output (e.g., `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...`) and share it with the user so they can view the recording and results. This is CRITICAL - users need to see the visual recording to understand test behavior.
|
|
@@ -604,7 +602,7 @@ await testdriver.screenshot(1, false, true);
|
|
|
604
602
|
## Tips for Agents
|
|
605
603
|
|
|
606
604
|
1. **⚠️ WRITE CODE IMMEDIATELY** - After EVERY successful MCP action, append the generated code to the test file RIGHT AWAY. Do NOT wait until the session ends.
|
|
607
|
-
2. **⚠️ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `vitest run <testFile
|
|
605
|
+
2. **⚠️ RUN TESTS YOURSELF** - Do NOT tell the user to run tests. YOU must run the tests using `vitest run <testFile>` (NEVER use `npx vitest`). Analyze the output and iterate until the test passes.
|
|
608
606
|
3. **⚠️ SHARE THE TEST REPORT URL** - After EVERY test run, find `TESTDRIVER_RUN_URL=https://console.testdriver.ai/runs/...` in the output and share it with the user. This is CRITICAL - users need to view the recording to understand what happened.
|
|
609
607
|
3. **Screenshots are automatic** - TestDriver captures screenshots before/after every command by default. Each screenshot filename includes the line number (e.g., `001-click-before-L42-submit-button.png`) making it easy to trace issues.
|
|
610
608
|
4. **⚠️ USE SCREENSHOT VIEWING FOR DEBUGGING** - When tests fail, use `list_local_screenshots` and `view_local_screenshot` MCP commands to see exactly what the UI looked like. The filenames tell you which line of code triggered each screenshot.
|
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
"$schema": "./examples-manifest.schema.json",
|
|
3
3
|
"examples": {
|
|
4
4
|
"assert.test.mjs": {
|
|
5
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
6
|
-
"lastUpdated": "2026-
|
|
5
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b0f6a1c5774e805b31",
|
|
6
|
+
"lastUpdated": "2026-04-01T00:51:43.773Z"
|
|
7
7
|
},
|
|
8
8
|
"drag-and-drop.test.mjs": {
|
|
9
9
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62b42fc0ac3cc632a918b",
|
|
10
10
|
"lastUpdated": "2026-03-03T00:32:25.275Z"
|
|
11
11
|
},
|
|
12
12
|
"exec-pwsh.test.mjs": {
|
|
13
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
14
|
-
"lastUpdated": "2026-
|
|
13
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a2f6a1c5774e805b25",
|
|
14
|
+
"lastUpdated": "2026-04-01T00:37:55.050Z"
|
|
15
15
|
},
|
|
16
16
|
"match-image.test.mjs": {
|
|
17
17
|
"url": "https://console-test.testdriver.ai/runs/69c8738614b73310c7839412/69c8738c14b73310c783941d",
|
|
@@ -22,84 +22,84 @@
|
|
|
22
22
|
"lastUpdated": "2026-03-03T00:32:25.282Z"
|
|
23
23
|
},
|
|
24
24
|
"hover-text-with-description.test.mjs": {
|
|
25
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
26
|
-
"lastUpdated": "2026-
|
|
25
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a4f6a1c5774e805b26",
|
|
26
|
+
"lastUpdated": "2026-04-01T00:36:52.167Z"
|
|
27
27
|
},
|
|
28
28
|
"windows-installer.test.mjs": {
|
|
29
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
30
|
-
"lastUpdated": "2026-
|
|
29
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a7f6a1c5774e805b29",
|
|
30
|
+
"lastUpdated": "2026-04-01T00:37:59.577Z"
|
|
31
31
|
},
|
|
32
32
|
"exec-output.test.mjs": {
|
|
33
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
34
|
-
"lastUpdated": "2026-
|
|
33
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a8f6a1c5774e805b2a",
|
|
34
|
+
"lastUpdated": "2026-04-01T00:38:00.948Z"
|
|
35
35
|
},
|
|
36
36
|
"chrome-extension.test.mjs": {
|
|
37
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
38
|
-
"lastUpdated": "2026-
|
|
37
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a1f6a1c5774e805b24",
|
|
38
|
+
"lastUpdated": "2026-04-01T00:37:21.547Z"
|
|
39
39
|
},
|
|
40
40
|
"launch-vscode-linux.test.mjs": {
|
|
41
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
42
|
-
"lastUpdated": "2026-
|
|
41
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68a5f6a1c5774e805b27",
|
|
42
|
+
"lastUpdated": "2026-04-01T00:42:55.421Z"
|
|
43
43
|
},
|
|
44
44
|
"hover-image.test.mjs": {
|
|
45
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
46
|
-
"lastUpdated": "2026-
|
|
45
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68aaf6a1c5774e805b2b",
|
|
46
|
+
"lastUpdated": "2026-04-01T00:37:30.348Z"
|
|
47
47
|
},
|
|
48
48
|
"installer.test.mjs": {
|
|
49
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
50
|
-
"lastUpdated": "2026-
|
|
49
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68abf6a1c5774e805b2c",
|
|
50
|
+
"lastUpdated": "2026-04-01T00:36:59.750Z"
|
|
51
51
|
},
|
|
52
52
|
"type.test.mjs": {
|
|
53
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
54
|
-
"lastUpdated": "2026-
|
|
53
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68adf6a1c5774e805b2f",
|
|
54
|
+
"lastUpdated": "2026-04-01T00:37:33.578Z"
|
|
55
55
|
},
|
|
56
56
|
"press-keys.test.mjs": {
|
|
57
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
58
|
-
"lastUpdated": "2026-
|
|
57
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68aff6a1c5774e805b30",
|
|
58
|
+
"lastUpdated": "2026-04-01T00:50:27.244Z"
|
|
59
59
|
},
|
|
60
60
|
"scroll-keyboard.test.mjs": {
|
|
61
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
62
|
-
"lastUpdated": "2026-
|
|
61
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b3f6a1c5774e805b33",
|
|
62
|
+
"lastUpdated": "2026-04-01T00:38:11.893Z"
|
|
63
63
|
},
|
|
64
64
|
"scroll.test.mjs": {
|
|
65
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
66
|
-
"lastUpdated": "2026-
|
|
65
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b9f6a1c5774e805b37",
|
|
66
|
+
"lastUpdated": "2026-04-01T00:56:46.559Z"
|
|
67
67
|
},
|
|
68
68
|
"scroll-until-image.test.mjs": {
|
|
69
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
70
|
-
"lastUpdated": "2026-
|
|
69
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b5f6a1c5774e805b34",
|
|
70
|
+
"lastUpdated": "2026-04-01T00:37:09.297Z"
|
|
71
71
|
},
|
|
72
72
|
"prompt.test.mjs": {
|
|
73
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
74
|
-
"lastUpdated": "2026-
|
|
73
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b6f6a1c5774e805b35",
|
|
74
|
+
"lastUpdated": "2026-04-01T00:38:14.684Z"
|
|
75
75
|
},
|
|
76
76
|
"focus-window.test.mjs": {
|
|
77
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
78
|
-
"lastUpdated": "2026-
|
|
77
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b8f6a1c5774e805b36",
|
|
78
|
+
"lastUpdated": "2026-04-01T00:37:12.144Z"
|
|
79
79
|
},
|
|
80
80
|
"captcha-api.test.mjs": {
|
|
81
81
|
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb0d3b320ad547d9d44",
|
|
82
82
|
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
83
83
|
},
|
|
84
84
|
"element-not-found.test.mjs": {
|
|
85
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
86
|
-
"lastUpdated": "2026-
|
|
85
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68bbf6a1c5774e805b38",
|
|
86
|
+
"lastUpdated": "2026-04-01T00:38:19.040Z"
|
|
87
87
|
},
|
|
88
88
|
"formatted-logging.test.mjs": {
|
|
89
89
|
"url": "https://console-test.testdriver.ai/runs/69c8738614b73310c7839412/69c873a714b73310c7839450",
|
|
90
90
|
"lastUpdated": "2026-03-29T00:36:10.628Z"
|
|
91
91
|
},
|
|
92
92
|
"hover-text.test.mjs": {
|
|
93
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
94
|
-
"lastUpdated": "2026-
|
|
93
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68bef6a1c5774e805b3a",
|
|
94
|
+
"lastUpdated": "2026-04-01T00:37:18.214Z"
|
|
95
95
|
},
|
|
96
96
|
"no-provision.test.mjs": {
|
|
97
97
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62b7706a177a05bccd1cf",
|
|
98
98
|
"lastUpdated": "2026-03-03T00:32:25.279Z"
|
|
99
99
|
},
|
|
100
100
|
"ai.test.mjs": {
|
|
101
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
102
|
-
"lastUpdated": "2026-
|
|
101
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68bcf6a1c5774e805b39",
|
|
102
|
+
"lastUpdated": "2026-04-01T01:00:32.812Z"
|
|
103
103
|
},
|
|
104
104
|
"popup-loading.test.mjs": {
|
|
105
105
|
"url": "https://console.testdriver.ai/runs/698bc89f7140c3fa7daaca8d/698bca7f7140c3fa7daacbf7",
|
|
@@ -134,12 +134,12 @@
|
|
|
134
134
|
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
135
135
|
},
|
|
136
136
|
"findall-coffee-icons.test.mjs": {
|
|
137
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
138
|
-
"lastUpdated": "2026-
|
|
137
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68b2f6a1c5774e805b32",
|
|
138
|
+
"lastUpdated": "2026-04-01T00:37:38.507Z"
|
|
139
139
|
},
|
|
140
140
|
"parse.test.mjs": {
|
|
141
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
142
|
-
"lastUpdated": "2026-
|
|
141
|
+
"url": "https://console-test.testdriver.ai/runs/69cc689ff6a1c5774e805b23/69cc68bff6a1c5774e805b3b",
|
|
142
|
+
"lastUpdated": "2026-04-01T00:38:23.690Z"
|
|
143
143
|
},
|
|
144
144
|
"flake-diffthreshold-001.test.mjs": {
|
|
145
145
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62bcafc0ac3cc632a91aa",
|
package/docs/docs.json
CHANGED
package/docs/v7/examples/ai.mdx
CHANGED
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* ai.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68bcf6a1c5774e805b39/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* assert.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b0f6a1c5774e805b31/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* chrome-extension.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a1f6a1c5774e805b24/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* element-not-found.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68bbf6a1c5774e805b38/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* exec-output.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a8f6a1c5774e805b2a/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* exec-pwsh.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a2f6a1c5774e805b25/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -18,7 +18,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
18
18
|
|
|
19
19
|
{/* findall-coffee-icons.test.mjs output */}
|
|
20
20
|
<iframe
|
|
21
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
21
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b2f6a1c5774e805b32/replay"
|
|
22
22
|
width="100%"
|
|
23
23
|
height="600"
|
|
24
24
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* focus-window.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b8f6a1c5774e805b36/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68aaf6a1c5774e805b2b/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -18,7 +18,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
18
18
|
|
|
19
19
|
{/* hover-text-with-description.test.mjs output */}
|
|
20
20
|
<iframe
|
|
21
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
21
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a4f6a1c5774e805b26/replay"
|
|
22
22
|
width="100%"
|
|
23
23
|
height="600"
|
|
24
24
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-text.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68bef6a1c5774e805b3a/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* installer.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68abf6a1c5774e805b2c/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* launch-vscode-linux.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a5f6a1c5774e805b27/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -17,7 +17,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
17
17
|
|
|
18
18
|
{/* parse.test.mjs output */}
|
|
19
19
|
<iframe
|
|
20
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
20
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68bff6a1c5774e805b3b/replay"
|
|
21
21
|
width="100%"
|
|
22
22
|
height="600"
|
|
23
23
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* press-keys.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68aff6a1c5774e805b30/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -17,7 +17,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
17
17
|
|
|
18
18
|
{/* prompt.test.mjs output */}
|
|
19
19
|
<iframe
|
|
20
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
20
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b6f6a1c5774e805b35/replay"
|
|
21
21
|
width="100%"
|
|
22
22
|
height="600"
|
|
23
23
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-keyboard.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b3f6a1c5774e805b33/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-until-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b5f6a1c5774e805b34/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68b9f6a1c5774e805b37/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* type.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68adf6a1c5774e805b2f/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* windows-installer.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/69cc68a7f6a1c5774e805b29/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -10,6 +10,7 @@ import { setTestRunInfo } from "./shared-test-state.mjs";
|
|
|
10
10
|
// Use createRequire to import CommonJS modules without esbuild processing
|
|
11
11
|
const require = createRequire(import.meta.url);
|
|
12
12
|
const channelConfig = require("../lib/resolve-channel.js");
|
|
13
|
+
const environments = require("../lib/environments.json");
|
|
13
14
|
|
|
14
15
|
// Import Sentry for error reporting
|
|
15
16
|
const Sentry = require("@sentry/node");
|
|
@@ -1376,11 +1377,11 @@ function getConsoleUrl(apiRoot) {
|
|
|
1376
1377
|
return `https://${flyMatch[1]}-web.fly.dev`;
|
|
1377
1378
|
}
|
|
1378
1379
|
|
|
1379
|
-
// Known channel API URLs -> console equivalents
|
|
1380
|
-
// e.g. https://api-
|
|
1381
|
-
for (const
|
|
1382
|
-
if (
|
|
1383
|
-
return
|
|
1380
|
+
// Known channel API URLs -> console equivalents (from environments.json)
|
|
1381
|
+
// e.g. https://api-dev.testdriver.ai -> https://console-dev.testdriver.ai
|
|
1382
|
+
for (const env of Object.values(environments)) {
|
|
1383
|
+
if (env.apiRoot === apiRoot && env.consoleUrl) {
|
|
1384
|
+
return env.consoleUrl;
|
|
1384
1385
|
}
|
|
1385
1386
|
}
|
|
1386
1387
|
|
package/lib/core/Dashcam.js
CHANGED
|
@@ -98,7 +98,7 @@ class Dashcam {
|
|
|
98
98
|
if (!apiRoot) return "https://console.testdriver.ai";
|
|
99
99
|
|
|
100
100
|
// Dash-separated environments: api-{env}.testdriver.ai -> console-{env}.testdriver.ai
|
|
101
|
-
const envMatch = apiRoot.match(/^https:\/\/api-(test|canary)\.testdriver\.ai/);
|
|
101
|
+
const envMatch = apiRoot.match(/^https:\/\/api-(dev|test|canary)\.testdriver\.ai/);
|
|
102
102
|
if (envMatch) {
|
|
103
103
|
return `https://console-${envMatch[1]}.testdriver.ai`;
|
|
104
104
|
}
|
package/lib/github-comment.mjs
CHANGED
|
@@ -8,7 +8,13 @@
|
|
|
8
8
|
* - Links to test runs
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Lazily load Octokit to avoid hard failure when @octokit/rest is not installed
|
|
13
|
+
*/
|
|
14
|
+
async function getOctokit(token) {
|
|
15
|
+
const { Octokit } = await import('@octokit/rest');
|
|
16
|
+
return new Octokit({ auth: token });
|
|
17
|
+
}
|
|
12
18
|
|
|
13
19
|
/**
|
|
14
20
|
* Format test duration in human-readable format
|
|
@@ -334,7 +340,7 @@ export async function postGitHubComment(options) {
|
|
|
334
340
|
throw new Error('Either prNumber or commitSha must be provided');
|
|
335
341
|
}
|
|
336
342
|
|
|
337
|
-
const octokit =
|
|
343
|
+
const octokit = await getOctokit(token);
|
|
338
344
|
|
|
339
345
|
if (prNumber) {
|
|
340
346
|
// Comment on PR
|
|
@@ -374,7 +380,7 @@ export async function updateGitHubComment(options) {
|
|
|
374
380
|
throw new Error('Token, owner, repo, and commentId are required');
|
|
375
381
|
}
|
|
376
382
|
|
|
377
|
-
const octokit =
|
|
383
|
+
const octokit = await getOctokit(token);
|
|
378
384
|
|
|
379
385
|
const response = await octokit.rest.issues.updateComment({
|
|
380
386
|
owner,
|
|
@@ -402,7 +408,7 @@ export async function findExistingComment(options) {
|
|
|
402
408
|
return null;
|
|
403
409
|
}
|
|
404
410
|
|
|
405
|
-
const octokit =
|
|
411
|
+
const octokit = await getOctokit(token);
|
|
406
412
|
|
|
407
413
|
const comments = await octokit.rest.issues.listComments({
|
|
408
414
|
owner,
|
|
@@ -435,7 +441,7 @@ export async function postOrUpdateTestResults(testRunData, testCases, githubOpti
|
|
|
435
441
|
|
|
436
442
|
if (existingComment) {
|
|
437
443
|
// Delete the old comment
|
|
438
|
-
const octokit =
|
|
444
|
+
const octokit = await getOctokit(githubOptions.token);
|
|
439
445
|
await octokit.rest.issues.deleteComment({
|
|
440
446
|
owner: githubOptions.owner,
|
|
441
447
|
repo: githubOptions.repo,
|
package/package.json
CHANGED
package/sdk.d.ts
CHANGED
|
@@ -1086,7 +1086,7 @@ export default class TestDriverSDK {
|
|
|
1086
1086
|
find(description: string, cacheThreshold?: number): ChainableElementPromise;
|
|
1087
1087
|
find(
|
|
1088
1088
|
description: string,
|
|
1089
|
-
options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; confidence?: number; type?: "text" | "image" | "ui" | "any"; ai?: AIConfig; cache?: { thresholds?: { screen?: number; element?: number } } },
|
|
1089
|
+
options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; confidence?: number; type?: "text" | "image" | "ui" | "any"; zoom?: boolean | number; verify?: boolean; ai?: AIConfig; cache?: { thresholds?: { screen?: number; element?: number } } },
|
|
1090
1090
|
): ChainableElementPromise;
|
|
1091
1091
|
|
|
1092
1092
|
/**
|
package/sdk.js
CHANGED
|
@@ -481,7 +481,8 @@ class Element {
|
|
|
481
481
|
let cacheKey = null;
|
|
482
482
|
let cacheThreshold = null;
|
|
483
483
|
let perCommandThresholds = null; // Per-command { screen, element } override
|
|
484
|
-
let zoom =
|
|
484
|
+
let zoom = false; // Default to disabled
|
|
485
|
+
let verify = false; // Default to disabled (skip AI verification)
|
|
485
486
|
let perCommandAi = null; // Per-command AI config override
|
|
486
487
|
|
|
487
488
|
let minConfidence = null; // Minimum confidence threshold
|
|
@@ -494,8 +495,10 @@ class Element {
|
|
|
494
495
|
// New: options is an object with cacheKey and/or cacheThreshold
|
|
495
496
|
cacheKey = options.cacheKey || null;
|
|
496
497
|
cacheThreshold = options.cacheThreshold ?? null;
|
|
497
|
-
// zoom defaults to
|
|
498
|
-
zoom = options.zoom
|
|
498
|
+
// zoom defaults to false unless explicitly set to true
|
|
499
|
+
zoom = options.zoom === true;
|
|
500
|
+
// verify defaults to false unless explicitly set to true
|
|
501
|
+
verify = options.verify === true;
|
|
499
502
|
// Minimum confidence threshold: fail find if AI confidence is below this value
|
|
500
503
|
minConfidence = options.confidence ?? null;
|
|
501
504
|
// Element type hint for prompt wrapping
|
|
@@ -569,6 +572,7 @@ class Element {
|
|
|
569
572
|
os: this.sdk.os,
|
|
570
573
|
resolution: this.sdk.resolution,
|
|
571
574
|
zoom: zoom === true ? 1 : zoom === false ? 0 : zoom,
|
|
575
|
+
skipVerify: !verify,
|
|
572
576
|
confidence: minConfidence,
|
|
573
577
|
type: elementType,
|
|
574
578
|
ai: {
|