testdriverai 7.3.9 → 7.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/agent/lib/commands.js +3 -2
- package/docs/_data/examples-manifest.json +64 -64
- package/docs/v7/assert.mdx +31 -0
- package/docs/v7/client.mdx +29 -0
- package/docs/v7/examples/ai.mdx +1 -1
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/captcha-api.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +1 -1
- package/docs/v7/examples/drag-and-drop.mdx +1 -1
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +1 -1
- package/docs/v7/examples/match-image.mdx +1 -1
- package/docs/v7/examples/press-keys.mdx +1 -1
- package/docs/v7/examples/scroll-keyboard.mdx +1 -1
- package/docs/v7/examples/scroll-until-image.mdx +1 -1
- package/docs/v7/examples/scroll-until-text.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/docs/v7/examples/windows-installer.mdx +1 -1
- package/docs/v7/find.mdx +24 -0
- package/examples/z_flake-shared.mjs +1 -1
- package/lib/vitest/hooks.mjs +3 -0
- package/mcp-server/dist/server.mjs +2 -2
- package/mcp-server/src/server.ts +2 -2
- package/package.json +1 -1
- package/sdk.d.ts +19 -2
- package/sdk.js +130 -103
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
## [7.3.11](https://github.com/testdriverai/testdriverai/compare/v7.3.10...v7.3.11) (2026-02-17)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
## [7.3.10](https://github.com/testdriverai/testdriverai/compare/v7.3.9...v7.3.10) (2026-02-16)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
1
9
|
## [7.3.9](https://github.com/testdriverai/testdriverai/compare/v7.3.8...v7.3.9) (2026-02-12)
|
|
2
10
|
|
|
3
11
|
|
package/agent/lib/commands.js
CHANGED
|
@@ -226,8 +226,8 @@ const createCommands = (
|
|
|
226
226
|
const assertTimestamp = Date.now();
|
|
227
227
|
const assertStartTime = assertTimestamp;
|
|
228
228
|
|
|
229
|
-
// Extract cache options
|
|
230
|
-
const { threshold = -1, cacheKey, os, resolution } = options;
|
|
229
|
+
// Extract cache and AI options
|
|
230
|
+
const { threshold = -1, cacheKey, os, resolution, ai } = options;
|
|
231
231
|
|
|
232
232
|
// Debug log cache settings
|
|
233
233
|
emitter.emit(
|
|
@@ -243,6 +243,7 @@ const createCommands = (
|
|
|
243
243
|
cacheKey,
|
|
244
244
|
os,
|
|
245
245
|
resolution,
|
|
246
|
+
ai,
|
|
246
247
|
});
|
|
247
248
|
|
|
248
249
|
const assertDuration = Date.now() - assertStartTime;
|
|
@@ -2,136 +2,136 @@
|
|
|
2
2
|
"$schema": "./examples-manifest.schema.json",
|
|
3
3
|
"examples": {
|
|
4
4
|
"assert.test.mjs": {
|
|
5
|
-
"url": "https://console.testdriver.ai/runs/
|
|
6
|
-
"lastUpdated": "2026-02-
|
|
5
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f88dd202a57999808e3",
|
|
6
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
7
7
|
},
|
|
8
8
|
"drag-and-drop.test.mjs": {
|
|
9
|
-
"url": "https://console.testdriver.ai/runs/
|
|
10
|
-
"lastUpdated": "2026-02-
|
|
9
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7df8d3b320ad547d9cdc",
|
|
10
|
+
"lastUpdated": "2026-02-13T19:55:05.944Z"
|
|
11
11
|
},
|
|
12
12
|
"exec-pwsh.test.mjs": {
|
|
13
|
-
"url": "https://console.testdriver.ai/runs/
|
|
14
|
-
"lastUpdated": "2026-02-
|
|
13
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7df9d3b320ad547d9cdf",
|
|
14
|
+
"lastUpdated": "2026-02-13T19:55:05.944Z"
|
|
15
15
|
},
|
|
16
16
|
"match-image.test.mjs": {
|
|
17
|
-
"url": "https://console.testdriver.ai/runs/
|
|
18
|
-
"lastUpdated": "2026-02-
|
|
17
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7dfad3b320ad547d9ce0",
|
|
18
|
+
"lastUpdated": "2026-02-13T19:55:05.944Z"
|
|
19
19
|
},
|
|
20
20
|
"scroll-until-text.test.mjs": {
|
|
21
|
-
"url": "https://console.testdriver.ai/runs/
|
|
22
|
-
"lastUpdated": "2026-02-
|
|
21
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5cd3b320ad547d9d03",
|
|
22
|
+
"lastUpdated": "2026-02-13T19:55:05.945Z"
|
|
23
23
|
},
|
|
24
24
|
"hover-text-with-description.test.mjs": {
|
|
25
|
-
"url": "https://console.testdriver.ai/runs/
|
|
26
|
-
"lastUpdated": "2026-02-
|
|
25
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e4e9e27ce1528d7d0a0",
|
|
26
|
+
"lastUpdated": "2026-02-13T19:55:05.944Z"
|
|
27
27
|
},
|
|
28
28
|
"windows-installer.test.mjs": {
|
|
29
|
-
"url": "https://console.testdriver.ai/runs/
|
|
30
|
-
"lastUpdated": "2026-02-
|
|
29
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5ddd202a57999808ae",
|
|
30
|
+
"lastUpdated": "2026-02-13T19:55:05.945Z"
|
|
31
31
|
},
|
|
32
32
|
"exec-output.test.mjs": {
|
|
33
|
-
"url": "https://console.testdriver.ai/runs/
|
|
34
|
-
"lastUpdated": "2026-02-
|
|
33
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5edd202a57999808af",
|
|
34
|
+
"lastUpdated": "2026-02-13T19:55:05.945Z"
|
|
35
35
|
},
|
|
36
36
|
"chrome-extension.test.mjs": {
|
|
37
|
-
"url": "https://console.testdriver.ai/runs/
|
|
38
|
-
"lastUpdated": "2026-02-
|
|
37
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7eb69e27ce1528d7d0bc",
|
|
38
|
+
"lastUpdated": "2026-02-13T19:55:05.946Z"
|
|
39
39
|
},
|
|
40
40
|
"launch-vscode-linux.test.mjs": {
|
|
41
|
-
"url": "https://console.testdriver.ai/runs/
|
|
42
|
-
"lastUpdated": "2026-02-
|
|
41
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7eb6d3b320ad547d9d16",
|
|
42
|
+
"lastUpdated": "2026-02-13T19:55:05.946Z"
|
|
43
43
|
},
|
|
44
44
|
"hover-image.test.mjs": {
|
|
45
|
-
"url": "https://console.testdriver.ai/runs/
|
|
46
|
-
"lastUpdated": "2026-02-
|
|
45
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7ea7dd202a57999808bc",
|
|
46
|
+
"lastUpdated": "2026-02-13T19:55:05.946Z"
|
|
47
47
|
},
|
|
48
48
|
"installer.test.mjs": {
|
|
49
|
-
"url": "https://console.testdriver.ai/runs/
|
|
50
|
-
"lastUpdated": "2026-02-
|
|
49
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f069e27ce1528d7d0cd",
|
|
50
|
+
"lastUpdated": "2026-02-13T19:55:05.948Z"
|
|
51
51
|
},
|
|
52
52
|
"type.test.mjs": {
|
|
53
|
-
"url": "https://console.testdriver.ai/runs/
|
|
54
|
-
"lastUpdated": "2026-02-
|
|
53
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f2a9e27ce1528d7d0d2",
|
|
54
|
+
"lastUpdated": "2026-02-13T19:55:05.948Z"
|
|
55
55
|
},
|
|
56
56
|
"press-keys.test.mjs": {
|
|
57
|
-
"url": "https://console.testdriver.ai/runs/
|
|
58
|
-
"lastUpdated": "2026-02-
|
|
57
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f739e27ce1528d7d0d9",
|
|
58
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
59
59
|
},
|
|
60
60
|
"scroll-keyboard.test.mjs": {
|
|
61
|
-
"url": "https://console.testdriver.ai/runs/
|
|
62
|
-
"lastUpdated": "2026-02-
|
|
61
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f55dd202a57999808d6",
|
|
62
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
63
63
|
},
|
|
64
64
|
"scroll.test.mjs": {
|
|
65
|
-
"url": "https://console.testdriver.ai/runs/
|
|
66
|
-
"lastUpdated": "2026-02-
|
|
65
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f0add202a57999808c8",
|
|
66
|
+
"lastUpdated": "2026-02-13T19:55:05.948Z"
|
|
67
67
|
},
|
|
68
68
|
"scroll-until-image.test.mjs": {
|
|
69
|
-
"url": "https://console.testdriver.ai/runs/
|
|
70
|
-
"lastUpdated": "2026-02-
|
|
69
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f749e27ce1528d7d0da",
|
|
70
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
71
71
|
},
|
|
72
72
|
"prompt.test.mjs": {
|
|
73
|
-
"url": "https://console.testdriver.ai/runs/
|
|
74
|
-
"lastUpdated": "2026-02-
|
|
73
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f75d3b320ad547d9d39",
|
|
74
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
75
75
|
},
|
|
76
76
|
"focus-window.test.mjs": {
|
|
77
|
-
"url": "https://console.testdriver.ai/runs/
|
|
78
|
-
"lastUpdated": "2026-02-
|
|
77
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f76d3b320ad547d9d3a",
|
|
78
|
+
"lastUpdated": "2026-02-13T19:55:05.949Z"
|
|
79
79
|
},
|
|
80
80
|
"captcha-api.test.mjs": {
|
|
81
|
-
"url": "https://console.testdriver.ai/runs/
|
|
82
|
-
"lastUpdated": "2026-02-
|
|
81
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb0d3b320ad547d9d44",
|
|
82
|
+
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
83
83
|
},
|
|
84
84
|
"element-not-found.test.mjs": {
|
|
85
|
-
"url": "https://console.testdriver.ai/runs/
|
|
86
|
-
"lastUpdated": "2026-02-
|
|
85
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fbcdd202a57999808f0",
|
|
86
|
+
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
87
87
|
},
|
|
88
88
|
"formatted-logging.test.mjs": {
|
|
89
|
-
"url": "https://console.testdriver.ai/runs/
|
|
90
|
-
"lastUpdated": "2026-02-
|
|
89
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb1dd202a57999808e9",
|
|
90
|
+
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
91
91
|
},
|
|
92
92
|
"hover-text.test.mjs": {
|
|
93
|
-
"url": "https://console.testdriver.ai/runs/
|
|
94
|
-
"lastUpdated": "2026-02-
|
|
93
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fe69e27ce1528d7d0ed",
|
|
94
|
+
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
95
95
|
},
|
|
96
96
|
"no-provision.test.mjs": {
|
|
97
|
-
"url": "https://console.testdriver.ai/runs/
|
|
98
|
-
"lastUpdated": "2026-02-
|
|
97
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fdf9e27ce1528d7d0ec",
|
|
98
|
+
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
99
99
|
},
|
|
100
100
|
"ai.test.mjs": {
|
|
101
|
-
"url": "https://console.testdriver.ai/runs/
|
|
102
|
-
"lastUpdated": "2026-02-
|
|
101
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8010d3b320ad547d9d52",
|
|
102
|
+
"lastUpdated": "2026-02-13T19:55:05.952Z"
|
|
103
103
|
},
|
|
104
104
|
"popup-loading.test.mjs": {
|
|
105
105
|
"url": "https://console.testdriver.ai/runs/698bc89f7140c3fa7daaca8d/698bca7f7140c3fa7daacbf7",
|
|
106
106
|
"lastUpdated": "2026-02-11T00:20:33.687Z"
|
|
107
107
|
},
|
|
108
108
|
"z_flake-diffthreshold-001.test.mjs": {
|
|
109
|
-
"url": "https://console.testdriver.ai/runs/
|
|
110
|
-
"lastUpdated": "2026-02-
|
|
109
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8090d3b320ad547d9d67",
|
|
110
|
+
"lastUpdated": "2026-02-13T19:55:05.952Z"
|
|
111
111
|
},
|
|
112
112
|
"z_flake-diffthreshold-01.test.mjs": {
|
|
113
|
-
"url": "https://console.testdriver.ai/runs/
|
|
114
|
-
"lastUpdated": "2026-02-
|
|
113
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f80bc9e27ce1528d7d106",
|
|
114
|
+
"lastUpdated": "2026-02-13T19:55:05.952Z"
|
|
115
115
|
},
|
|
116
116
|
"z_flake-diffthreshold-05.test.mjs": {
|
|
117
|
-
"url": "https://console.testdriver.ai/runs/
|
|
118
|
-
"lastUpdated": "2026-02-
|
|
117
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8090dd202a5799980910",
|
|
118
|
+
"lastUpdated": "2026-02-13T19:55:05.952Z"
|
|
119
119
|
},
|
|
120
120
|
"z_flake-noredraw-cache.test.mjs": {
|
|
121
|
-
"url": "https://console.testdriver.ai/runs/
|
|
122
|
-
"lastUpdated": "2026-02-
|
|
121
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8105d3b320ad547d9d75",
|
|
122
|
+
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
123
123
|
},
|
|
124
124
|
"z_flake-redraw-nocache.test.mjs": {
|
|
125
|
-
"url": "https://console.testdriver.ai/runs/
|
|
126
|
-
"lastUpdated": "2026-02-
|
|
125
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8142d3b320ad547d9d86",
|
|
126
|
+
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
127
127
|
},
|
|
128
128
|
"z_flake-redraw-cache.test.mjs": {
|
|
129
|
-
"url": "https://console.testdriver.ai/runs/
|
|
130
|
-
"lastUpdated": "2026-02-
|
|
129
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f818f9e27ce1528d7d11d",
|
|
130
|
+
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
131
131
|
},
|
|
132
132
|
"z_flake-noredraw-nocache.test.mjs": {
|
|
133
|
-
"url": "https://console.testdriver.ai/runs/
|
|
134
|
-
"lastUpdated": "2026-02-
|
|
133
|
+
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f816ed3b320ad547d9d8c",
|
|
134
|
+
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
}
|
package/docs/v7/assert.mdx
CHANGED
|
@@ -13,6 +13,7 @@ Make AI-powered assertions about the current screen state using natural language
|
|
|
13
13
|
|
|
14
14
|
```javascript
|
|
15
15
|
await testdriver.assert(assertion)
|
|
16
|
+
await testdriver.assert(assertion, options)
|
|
16
17
|
```
|
|
17
18
|
|
|
18
19
|
## Parameters
|
|
@@ -21,6 +22,36 @@ await testdriver.assert(assertion)
|
|
|
21
22
|
Natural language description of what should be true
|
|
22
23
|
</ParamField>
|
|
23
24
|
|
|
25
|
+
<ParamField path="options" type="object">
|
|
26
|
+
Optional configuration
|
|
27
|
+
|
|
28
|
+
<Expandable title="properties">
|
|
29
|
+
<ParamField path="ai" type="object">
|
|
30
|
+
AI sampling configuration for this assert call (overrides global `ai` config from constructor).
|
|
31
|
+
|
|
32
|
+
<Expandable title="properties">
|
|
33
|
+
<ParamField path="temperature" type="number">
|
|
34
|
+
Controls randomness. `0` = deterministic, higher = more creative. Default: model default.
|
|
35
|
+
</ParamField>
|
|
36
|
+
|
|
37
|
+
<ParamField path="top" type="object">
|
|
38
|
+
Sampling parameters
|
|
39
|
+
|
|
40
|
+
<Expandable title="properties">
|
|
41
|
+
<ParamField path="p" type="number">
|
|
42
|
+
Top-P (nucleus sampling). Range: 0-1.
|
|
43
|
+
</ParamField>
|
|
44
|
+
|
|
45
|
+
<ParamField path="k" type="number">
|
|
46
|
+
Top-K sampling. `1` = most deterministic.
|
|
47
|
+
</ParamField>
|
|
48
|
+
</Expandable>
|
|
49
|
+
</ParamField>
|
|
50
|
+
</Expandable>
|
|
51
|
+
</ParamField>
|
|
52
|
+
</Expandable>
|
|
53
|
+
</ParamField>
|
|
54
|
+
|
|
24
55
|
## Returns
|
|
25
56
|
|
|
26
57
|
`Promise<boolean>` - `true` if assertion passes, throws error if assertion fails
|
package/docs/v7/client.mdx
CHANGED
|
@@ -52,6 +52,30 @@ const testdriver = new TestDriver(apiKey, options)
|
|
|
52
52
|
<ParamField path="environment" type="object">
|
|
53
53
|
Additional environment variables to pass to the sandbox
|
|
54
54
|
</ParamField>
|
|
55
|
+
|
|
56
|
+
<ParamField path="ai" type="object">
|
|
57
|
+
Global AI sampling configuration. Controls how the AI model generates responses for `find()` verification and `assert()` calls. Can be overridden per call.
|
|
58
|
+
|
|
59
|
+
<Expandable title="properties">
|
|
60
|
+
<ParamField path="temperature" type="number">
|
|
61
|
+
Controls randomness in AI responses. `0` = deterministic (best for verification), higher values = more creative. Default: `0` for find verification, model default for assert.
|
|
62
|
+
</ParamField>
|
|
63
|
+
|
|
64
|
+
<ParamField path="top" type="object">
|
|
65
|
+
Nucleus and top-k sampling parameters
|
|
66
|
+
|
|
67
|
+
<Expandable title="properties">
|
|
68
|
+
<ParamField path="p" type="number">
|
|
69
|
+
Top-P (nucleus sampling). Limits token choices to the smallest set whose cumulative probability exceeds P. Lower values = more focused responses. Range: 0-1.
|
|
70
|
+
</ParamField>
|
|
71
|
+
|
|
72
|
+
<ParamField path="k" type="number">
|
|
73
|
+
Top-K sampling. Limits token choices to the top K most likely tokens. `1` = always pick the most likely token. `0` = disabled (consider all tokens).
|
|
74
|
+
</ParamField>
|
|
75
|
+
</Expandable>
|
|
76
|
+
</ParamField>
|
|
77
|
+
</Expandable>
|
|
78
|
+
</ParamField>
|
|
55
79
|
</Expandable>
|
|
56
80
|
</ParamField>
|
|
57
81
|
|
|
@@ -68,6 +92,11 @@ const testdriver = new TestDriver({
|
|
|
68
92
|
analytics: true
|
|
69
93
|
});
|
|
70
94
|
|
|
95
|
+
// With AI config for stricter verification
|
|
96
|
+
const testdriver = new TestDriver({
|
|
97
|
+
ai: { temperature: 0, top: { p: 0.9, k: 40 } }
|
|
98
|
+
});
|
|
99
|
+
|
|
71
100
|
// Or pass API key explicitly
|
|
72
101
|
const testdriver = new TestDriver('your-api-key', {
|
|
73
102
|
os: 'windows'
|
package/docs/v7/examples/ai.mdx
CHANGED
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* ai.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f8010d3b320ad547d9d52/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* assert.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f88dd202a57999808e3/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* captcha-api.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fb0d3b320ad547d9d44/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* chrome-extension.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7eb69e27ce1528d7d0bc/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* drag-and-drop.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7df8d3b320ad547d9cdc/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* element-not-found.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fbcdd202a57999808f0/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7ea7dd202a57999808bc/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-text.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fe69e27ce1528d7d0ed/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* installer.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f069e27ce1528d7d0cd/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* launch-vscode-linux.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7eb6d3b320ad547d9d16/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* match-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7dfad3b320ad547d9ce0/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* press-keys.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f739e27ce1528d7d0d9/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-keyboard.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f55dd202a57999808d6/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-until-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f749e27ce1528d7d0da/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-until-text.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7e5cd3b320ad547d9d03/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f0add202a57999808c8/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* type.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f2a9e27ce1528d7d0d2/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* windows-installer.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7e5ddd202a57999808ae/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
package/docs/v7/find.mdx
CHANGED
|
@@ -41,6 +41,30 @@ const element = await testdriver.find(description, options)
|
|
|
41
41
|
<ParamField path="zoom" type="boolean" default={false}>
|
|
42
42
|
Enable two-phase zoom mode for better precision in crowded UIs with many similar elements.
|
|
43
43
|
</ParamField>
|
|
44
|
+
|
|
45
|
+
<ParamField path="ai" type="object">
|
|
46
|
+
AI sampling configuration for this find call (overrides global `ai` config from constructor).
|
|
47
|
+
|
|
48
|
+
<Expandable title="properties">
|
|
49
|
+
<ParamField path="temperature" type="number">
|
|
50
|
+
Controls randomness. `0` = deterministic. Default: `0` for find verification.
|
|
51
|
+
</ParamField>
|
|
52
|
+
|
|
53
|
+
<ParamField path="top" type="object">
|
|
54
|
+
Sampling parameters
|
|
55
|
+
|
|
56
|
+
<Expandable title="properties">
|
|
57
|
+
<ParamField path="p" type="number">
|
|
58
|
+
Top-P (nucleus sampling). Range: 0-1.
|
|
59
|
+
</ParamField>
|
|
60
|
+
|
|
61
|
+
<ParamField path="k" type="number">
|
|
62
|
+
Top-K sampling. `1` = most deterministic.
|
|
63
|
+
</ParamField>
|
|
64
|
+
</Expandable>
|
|
65
|
+
</ParamField>
|
|
66
|
+
</Expandable>
|
|
67
|
+
</ParamField>
|
|
44
68
|
</Expandable>
|
|
45
69
|
</ParamField>
|
|
46
70
|
|
|
@@ -22,7 +22,7 @@ export function popupLoadingTest(label, options = {}) {
|
|
|
22
22
|
// Accept the cookie banner to trigger the loading process
|
|
23
23
|
let acceptButton = await testdriver.find("Accept All button on the cookie banner", {timeout: 60000});
|
|
24
24
|
|
|
25
|
-
if (acceptButton.found()) {
|
|
25
|
+
if (await acceptButton.found()) {
|
|
26
26
|
await acceptButton.click();
|
|
27
27
|
} else {
|
|
28
28
|
console.log('no cookie banner found, proceeding without accepting cookies');
|
package/lib/vitest/hooks.mjs
CHANGED
|
@@ -324,6 +324,9 @@ export function TestDriver(context, options = {}) {
|
|
|
324
324
|
// Add testdriver log to dashcam tracking
|
|
325
325
|
await testdriver.dashcam.addFileLog(logPath, "TestDriver Log");
|
|
326
326
|
|
|
327
|
+
// Add web log tracking before starting dashcam
|
|
328
|
+
await testdriver.dashcam.addWebLog("**", "Web Logs");
|
|
329
|
+
|
|
327
330
|
// Start dashcam recording
|
|
328
331
|
await testdriver.dashcam.start();
|
|
329
332
|
}
|
|
@@ -401,8 +401,8 @@ Debug mode (connect to existing sandbox):
|
|
|
401
401
|
const TestDriverSDK = (await import("../../sdk.js")).default;
|
|
402
402
|
// Determine preview mode from environment variable
|
|
403
403
|
// TD_PREVIEW can be "ide", "browser", or "none"
|
|
404
|
-
// Default to "
|
|
405
|
-
const previewMode = process.env.TD_PREVIEW || "
|
|
404
|
+
// Default to "ide" so the live preview shows within the IDE
|
|
405
|
+
const previewMode = process.env.TD_PREVIEW || "ide";
|
|
406
406
|
logger.debug("session_start: Preview mode", { preview: previewMode });
|
|
407
407
|
// Get IP from params or environment (for self-hosted instances)
|
|
408
408
|
const instanceIp = params.ip || process.env.TD_IP;
|
package/mcp-server/src/server.ts
CHANGED
|
@@ -509,8 +509,8 @@ Debug mode (connect to existing sandbox):
|
|
|
509
509
|
|
|
510
510
|
// Determine preview mode from environment variable
|
|
511
511
|
// TD_PREVIEW can be "ide", "browser", or "none"
|
|
512
|
-
// Default to "
|
|
513
|
-
const previewMode = process.env.TD_PREVIEW || "
|
|
512
|
+
// Default to "ide" so the live preview shows within the IDE
|
|
513
|
+
const previewMode = process.env.TD_PREVIEW || "ide";
|
|
514
514
|
logger.debug("session_start: Preview mode", { preview: previewMode });
|
|
515
515
|
|
|
516
516
|
// Get IP from params or environment (for self-hosted instances)
|
package/package.json
CHANGED
package/sdk.d.ts
CHANGED
|
@@ -230,6 +230,8 @@ export interface TestDriverOptions {
|
|
|
230
230
|
logging?: boolean;
|
|
231
231
|
/** Enable/disable cache (default: true). Set to false to force regeneration on all find operations */
|
|
232
232
|
cache?: boolean;
|
|
233
|
+
/** Global AI sampling configuration. Can be overridden per find() or assert() call. */
|
|
234
|
+
ai?: AIConfig;
|
|
233
235
|
/** Cache threshold configuration for different methods */
|
|
234
236
|
cacheThreshold?: {
|
|
235
237
|
/** Threshold for find operations (default: 0.05 = 5% difference, 95% similarity) */
|
|
@@ -546,6 +548,19 @@ export interface FocusApplicationOptions {
|
|
|
546
548
|
name: string;
|
|
547
549
|
}
|
|
548
550
|
|
|
551
|
+
/** AI sampling configuration for controlling model behavior */
|
|
552
|
+
export interface AIConfig {
|
|
553
|
+
/** Temperature for AI sampling (0 = deterministic, higher = more creative). Default: 0 for find verification, model default for assert. */
|
|
554
|
+
temperature?: number;
|
|
555
|
+
/** Top-P and Top-K sampling parameters */
|
|
556
|
+
top?: {
|
|
557
|
+
/** Top-P (nucleus sampling). Controls diversity by limiting to top P probability mass. Range: 0-1. */
|
|
558
|
+
p?: number;
|
|
559
|
+
/** Top-K sampling. Limits choices to top K tokens. 1 = always pick most likely. 0 = disabled. */
|
|
560
|
+
k?: number;
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
|
|
549
564
|
/** Options for extract command */
|
|
550
565
|
export interface ExtractOptions {
|
|
551
566
|
/** What to extract */
|
|
@@ -564,6 +579,8 @@ export interface AssertOptions {
|
|
|
564
579
|
os?: string;
|
|
565
580
|
/** Screen resolution for cache partitioning */
|
|
566
581
|
resolution?: string;
|
|
582
|
+
/** AI sampling configuration (overrides global ai config) */
|
|
583
|
+
ai?: AIConfig;
|
|
567
584
|
}
|
|
568
585
|
|
|
569
586
|
/** Options for exec command */
|
|
@@ -1028,7 +1045,7 @@ export default class TestDriverSDK {
|
|
|
1028
1045
|
find(description: string, cacheThreshold?: number): ChainableElementPromise;
|
|
1029
1046
|
find(
|
|
1030
1047
|
description: string,
|
|
1031
|
-
options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number },
|
|
1048
|
+
options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; ai?: AIConfig },
|
|
1032
1049
|
): ChainableElementPromise;
|
|
1033
1050
|
|
|
1034
1051
|
/**
|
|
@@ -1267,7 +1284,7 @@ export default class TestDriverSDK {
|
|
|
1267
1284
|
* // With custom threshold
|
|
1268
1285
|
* await client.assert('the page loaded', { threshold: 0.01, cacheKey: 'login-test' });
|
|
1269
1286
|
*/
|
|
1270
|
-
assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string }): Promise<boolean>;
|
|
1287
|
+
assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string; ai?: AIConfig }): Promise<boolean>;
|
|
1271
1288
|
|
|
1272
1289
|
/**
|
|
1273
1290
|
* Extract information from the screen using AI
|
package/sdk.js
CHANGED
|
@@ -430,8 +430,9 @@ class Element {
|
|
|
430
430
|
/**
|
|
431
431
|
* Find the element on screen
|
|
432
432
|
* @param {string} [newDescription] - Optional new description to search for
|
|
433
|
-
* @param {Object} [options] - Optional options object with
|
|
433
|
+
* @param {Object} [options] - Optional options object with cache thresholds, cacheKey, and/or timeout
|
|
434
434
|
* @param {number} [options.timeout] - Max time in ms to poll for element (polls every 5 seconds)
|
|
435
|
+
* @param {Object} [options.cache] - Cache configuration { thresholds: { screen, element } }
|
|
435
436
|
* @returns {Promise<Element>} This element instance
|
|
436
437
|
*/
|
|
437
438
|
async find(newDescription, options) {
|
|
@@ -468,10 +469,12 @@ class Element {
|
|
|
468
469
|
this._screenshot = screenshot;
|
|
469
470
|
}
|
|
470
471
|
|
|
471
|
-
// Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold
|
|
472
|
+
// Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold/cache
|
|
472
473
|
let cacheKey = null;
|
|
473
474
|
let cacheThreshold = null;
|
|
475
|
+
let perCommandThresholds = null; // Per-command { screen, element } override
|
|
474
476
|
let zoom = false; // Default to disabled, enable with zoom: true
|
|
477
|
+
let perCommandAi = null; // Per-command AI config override
|
|
475
478
|
|
|
476
479
|
if (typeof options === "number") {
|
|
477
480
|
// Legacy: options is just a number threshold
|
|
@@ -482,6 +485,10 @@ class Element {
|
|
|
482
485
|
cacheThreshold = options.cacheThreshold ?? null;
|
|
483
486
|
// zoom defaults to false unless explicitly set to true
|
|
484
487
|
zoom = options.zoom === true;
|
|
488
|
+
// Per-command cache thresholds: { cache: { thresholds: { screen: 0.1, element: 0.2 } } }
|
|
489
|
+
if (typeof options.cache === "object" && options.cache?.thresholds) {
|
|
490
|
+
perCommandThresholds = options.cache.thresholds;
|
|
491
|
+
}
|
|
485
492
|
}
|
|
486
493
|
|
|
487
494
|
// Use default cacheKey from SDK constructor if not provided in find() options
|
|
@@ -499,19 +506,25 @@ class Element {
|
|
|
499
506
|
// - If cacheKey is provided, enable cache with threshold
|
|
500
507
|
// - If no cacheKey, disable cache
|
|
501
508
|
let threshold;
|
|
509
|
+
let elementSimilarity;
|
|
502
510
|
if (this.sdk._cacheExplicitlyDisabled) {
|
|
503
511
|
// Cache explicitly disabled via cache: false option or TD_NO_CACHE env
|
|
504
512
|
threshold = -1;
|
|
513
|
+
elementSimilarity = -1;
|
|
505
514
|
cacheKey = null; // Clear any cacheKey to ensure cache is truly disabled
|
|
506
515
|
} else if (cacheKey) {
|
|
507
516
|
// cacheKey provided - enable cache with threshold
|
|
508
|
-
|
|
517
|
+
// Per-command thresholds > legacy cacheThreshold > global config
|
|
518
|
+
threshold = perCommandThresholds?.screen ?? cacheThreshold ?? this.sdk.cacheConfig?.thresholds?.find?.screen ?? 0.01;
|
|
519
|
+
elementSimilarity = perCommandThresholds?.element ?? this.sdk.cacheConfig?.thresholds?.find?.element ?? 0.8;
|
|
509
520
|
} else if (cacheThreshold !== null) {
|
|
510
521
|
// Explicit threshold provided without cacheKey
|
|
511
|
-
threshold = cacheThreshold;
|
|
522
|
+
threshold = perCommandThresholds?.screen ?? cacheThreshold;
|
|
523
|
+
elementSimilarity = perCommandThresholds?.element ?? this.sdk.cacheConfig?.thresholds?.find?.element ?? 0.8;
|
|
512
524
|
} else {
|
|
513
525
|
// No cacheKey, no explicit threshold - disable cache
|
|
514
526
|
threshold = -1;
|
|
527
|
+
elementSimilarity = -1;
|
|
515
528
|
}
|
|
516
529
|
|
|
517
530
|
// Store the threshold for debugging
|
|
@@ -536,10 +549,16 @@ class Element {
|
|
|
536
549
|
element: description,
|
|
537
550
|
image: screenshot,
|
|
538
551
|
threshold: threshold,
|
|
552
|
+
elementSimilarity: elementSimilarity,
|
|
539
553
|
cacheKey: cacheKey,
|
|
540
554
|
os: this.sdk.os,
|
|
541
555
|
resolution: this.sdk.resolution,
|
|
542
556
|
zoom: zoom,
|
|
557
|
+
ai: {
|
|
558
|
+
...this.sdk.aiConfig,
|
|
559
|
+
...(perCommandAi || {}),
|
|
560
|
+
top: { ...this.sdk.aiConfig?.top, ...(perCommandAi?.top || {}) },
|
|
561
|
+
},
|
|
543
562
|
});
|
|
544
563
|
|
|
545
564
|
const duration = Date.now() - startTime;
|
|
@@ -736,6 +755,9 @@ class Element {
|
|
|
736
755
|
cacheHit: debugInfo.cacheHit,
|
|
737
756
|
selectorId: this._response?.selector,
|
|
738
757
|
consoleUrl: consoleUrl,
|
|
758
|
+
validated: response.validated ?? null,
|
|
759
|
+
validationConfidence: response.validationConfidence ?? null,
|
|
760
|
+
coordsUpdated: response.coordsUpdated ?? null,
|
|
739
761
|
};
|
|
740
762
|
if (!debugInfo.cacheHit) {
|
|
741
763
|
meta.confidence = debugInfo.confidence;
|
|
@@ -1441,15 +1463,49 @@ class TestDriverSDK {
|
|
|
1441
1463
|
findAll: -1,
|
|
1442
1464
|
assert: -1,
|
|
1443
1465
|
};
|
|
1466
|
+
this.cacheConfig = {
|
|
1467
|
+
enabled: false,
|
|
1468
|
+
thresholds: {
|
|
1469
|
+
find: { screen: -1, element: -1 },
|
|
1470
|
+
assert: -1,
|
|
1471
|
+
},
|
|
1472
|
+
};
|
|
1444
1473
|
} else {
|
|
1445
|
-
//
|
|
1474
|
+
// Support cache object format: { cache: { thresholds: { find: { screen: 0.01, element: 0.8 }, assert: 0.05 } } }
|
|
1475
|
+
const cacheOpts = typeof options.cache === "object" ? options.cache : {};
|
|
1476
|
+
const thresholds = cacheOpts.thresholds || {};
|
|
1477
|
+
const findThresholds = typeof thresholds.find === "object" ? thresholds.find : {};
|
|
1478
|
+
|
|
1479
|
+
this.cacheConfig = {
|
|
1480
|
+
enabled: cacheOpts.enabled !== false,
|
|
1481
|
+
thresholds: {
|
|
1482
|
+
find: {
|
|
1483
|
+
screen: findThresholds.screen ?? 0.01, // Default: 1% pixel diff allowed
|
|
1484
|
+
element: findThresholds.element ?? 0.8, // Default: 80% OpenCV correlation
|
|
1485
|
+
},
|
|
1486
|
+
assert: thresholds.assert ?? 0.05, // Default: 5% pixel diff for assertions
|
|
1487
|
+
},
|
|
1488
|
+
};
|
|
1489
|
+
|
|
1490
|
+
// Legacy cacheThresholds - keep for backwards compatibility
|
|
1446
1491
|
this.cacheThresholds = {
|
|
1447
|
-
find: options.cacheThreshold?.find ??
|
|
1448
|
-
findAll: options.cacheThreshold?.findAll ??
|
|
1449
|
-
assert: options.cacheThreshold?.assert ??
|
|
1492
|
+
find: options.cacheThreshold?.find ?? this.cacheConfig.thresholds.find.screen,
|
|
1493
|
+
findAll: options.cacheThreshold?.findAll ?? this.cacheConfig.thresholds.find.screen,
|
|
1494
|
+
assert: options.cacheThreshold?.assert ?? this.cacheConfig.thresholds.assert,
|
|
1450
1495
|
};
|
|
1451
1496
|
}
|
|
1452
1497
|
|
|
1498
|
+
// AI sampling configuration
|
|
1499
|
+
// Supports: { ai: { temperature: 0, top: { p: 1, k: 0 } } }
|
|
1500
|
+
// Can be overridden per find() or assert() call
|
|
1501
|
+
this.aiConfig = typeof options.ai === "object" ? {
|
|
1502
|
+
temperature: options.ai.temperature,
|
|
1503
|
+
top: {
|
|
1504
|
+
p: options.ai.top?.p,
|
|
1505
|
+
k: options.ai.top?.k,
|
|
1506
|
+
},
|
|
1507
|
+
} : {};
|
|
1508
|
+
|
|
1453
1509
|
// Redraw configuration
|
|
1454
1510
|
// Supports both:
|
|
1455
1511
|
// - redraw: { enabled: true, diffThreshold: 0.1, screenRedraw: true, networkMonitor: true }
|
|
@@ -1656,26 +1712,6 @@ class TestDriverSDK {
|
|
|
1656
1712
|
guest = false,
|
|
1657
1713
|
} = options;
|
|
1658
1714
|
|
|
1659
|
-
// If dashcam is enabled, add web logs for all websites
|
|
1660
|
-
// Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
|
|
1661
|
-
if (this.dashcamEnabled) {
|
|
1662
|
-
// get the domain from the url for more specific logging, e.g. "Web Logs - example.com"
|
|
1663
|
-
let domain = url;
|
|
1664
|
-
let protocol = "https:";
|
|
1665
|
-
try {
|
|
1666
|
-
const urlObj = new URL(url);
|
|
1667
|
-
domain = urlObj.hostname;
|
|
1668
|
-
protocol = urlObj.protocol;
|
|
1669
|
-
} catch (err) {
|
|
1670
|
-
// If URL parsing fails, fall back to using the full URL as the domain
|
|
1671
|
-
}
|
|
1672
|
-
|
|
1673
|
-
// the pattern should be protocol://domain* to match all subpages of the domain
|
|
1674
|
-
const webLogPattern = `${protocol}//${domain}*`;
|
|
1675
|
-
|
|
1676
|
-
await this.dashcam.addWebLog(webLogPattern, "Web Logs");
|
|
1677
|
-
}
|
|
1678
|
-
|
|
1679
1715
|
// Set up Chrome profile with preferences
|
|
1680
1716
|
const shell = this.os === "windows" ? "pwsh" : "sh";
|
|
1681
1717
|
const userDataDir =
|
|
@@ -1930,12 +1966,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
|
|
1930
1966
|
);
|
|
1931
1967
|
}
|
|
1932
1968
|
|
|
1933
|
-
// If dashcam is enabled, add web logs for all websites
|
|
1934
|
-
// Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
|
|
1935
|
-
if (this.dashcamEnabled) {
|
|
1936
|
-
await this.dashcam.addWebLog("**", "Web Logs");
|
|
1937
|
-
}
|
|
1938
|
-
|
|
1939
1969
|
// Set up Chrome profile with preferences
|
|
1940
1970
|
const userDataDir =
|
|
1941
1971
|
this.os === "windows"
|
|
@@ -2056,12 +2086,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
|
|
2056
2086
|
|
|
2057
2087
|
const shell = this.os === "windows" ? "pwsh" : "sh";
|
|
2058
2088
|
|
|
2059
|
-
// If dashcam is enabled, add web logs for all websites
|
|
2060
|
-
// Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
|
|
2061
|
-
if (this.dashcamEnabled) {
|
|
2062
|
-
await this.dashcam.addWebLog("**", "Web Logs");
|
|
2063
|
-
}
|
|
2064
|
-
|
|
2065
2089
|
// Install extensions if provided
|
|
2066
2090
|
for (const extension of extensions) {
|
|
2067
2091
|
console.log(`[provision.vscode] Installing extension: ${extension}`);
|
|
@@ -2132,12 +2156,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
|
|
2132
2156
|
|
|
2133
2157
|
const shell = this.os === "windows" ? "pwsh" : "sh";
|
|
2134
2158
|
|
|
2135
|
-
// If dashcam is enabled, add web logs for all websites
|
|
2136
|
-
// Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
|
|
2137
|
-
if (this.dashcamEnabled) {
|
|
2138
|
-
await this.dashcam.addWebLog("**", "Web Logs");
|
|
2139
|
-
}
|
|
2140
|
-
|
|
2141
2159
|
// Determine download directory
|
|
2142
2160
|
const downloadDir =
|
|
2143
2161
|
this.os === "windows" ? "C:\\Users\\testdriver\\Downloads" : "/tmp";
|
|
@@ -2272,12 +2290,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
|
|
2272
2290
|
|
|
2273
2291
|
const shell = this.os === "windows" ? "pwsh" : "sh";
|
|
2274
2292
|
|
|
2275
|
-
// If dashcam is enabled, add web logs for all websites
|
|
2276
|
-
// Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
|
|
2277
|
-
if (this.dashcamEnabled) {
|
|
2278
|
-
await this.dashcam.addWebLog("**", "Web Logs");
|
|
2279
|
-
}
|
|
2280
|
-
|
|
2281
2293
|
const argsString = args.join(" ");
|
|
2282
2294
|
|
|
2283
2295
|
if (this.os === "windows") {
|
|
@@ -2791,7 +2803,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2791
2803
|
* Automatically locates the element and returns it
|
|
2792
2804
|
*
|
|
2793
2805
|
* @param {string} description - Description of the element to find
|
|
2794
|
-
* @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey,
|
|
2806
|
+
* @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cache: { thresholds: { screen, element } }}
|
|
2795
2807
|
* @returns {Promise<Element> & ChainableElement} Element instance that has been located, with chainable methods
|
|
2796
2808
|
*
|
|
2797
2809
|
* @example
|
|
@@ -2880,7 +2892,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2880
2892
|
* Automatically locates all matching elements and returns them as an array
|
|
2881
2893
|
*
|
|
2882
2894
|
* @param {string} description - Description of the elements to find
|
|
2883
|
-
* @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey,
|
|
2895
|
+
* @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cache: { thresholds: { screen } }}
|
|
2884
2896
|
* @returns {Promise<Element[]>} Array of Element instances that have been located
|
|
2885
2897
|
*
|
|
2886
2898
|
* @example
|
|
@@ -2936,9 +2948,10 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2936
2948
|
try {
|
|
2937
2949
|
const screenshot = await this.system.captureScreenBase64();
|
|
2938
2950
|
|
|
2939
|
-
// Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold
|
|
2951
|
+
// Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold/cache
|
|
2940
2952
|
let cacheKey = null;
|
|
2941
2953
|
let cacheThreshold = null;
|
|
2954
|
+
let perCommandThresholds = null; // Per-command { screen } override (findAll has no element threshold)
|
|
2942
2955
|
|
|
2943
2956
|
if (typeof options === "number") {
|
|
2944
2957
|
// Legacy: options is just a number threshold
|
|
@@ -2947,6 +2960,10 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2947
2960
|
// New: options is an object with cacheKey and/or cacheThreshold
|
|
2948
2961
|
cacheKey = options.cacheKey || null;
|
|
2949
2962
|
cacheThreshold = options.cacheThreshold ?? null;
|
|
2963
|
+
// Per-command cache thresholds: { cache: { thresholds: { screen: 0.1 } } }
|
|
2964
|
+
if (typeof options.cache === "object" && options.cache?.thresholds) {
|
|
2965
|
+
perCommandThresholds = options.cache.thresholds;
|
|
2966
|
+
}
|
|
2950
2967
|
}
|
|
2951
2968
|
|
|
2952
2969
|
// Use default cacheKey from SDK constructor if not provided in findAll() options
|
|
@@ -2969,11 +2986,11 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2969
2986
|
threshold = -1;
|
|
2970
2987
|
cacheKey = null; // Clear any cacheKey to ensure cache is truly disabled
|
|
2971
2988
|
} else if (cacheKey) {
|
|
2972
|
-
// cacheKey provided - enable cache with threshold
|
|
2973
|
-
threshold = cacheThreshold ?? this.
|
|
2989
|
+
// cacheKey provided - enable cache with threshold (findAll only uses screen, no element)
|
|
2990
|
+
threshold = perCommandThresholds?.screen ?? cacheThreshold ?? this.cacheConfig?.thresholds?.find?.screen ?? 0.01;
|
|
2974
2991
|
} else if (cacheThreshold !== null) {
|
|
2975
2992
|
// Explicit threshold provided without cacheKey
|
|
2976
|
-
threshold = cacheThreshold;
|
|
2993
|
+
threshold = perCommandThresholds?.screen ?? cacheThreshold;
|
|
2977
2994
|
} else {
|
|
2978
2995
|
// No cacheKey, no explicit threshold - disable cache
|
|
2979
2996
|
threshold = -1;
|
|
@@ -2994,7 +3011,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
2994
3011
|
}
|
|
2995
3012
|
|
|
2996
3013
|
const response = await this.apiClient.req(
|
|
2997
|
-
"/api/v7.0.0/testdriver
|
|
3014
|
+
"/api/v7.0.0/testdriver/find-all",
|
|
2998
3015
|
{
|
|
2999
3016
|
session: this.getSessionId(),
|
|
3000
3017
|
element: description,
|
|
@@ -3010,7 +3027,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
3010
3027
|
|
|
3011
3028
|
if (response && response.elements && response.elements.length > 0) {
|
|
3012
3029
|
// Single log at the end - found elements
|
|
3013
|
-
const formattedMessage = formatter.
|
|
3030
|
+
const formattedMessage = formatter.formatElementsFound(
|
|
3014
3031
|
description,
|
|
3015
3032
|
response.elements.length,
|
|
3016
3033
|
{
|
|
@@ -3093,7 +3110,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
3093
3110
|
const duration = Date.now() - startTime;
|
|
3094
3111
|
|
|
3095
3112
|
// Single log at the end - no elements found
|
|
3096
|
-
const formattedMessage = formatter.
|
|
3113
|
+
const formattedMessage = formatter.formatElementsFound(
|
|
3097
3114
|
description,
|
|
3098
3115
|
0,
|
|
3099
3116
|
{
|
|
@@ -3139,7 +3156,7 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
3139
3156
|
const duration = Date.now() - startTime;
|
|
3140
3157
|
|
|
3141
3158
|
// Single log at the end - error
|
|
3142
|
-
const formattedMessage = formatter.
|
|
3159
|
+
const formattedMessage = formatter.formatElementsFound(
|
|
3143
3160
|
description,
|
|
3144
3161
|
0,
|
|
3145
3162
|
{
|
|
@@ -3334,16 +3351,30 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
3334
3351
|
let result;
|
|
3335
3352
|
// Special handling for assert to inject SDK options (cacheKey, os, resolution, threshold)
|
|
3336
3353
|
// similar to how find() handles these in the Element class
|
|
3354
|
+
// Note: assert does NOT use elementSimilarity (template matching not relevant for assertions)
|
|
3337
3355
|
if (commandName === 'assert') {
|
|
3338
3356
|
const assertion = args[0];
|
|
3339
3357
|
const userOptions = args[1] || {};
|
|
3340
3358
|
|
|
3359
|
+
// Support per-command cache threshold override: { cache: { threshold: 0.05 } }
|
|
3360
|
+
const perCommandThreshold = typeof userOptions.cache === "object"
|
|
3361
|
+
? userOptions.cache.threshold
|
|
3362
|
+
: undefined;
|
|
3363
|
+
|
|
3341
3364
|
// Merge SDK defaults with user options (user options take precedence)
|
|
3342
3365
|
const mergedOptions = {
|
|
3343
3366
|
cacheKey: userOptions.cacheKey ?? sdk.options.cacheKey,
|
|
3344
3367
|
os: userOptions.os ?? sdk.os,
|
|
3345
3368
|
resolution: userOptions.resolution ?? sdk.resolution,
|
|
3346
|
-
threshold:
|
|
3369
|
+
threshold: perCommandThreshold ?? userOptions.threshold ?? (sdk.cacheConfig?.thresholds?.assert ?? sdk.cacheThresholds?.assert ?? 0.05),
|
|
3370
|
+
ai: {
|
|
3371
|
+
...sdk.aiConfig,
|
|
3372
|
+
...(typeof userOptions.ai === "object" ? userOptions.ai : {}),
|
|
3373
|
+
top: {
|
|
3374
|
+
...sdk.aiConfig?.top,
|
|
3375
|
+
...(typeof userOptions.ai === "object" ? userOptions.ai?.top : {}),
|
|
3376
|
+
},
|
|
3377
|
+
},
|
|
3347
3378
|
};
|
|
3348
3379
|
|
|
3349
3380
|
// Note: commands.assert takes (assertion, options), shouldThrow is determined internally
|
|
@@ -3451,74 +3482,70 @@ CAPTCHA_SOLVER_EOF`,
|
|
|
3451
3482
|
}
|
|
3452
3483
|
|
|
3453
3484
|
/**
|
|
3454
|
-
*
|
|
3455
|
-
* Returns structured data with
|
|
3485
|
+
* Parse the current screen using OmniParser v2 to detect all UI elements
|
|
3486
|
+
* Returns structured data with element types, bounding boxes, and content
|
|
3487
|
+
* Requires enterprise or self-hosted plan.
|
|
3456
3488
|
*
|
|
3457
|
-
* @returns {Promise<
|
|
3489
|
+
* @returns {Promise<ParseResult>} Parsed screen elements
|
|
3458
3490
|
*
|
|
3459
|
-
* @typedef {Object}
|
|
3460
|
-
* @property {
|
|
3461
|
-
* @property {string}
|
|
3462
|
-
* @property {number} confidence - Overall OCR confidence (0-100)
|
|
3491
|
+
* @typedef {Object} ParseResult
|
|
3492
|
+
* @property {ParsedElement[]} elements - Array of detected UI elements
|
|
3493
|
+
* @property {string} annotatedImageUrl - URL of the annotated screenshot
|
|
3463
3494
|
* @property {number} imageWidth - Width of the analyzed image
|
|
3464
3495
|
* @property {number} imageHeight - Height of the analyzed image
|
|
3465
3496
|
*
|
|
3466
|
-
* @typedef {Object}
|
|
3467
|
-
* @property {
|
|
3468
|
-
* @property {
|
|
3469
|
-
* @property {
|
|
3497
|
+
* @typedef {Object} ParsedElement
|
|
3498
|
+
* @property {number} index - Element index
|
|
3499
|
+
* @property {string} type - Element type (e.g. "text", "icon", "button")
|
|
3500
|
+
* @property {string} content - Text content or description
|
|
3501
|
+
* @property {string} interactivity - Interactivity level (e.g. "clickable", "non-interactive")
|
|
3502
|
+
* @property {Object} bbox - Bounding box in pixel coordinates
|
|
3470
3503
|
* @property {number} bbox.x0 - Left edge X coordinate
|
|
3471
3504
|
* @property {number} bbox.y0 - Top edge Y coordinate
|
|
3472
3505
|
* @property {number} bbox.x1 - Right edge X coordinate
|
|
3473
3506
|
* @property {number} bbox.y1 - Bottom edge Y coordinate
|
|
3507
|
+
* @property {Object} boundingBox - Bounding box as {left, top, width, height}
|
|
3508
|
+
* @property {number} boundingBox.left - Left position
|
|
3509
|
+
* @property {number} boundingBox.top - Top position
|
|
3510
|
+
* @property {number} boundingBox.width - Element width
|
|
3511
|
+
* @property {number} boundingBox.height - Element height
|
|
3474
3512
|
*
|
|
3475
3513
|
* @example
|
|
3476
|
-
* // Get all
|
|
3477
|
-
* const result = await testdriver.
|
|
3478
|
-
* console.log(result.
|
|
3479
|
-
* // "Welcome to TestDriver Sign In Email Password Submit"
|
|
3480
|
-
*
|
|
3481
|
-
* @example
|
|
3482
|
-
* // Find words matching a pattern
|
|
3483
|
-
* const result = await testdriver.ocr();
|
|
3484
|
-
* const buttons = result.words.filter(w =>
|
|
3485
|
-
* w.content.toLowerCase().includes('button')
|
|
3486
|
-
* );
|
|
3514
|
+
* // Get all elements on screen
|
|
3515
|
+
* const result = await testdriver.parse();
|
|
3516
|
+
* console.log(`Found ${result.elements.length} elements`);
|
|
3487
3517
|
*
|
|
3488
3518
|
* @example
|
|
3489
|
-
* //
|
|
3490
|
-
* const result = await testdriver.
|
|
3491
|
-
* const
|
|
3492
|
-
* if (submitWord) {
|
|
3493
|
-
* // Calculate center of the word
|
|
3494
|
-
* const x = (submitWord.bbox.x0 + submitWord.bbox.x1) / 2;
|
|
3495
|
-
* const y = (submitWord.bbox.y0 + submitWord.bbox.y1) / 2;
|
|
3496
|
-
* await testdriver.click({ x, y });
|
|
3497
|
-
* }
|
|
3519
|
+
* // Find clickable elements
|
|
3520
|
+
* const result = await testdriver.parse();
|
|
3521
|
+
* const clickable = result.elements.filter(e => e.interactivity === 'clickable');
|
|
3498
3522
|
*
|
|
3499
3523
|
* @example
|
|
3500
|
-
* //
|
|
3501
|
-
* const result = await testdriver.
|
|
3502
|
-
* const
|
|
3503
|
-
*
|
|
3504
|
-
* );
|
|
3524
|
+
* // Find text content
|
|
3525
|
+
* const result = await testdriver.parse();
|
|
3526
|
+
* const textElements = result.elements.filter(e => e.type === 'text');
|
|
3527
|
+
* textElements.forEach(e => console.log(e.content));
|
|
3505
3528
|
*/
|
|
3506
|
-
async
|
|
3529
|
+
async parse() {
|
|
3507
3530
|
this._ensureConnected();
|
|
3508
3531
|
|
|
3509
3532
|
const { events } = require("./agent/events.js");
|
|
3510
|
-
this.emitter.emit(events.log.log, "🔍 Running
|
|
3533
|
+
this.emitter.emit(events.log.log, "🔍 Running OmniParser screen analysis...");
|
|
3511
3534
|
|
|
3512
3535
|
const screenshot = await this.system.captureScreenBase64();
|
|
3513
3536
|
|
|
3514
|
-
const response = await this.apiClient.req("
|
|
3537
|
+
const response = await this.apiClient.req("parse", {
|
|
3515
3538
|
session: this.getSessionId(),
|
|
3516
3539
|
image: screenshot,
|
|
3517
3540
|
});
|
|
3518
3541
|
|
|
3542
|
+
if (response.error) {
|
|
3543
|
+
throw new Error(response.error);
|
|
3544
|
+
}
|
|
3545
|
+
|
|
3519
3546
|
this.emitter.emit(
|
|
3520
3547
|
events.log.log,
|
|
3521
|
-
`✅
|
|
3548
|
+
`✅ Parse complete: ${response.elements?.length || 0} elements detected`,
|
|
3522
3549
|
);
|
|
3523
3550
|
|
|
3524
3551
|
return response;
|