testdriverai 7.3.9 → 7.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,11 @@
1
+ ## [7.3.11](https://github.com/testdriverai/testdriverai/compare/v7.3.10...v7.3.11) (2026-02-17)
2
+
3
+
4
+
5
+ ## [7.3.10](https://github.com/testdriverai/testdriverai/compare/v7.3.9...v7.3.10) (2026-02-16)
6
+
7
+
8
+
1
9
  ## [7.3.9](https://github.com/testdriverai/testdriverai/compare/v7.3.8...v7.3.9) (2026-02-12)
2
10
 
3
11
 
@@ -226,8 +226,8 @@ const createCommands = (
226
226
  const assertTimestamp = Date.now();
227
227
  const assertStartTime = assertTimestamp;
228
228
 
229
- // Extract cache options
230
- const { threshold = -1, cacheKey, os, resolution } = options;
229
+ // Extract cache and AI options
230
+ const { threshold = -1, cacheKey, os, resolution, ai } = options;
231
231
 
232
232
  // Debug log cache settings
233
233
  emitter.emit(
@@ -243,6 +243,7 @@ const createCommands = (
243
243
  cacheKey,
244
244
  os,
245
245
  resolution,
246
+ ai,
246
247
  });
247
248
 
248
249
  const assertDuration = Date.now() - assertStartTime;
@@ -2,136 +2,136 @@
2
2
  "$schema": "./examples-manifest.schema.json",
3
3
  "examples": {
4
4
  "assert.test.mjs": {
5
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16d4a2aa03cd95d96b60",
6
- "lastUpdated": "2026-02-12T00:06:25.201Z"
5
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f88dd202a57999808e3",
6
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
7
7
  },
8
8
  "drag-and-drop.test.mjs": {
9
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1528a2aa03cd95d96a2c",
10
- "lastUpdated": "2026-02-12T00:06:25.196Z"
9
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7df8d3b320ad547d9cdc",
10
+ "lastUpdated": "2026-02-13T19:55:05.944Z"
11
11
  },
12
12
  "exec-pwsh.test.mjs": {
13
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d152aa2aa03cd95d96a33",
14
- "lastUpdated": "2026-02-12T00:06:25.197Z"
13
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7df9d3b320ad547d9cdf",
14
+ "lastUpdated": "2026-02-13T19:55:05.944Z"
15
15
  },
16
16
  "match-image.test.mjs": {
17
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d152ba2aa03cd95d96a34",
18
- "lastUpdated": "2026-02-12T00:06:25.197Z"
17
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7dfad3b320ad547d9ce0",
18
+ "lastUpdated": "2026-02-13T19:55:05.944Z"
19
19
  },
20
20
  "scroll-until-text.test.mjs": {
21
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1590a2aa03cd95d96a82",
22
- "lastUpdated": "2026-02-12T00:06:25.198Z"
21
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5cd3b320ad547d9d03",
22
+ "lastUpdated": "2026-02-13T19:55:05.945Z"
23
23
  },
24
24
  "hover-text-with-description.test.mjs": {
25
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d157fa2aa03cd95d96a73",
26
- "lastUpdated": "2026-02-12T00:06:25.197Z"
25
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e4e9e27ce1528d7d0a0",
26
+ "lastUpdated": "2026-02-13T19:55:05.944Z"
27
27
  },
28
28
  "windows-installer.test.mjs": {
29
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1591a2aa03cd95d96a84",
30
- "lastUpdated": "2026-02-12T00:06:25.198Z"
29
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5ddd202a57999808ae",
30
+ "lastUpdated": "2026-02-13T19:55:05.945Z"
31
31
  },
32
32
  "exec-output.test.mjs": {
33
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1592a2aa03cd95d96a85",
34
- "lastUpdated": "2026-02-12T00:06:25.198Z"
33
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7e5edd202a57999808af",
34
+ "lastUpdated": "2026-02-13T19:55:05.945Z"
35
35
  },
36
36
  "chrome-extension.test.mjs": {
37
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d15eea2aa03cd95d96ac8",
38
- "lastUpdated": "2026-02-12T00:06:25.198Z"
37
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7eb69e27ce1528d7d0bc",
38
+ "lastUpdated": "2026-02-13T19:55:05.946Z"
39
39
  },
40
40
  "launch-vscode-linux.test.mjs": {
41
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d15eda2aa03cd95d96ac7",
42
- "lastUpdated": "2026-02-12T00:06:25.198Z"
41
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7eb6d3b320ad547d9d16",
42
+ "lastUpdated": "2026-02-13T19:55:05.946Z"
43
43
  },
44
44
  "hover-image.test.mjs": {
45
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d15dfa2aa03cd95d96abd",
46
- "lastUpdated": "2026-02-12T00:06:25.198Z"
45
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7ea7dd202a57999808bc",
46
+ "lastUpdated": "2026-02-13T19:55:05.946Z"
47
47
  },
48
48
  "installer.test.mjs": {
49
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1640a2aa03cd95d96af3",
50
- "lastUpdated": "2026-02-12T00:06:25.200Z"
49
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f069e27ce1528d7d0cd",
50
+ "lastUpdated": "2026-02-13T19:55:05.948Z"
51
51
  },
52
52
  "type.test.mjs": {
53
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1665a2aa03cd95d96b08",
54
- "lastUpdated": "2026-02-12T00:06:25.200Z"
53
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f2a9e27ce1528d7d0d2",
54
+ "lastUpdated": "2026-02-13T19:55:05.948Z"
55
55
  },
56
56
  "press-keys.test.mjs": {
57
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16aca2aa03cd95d96b40",
58
- "lastUpdated": "2026-02-12T00:06:25.201Z"
57
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f739e27ce1528d7d0d9",
58
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
59
59
  },
60
60
  "scroll-keyboard.test.mjs": {
61
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d169fa2aa03cd95d96b35",
62
- "lastUpdated": "2026-02-12T00:06:25.201Z"
61
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f55dd202a57999808d6",
62
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
63
63
  },
64
64
  "scroll.test.mjs": {
65
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d164ca2aa03cd95d96afc",
66
- "lastUpdated": "2026-02-12T00:06:25.200Z"
65
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f0add202a57999808c8",
66
+ "lastUpdated": "2026-02-13T19:55:05.948Z"
67
67
  },
68
68
  "scroll-until-image.test.mjs": {
69
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16ada2aa03cd95d96b42",
70
- "lastUpdated": "2026-02-12T00:06:25.201Z"
69
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f749e27ce1528d7d0da",
70
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
71
71
  },
72
72
  "prompt.test.mjs": {
73
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16afa2aa03cd95d96b43",
74
- "lastUpdated": "2026-02-12T00:06:25.201Z"
73
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f75d3b320ad547d9d39",
74
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
75
75
  },
76
76
  "focus-window.test.mjs": {
77
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16b0a2aa03cd95d96b44",
78
- "lastUpdated": "2026-02-12T00:06:25.201Z"
77
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7f76d3b320ad547d9d3a",
78
+ "lastUpdated": "2026-02-13T19:55:05.949Z"
79
79
  },
80
80
  "captcha-api.test.mjs": {
81
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16cfa2aa03cd95d96b57",
82
- "lastUpdated": "2026-02-12T00:06:25.201Z"
81
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb0d3b320ad547d9d44",
82
+ "lastUpdated": "2026-02-13T19:55:05.951Z"
83
83
  },
84
84
  "element-not-found.test.mjs": {
85
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1705a2aa03cd95d96b79",
86
- "lastUpdated": "2026-02-12T00:06:25.202Z"
85
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fbcdd202a57999808f0",
86
+ "lastUpdated": "2026-02-13T19:55:05.951Z"
87
87
  },
88
88
  "formatted-logging.test.mjs": {
89
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d16eaa2aa03cd95d96b6b",
90
- "lastUpdated": "2026-02-12T00:06:25.202Z"
89
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb1dd202a57999808e9",
90
+ "lastUpdated": "2026-02-13T19:55:05.951Z"
91
91
  },
92
92
  "hover-text.test.mjs": {
93
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1724a2aa03cd95d96b8d",
94
- "lastUpdated": "2026-02-12T00:06:25.203Z"
93
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fe69e27ce1528d7d0ed",
94
+ "lastUpdated": "2026-02-13T19:55:05.951Z"
95
95
  },
96
96
  "no-provision.test.mjs": {
97
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d173ea2aa03cd95d96b9d",
98
- "lastUpdated": "2026-02-12T00:06:25.203Z"
97
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fdf9e27ce1528d7d0ec",
98
+ "lastUpdated": "2026-02-13T19:55:05.951Z"
99
99
  },
100
100
  "ai.test.mjs": {
101
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1763a2aa03cd95d96bae",
102
- "lastUpdated": "2026-02-12T00:06:25.203Z"
101
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8010d3b320ad547d9d52",
102
+ "lastUpdated": "2026-02-13T19:55:05.952Z"
103
103
  },
104
104
  "popup-loading.test.mjs": {
105
105
  "url": "https://console.testdriver.ai/runs/698bc89f7140c3fa7daaca8d/698bca7f7140c3fa7daacbf7",
106
106
  "lastUpdated": "2026-02-11T00:20:33.687Z"
107
107
  },
108
108
  "z_flake-diffthreshold-001.test.mjs": {
109
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d186aa2aa03cd95d96c33",
110
- "lastUpdated": "2026-02-12T00:06:25.204Z"
109
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8090d3b320ad547d9d67",
110
+ "lastUpdated": "2026-02-13T19:55:05.952Z"
111
111
  },
112
112
  "z_flake-diffthreshold-01.test.mjs": {
113
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1838a2aa03cd95d96c16",
114
- "lastUpdated": "2026-02-12T00:06:25.203Z"
113
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f80bc9e27ce1528d7d106",
114
+ "lastUpdated": "2026-02-13T19:55:05.952Z"
115
115
  },
116
116
  "z_flake-diffthreshold-05.test.mjs": {
117
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d183ba2aa03cd95d96c1d",
118
- "lastUpdated": "2026-02-12T00:06:25.203Z"
117
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8090dd202a5799980910",
118
+ "lastUpdated": "2026-02-13T19:55:05.952Z"
119
119
  },
120
120
  "z_flake-noredraw-cache.test.mjs": {
121
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d18b0a2aa03cd95d96c61",
122
- "lastUpdated": "2026-02-12T00:06:25.205Z"
121
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8105d3b320ad547d9d75",
122
+ "lastUpdated": "2026-02-13T19:55:05.953Z"
123
123
  },
124
124
  "z_flake-redraw-nocache.test.mjs": {
125
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1901a2aa03cd95d96c85",
126
- "lastUpdated": "2026-02-12T00:06:25.205Z"
125
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f8142d3b320ad547d9d86",
126
+ "lastUpdated": "2026-02-13T19:55:05.953Z"
127
127
  },
128
128
  "z_flake-redraw-cache.test.mjs": {
129
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d1976a2aa03cd95d96c9c",
130
- "lastUpdated": "2026-02-12T00:06:25.205Z"
129
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f818f9e27ce1528d7d11d",
130
+ "lastUpdated": "2026-02-13T19:55:05.953Z"
131
131
  },
132
132
  "z_flake-noredraw-nocache.test.mjs": {
133
- "url": "https://console.testdriver.ai/runs/698d1527a2aa03cd95d96a2b/698d190ca2aa03cd95d96c8a",
134
- "lastUpdated": "2026-02-12T00:06:25.205Z"
133
+ "url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f816ed3b320ad547d9d8c",
134
+ "lastUpdated": "2026-02-13T19:55:05.953Z"
135
135
  }
136
136
  }
137
137
  }
@@ -13,6 +13,7 @@ Make AI-powered assertions about the current screen state using natural language
13
13
 
14
14
  ```javascript
15
15
  await testdriver.assert(assertion)
16
+ await testdriver.assert(assertion, options)
16
17
  ```
17
18
 
18
19
  ## Parameters
@@ -21,6 +22,36 @@ await testdriver.assert(assertion)
21
22
  Natural language description of what should be true
22
23
  </ParamField>
23
24
 
25
+ <ParamField path="options" type="object">
26
+ Optional configuration
27
+
28
+ <Expandable title="properties">
29
+ <ParamField path="ai" type="object">
30
+ AI sampling configuration for this assert call (overrides global `ai` config from constructor).
31
+
32
+ <Expandable title="properties">
33
+ <ParamField path="temperature" type="number">
34
+ Controls randomness. `0` = deterministic, higher = more creative. Default: model default.
35
+ </ParamField>
36
+
37
+ <ParamField path="top" type="object">
38
+ Sampling parameters
39
+
40
+ <Expandable title="properties">
41
+ <ParamField path="p" type="number">
42
+ Top-P (nucleus sampling). Range: 0-1.
43
+ </ParamField>
44
+
45
+ <ParamField path="k" type="number">
46
+ Top-K sampling. `1` = most deterministic.
47
+ </ParamField>
48
+ </Expandable>
49
+ </ParamField>
50
+ </Expandable>
51
+ </ParamField>
52
+ </Expandable>
53
+ </ParamField>
54
+
24
55
  ## Returns
25
56
 
26
57
  `Promise<boolean>` - `true` if assertion passes, throws error if assertion fails
@@ -52,6 +52,30 @@ const testdriver = new TestDriver(apiKey, options)
52
52
  <ParamField path="environment" type="object">
53
53
  Additional environment variables to pass to the sandbox
54
54
  </ParamField>
55
+
56
+ <ParamField path="ai" type="object">
57
+ Global AI sampling configuration. Controls how the AI model generates responses for `find()` verification and `assert()` calls. Can be overridden per call.
58
+
59
+ <Expandable title="properties">
60
+ <ParamField path="temperature" type="number">
61
+ Controls randomness in AI responses. `0` = deterministic (best for verification), higher values = more creative. Default: `0` for find verification, model default for assert.
62
+ </ParamField>
63
+
64
+ <ParamField path="top" type="object">
65
+ Nucleus and top-k sampling parameters
66
+
67
+ <Expandable title="properties">
68
+ <ParamField path="p" type="number">
69
+ Top-P (nucleus sampling). Limits token choices to the smallest set whose cumulative probability exceeds P. Lower values = more focused responses. Range: 0-1.
70
+ </ParamField>
71
+
72
+ <ParamField path="k" type="number">
73
+ Top-K sampling. Limits token choices to the top K most likely tokens. `1` = always pick the most likely token. `0` = disabled (consider all tokens).
74
+ </ParamField>
75
+ </Expandable>
76
+ </ParamField>
77
+ </Expandable>
78
+ </ParamField>
55
79
  </Expandable>
56
80
  </ParamField>
57
81
 
@@ -68,6 +92,11 @@ const testdriver = new TestDriver({
68
92
  analytics: true
69
93
  });
70
94
 
95
+ // With AI config for stricter verification
96
+ const testdriver = new TestDriver({
97
+ ai: { temperature: 0, top: { p: 0.9, k: 40 } }
98
+ });
99
+
71
100
  // Or pass API key explicitly
72
101
  const testdriver = new TestDriver('your-api-key', {
73
102
  os: 'windows'
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* ai.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1763a2aa03cd95d96bae/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f8010d3b320ad547d9d52/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* assert.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d16d4a2aa03cd95d96b60/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f88dd202a57999808e3/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* captcha-api.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d16cfa2aa03cd95d96b57/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fb0d3b320ad547d9d44/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* chrome-extension.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d15eea2aa03cd95d96ac8/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7eb69e27ce1528d7d0bc/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* drag-and-drop.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1528a2aa03cd95d96a2c/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7df8d3b320ad547d9cdc/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* element-not-found.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1705a2aa03cd95d96b79/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fbcdd202a57999808f0/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* hover-image.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d15dfa2aa03cd95d96abd/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7ea7dd202a57999808bc/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* hover-text.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1724a2aa03cd95d96b8d/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7fe69e27ce1528d7d0ed/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* installer.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1640a2aa03cd95d96af3/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f069e27ce1528d7d0cd/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* launch-vscode-linux.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d15eda2aa03cd95d96ac7/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7eb6d3b320ad547d9d16/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* match-image.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d152ba2aa03cd95d96a34/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7dfad3b320ad547d9ce0/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* press-keys.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d16aca2aa03cd95d96b40/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f739e27ce1528d7d0d9/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* scroll-keyboard.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d169fa2aa03cd95d96b35/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f55dd202a57999808d6/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* scroll-until-image.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d16ada2aa03cd95d96b42/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f749e27ce1528d7d0da/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* scroll-until-text.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1590a2aa03cd95d96a82/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7e5cd3b320ad547d9d03/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* scroll.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d164ca2aa03cd95d96afc/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f0add202a57999808c8/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* type.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1665a2aa03cd95d96b08/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7f2a9e27ce1528d7d0d2/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
12
12
 
13
13
  {/* windows-installer.test.mjs output */}
14
14
  <iframe
15
- src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698d1591a2aa03cd95d96a84/replay"
15
+ src="https://testdriver-api.onrender.com/api/v1/testdriver/testcase/698f7e5ddd202a57999808ae/replay"
16
16
  width="100%"
17
17
  height="390"
18
18
  style={{ border: "1px solid #333", borderRadius: "8px" }}
package/docs/v7/find.mdx CHANGED
@@ -41,6 +41,30 @@ const element = await testdriver.find(description, options)
41
41
  <ParamField path="zoom" type="boolean" default={false}>
42
42
  Enable two-phase zoom mode for better precision in crowded UIs with many similar elements.
43
43
  </ParamField>
44
+
45
+ <ParamField path="ai" type="object">
46
+ AI sampling configuration for this find call (overrides global `ai` config from constructor).
47
+
48
+ <Expandable title="properties">
49
+ <ParamField path="temperature" type="number">
50
+ Controls randomness. `0` = deterministic. Default: `0` for find verification.
51
+ </ParamField>
52
+
53
+ <ParamField path="top" type="object">
54
+ Sampling parameters
55
+
56
+ <Expandable title="properties">
57
+ <ParamField path="p" type="number">
58
+ Top-P (nucleus sampling). Range: 0-1.
59
+ </ParamField>
60
+
61
+ <ParamField path="k" type="number">
62
+ Top-K sampling. `1` = most deterministic.
63
+ </ParamField>
64
+ </Expandable>
65
+ </ParamField>
66
+ </Expandable>
67
+ </ParamField>
44
68
  </Expandable>
45
69
  </ParamField>
46
70
 
@@ -22,7 +22,7 @@ export function popupLoadingTest(label, options = {}) {
22
22
  // Accept the cookie banner to trigger the loading process
23
23
  let acceptButton = await testdriver.find("Accept All button on the cookie banner", {timeout: 60000});
24
24
 
25
- if (acceptButton.found()) {
25
+ if (await acceptButton.found()) {
26
26
  await acceptButton.click();
27
27
  } else {
28
28
  console.log('no cookie banner found, proceeding without accepting cookies');
@@ -324,6 +324,9 @@ export function TestDriver(context, options = {}) {
324
324
  // Add testdriver log to dashcam tracking
325
325
  await testdriver.dashcam.addFileLog(logPath, "TestDriver Log");
326
326
 
327
+ // Add web log tracking before starting dashcam
328
+ await testdriver.dashcam.addWebLog("**", "Web Logs");
329
+
327
330
  // Start dashcam recording
328
331
  await testdriver.dashcam.start();
329
332
  }
@@ -401,8 +401,8 @@ Debug mode (connect to existing sandbox):
401
401
  const TestDriverSDK = (await import("../../sdk.js")).default;
402
402
  // Determine preview mode from environment variable
403
403
  // TD_PREVIEW can be "ide", "browser", or "none"
404
- // Default to "none" for MCP server (headless) unless explicitly set
405
- const previewMode = process.env.TD_PREVIEW || "none";
404
+ // Default to "ide" so the live preview shows within the IDE
405
+ const previewMode = process.env.TD_PREVIEW || "ide";
406
406
  logger.debug("session_start: Preview mode", { preview: previewMode });
407
407
  // Get IP from params or environment (for self-hosted instances)
408
408
  const instanceIp = params.ip || process.env.TD_IP;
@@ -509,8 +509,8 @@ Debug mode (connect to existing sandbox):
509
509
 
510
510
  // Determine preview mode from environment variable
511
511
  // TD_PREVIEW can be "ide", "browser", or "none"
512
- // Default to "none" for MCP server (headless) unless explicitly set
513
- const previewMode = process.env.TD_PREVIEW || "none";
512
+ // Default to "ide" so the live preview shows within the IDE
513
+ const previewMode = process.env.TD_PREVIEW || "ide";
514
514
  logger.debug("session_start: Preview mode", { preview: previewMode });
515
515
 
516
516
  // Get IP from params or environment (for self-hosted instances)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testdriverai",
3
- "version": "7.3.9",
3
+ "version": "7.3.11",
4
4
  "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
5
5
  "main": "sdk.js",
6
6
  "types": "sdk.d.ts",
package/sdk.d.ts CHANGED
@@ -230,6 +230,8 @@ export interface TestDriverOptions {
230
230
  logging?: boolean;
231
231
  /** Enable/disable cache (default: true). Set to false to force regeneration on all find operations */
232
232
  cache?: boolean;
233
+ /** Global AI sampling configuration. Can be overridden per find() or assert() call. */
234
+ ai?: AIConfig;
233
235
  /** Cache threshold configuration for different methods */
234
236
  cacheThreshold?: {
235
237
  /** Threshold for find operations (default: 0.05 = 5% difference, 95% similarity) */
@@ -546,6 +548,19 @@ export interface FocusApplicationOptions {
546
548
  name: string;
547
549
  }
548
550
 
551
+ /** AI sampling configuration for controlling model behavior */
552
+ export interface AIConfig {
553
+ /** Temperature for AI sampling (0 = deterministic, higher = more creative). Default: 0 for find verification, model default for assert. */
554
+ temperature?: number;
555
+ /** Top-P and Top-K sampling parameters */
556
+ top?: {
557
+ /** Top-P (nucleus sampling). Controls diversity by limiting to top P probability mass. Range: 0-1. */
558
+ p?: number;
559
+ /** Top-K sampling. Limits choices to top K tokens. 1 = always pick most likely. 0 = disabled. */
560
+ k?: number;
561
+ };
562
+ }
563
+
549
564
  /** Options for extract command */
550
565
  export interface ExtractOptions {
551
566
  /** What to extract */
@@ -564,6 +579,8 @@ export interface AssertOptions {
564
579
  os?: string;
565
580
  /** Screen resolution for cache partitioning */
566
581
  resolution?: string;
582
+ /** AI sampling configuration (overrides global ai config) */
583
+ ai?: AIConfig;
567
584
  }
568
585
 
569
586
  /** Options for exec command */
@@ -1028,7 +1045,7 @@ export default class TestDriverSDK {
1028
1045
  find(description: string, cacheThreshold?: number): ChainableElementPromise;
1029
1046
  find(
1030
1047
  description: string,
1031
- options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number },
1048
+ options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; ai?: AIConfig },
1032
1049
  ): ChainableElementPromise;
1033
1050
 
1034
1051
  /**
@@ -1267,7 +1284,7 @@ export default class TestDriverSDK {
1267
1284
  * // With custom threshold
1268
1285
  * await client.assert('the page loaded', { threshold: 0.01, cacheKey: 'login-test' });
1269
1286
  */
1270
- assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string }): Promise<boolean>;
1287
+ assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string; ai?: AIConfig }): Promise<boolean>;
1271
1288
 
1272
1289
  /**
1273
1290
  * Extract information from the screen using AI
package/sdk.js CHANGED
@@ -430,8 +430,9 @@ class Element {
430
430
  /**
431
431
  * Find the element on screen
432
432
  * @param {string} [newDescription] - Optional new description to search for
433
- * @param {Object} [options] - Optional options object with cacheThreshold, cacheKey, and/or timeout
433
+ * @param {Object} [options] - Optional options object with cache thresholds, cacheKey, and/or timeout
434
434
  * @param {number} [options.timeout] - Max time in ms to poll for element (polls every 5 seconds)
435
+ * @param {Object} [options.cache] - Cache configuration { thresholds: { screen, element } }
435
436
  * @returns {Promise<Element>} This element instance
436
437
  */
437
438
  async find(newDescription, options) {
@@ -468,10 +469,12 @@ class Element {
468
469
  this._screenshot = screenshot;
469
470
  }
470
471
 
471
- // Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold
472
+ // Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold/cache
472
473
  let cacheKey = null;
473
474
  let cacheThreshold = null;
475
+ let perCommandThresholds = null; // Per-command { screen, element } override
474
476
  let zoom = false; // Default to disabled, enable with zoom: true
477
+ let perCommandAi = null; // Per-command AI config override
475
478
 
476
479
  if (typeof options === "number") {
477
480
  // Legacy: options is just a number threshold
@@ -482,6 +485,10 @@ class Element {
482
485
  cacheThreshold = options.cacheThreshold ?? null;
483
486
  // zoom defaults to false unless explicitly set to true
484
487
  zoom = options.zoom === true;
488
+ // Per-command cache thresholds: { cache: { thresholds: { screen: 0.1, element: 0.2 } } }
489
+ if (typeof options.cache === "object" && options.cache?.thresholds) {
490
+ perCommandThresholds = options.cache.thresholds;
491
+ }
485
492
  }
486
493
 
487
494
  // Use default cacheKey from SDK constructor if not provided in find() options
@@ -499,19 +506,25 @@ class Element {
499
506
  // - If cacheKey is provided, enable cache with threshold
500
507
  // - If no cacheKey, disable cache
501
508
  let threshold;
509
+ let elementSimilarity;
502
510
  if (this.sdk._cacheExplicitlyDisabled) {
503
511
  // Cache explicitly disabled via cache: false option or TD_NO_CACHE env
504
512
  threshold = -1;
513
+ elementSimilarity = -1;
505
514
  cacheKey = null; // Clear any cacheKey to ensure cache is truly disabled
506
515
  } else if (cacheKey) {
507
516
  // cacheKey provided - enable cache with threshold
508
- threshold = cacheThreshold ?? this.sdk.cacheThresholds?.find ?? 0.01;
517
+ // Per-command thresholds > legacy cacheThreshold > global config
518
+ threshold = perCommandThresholds?.screen ?? cacheThreshold ?? this.sdk.cacheConfig?.thresholds?.find?.screen ?? 0.01;
519
+ elementSimilarity = perCommandThresholds?.element ?? this.sdk.cacheConfig?.thresholds?.find?.element ?? 0.8;
509
520
  } else if (cacheThreshold !== null) {
510
521
  // Explicit threshold provided without cacheKey
511
- threshold = cacheThreshold;
522
+ threshold = perCommandThresholds?.screen ?? cacheThreshold;
523
+ elementSimilarity = perCommandThresholds?.element ?? this.sdk.cacheConfig?.thresholds?.find?.element ?? 0.8;
512
524
  } else {
513
525
  // No cacheKey, no explicit threshold - disable cache
514
526
  threshold = -1;
527
+ elementSimilarity = -1;
515
528
  }
516
529
 
517
530
  // Store the threshold for debugging
@@ -536,10 +549,16 @@ class Element {
536
549
  element: description,
537
550
  image: screenshot,
538
551
  threshold: threshold,
552
+ elementSimilarity: elementSimilarity,
539
553
  cacheKey: cacheKey,
540
554
  os: this.sdk.os,
541
555
  resolution: this.sdk.resolution,
542
556
  zoom: zoom,
557
+ ai: {
558
+ ...this.sdk.aiConfig,
559
+ ...(perCommandAi || {}),
560
+ top: { ...this.sdk.aiConfig?.top, ...(perCommandAi?.top || {}) },
561
+ },
543
562
  });
544
563
 
545
564
  const duration = Date.now() - startTime;
@@ -736,6 +755,9 @@ class Element {
736
755
  cacheHit: debugInfo.cacheHit,
737
756
  selectorId: this._response?.selector,
738
757
  consoleUrl: consoleUrl,
758
+ validated: response.validated ?? null,
759
+ validationConfidence: response.validationConfidence ?? null,
760
+ coordsUpdated: response.coordsUpdated ?? null,
739
761
  };
740
762
  if (!debugInfo.cacheHit) {
741
763
  meta.confidence = debugInfo.confidence;
@@ -1441,15 +1463,49 @@ class TestDriverSDK {
1441
1463
  findAll: -1,
1442
1464
  assert: -1,
1443
1465
  };
1466
+ this.cacheConfig = {
1467
+ enabled: false,
1468
+ thresholds: {
1469
+ find: { screen: -1, element: -1 },
1470
+ assert: -1,
1471
+ },
1472
+ };
1444
1473
  } else {
1445
- // Cache enabled by default when cacheKey is provided
1474
+ // Support cache object format: { cache: { thresholds: { find: { screen: 0.01, element: 0.8 }, assert: 0.05 } } }
1475
+ const cacheOpts = typeof options.cache === "object" ? options.cache : {};
1476
+ const thresholds = cacheOpts.thresholds || {};
1477
+ const findThresholds = typeof thresholds.find === "object" ? thresholds.find : {};
1478
+
1479
+ this.cacheConfig = {
1480
+ enabled: cacheOpts.enabled !== false,
1481
+ thresholds: {
1482
+ find: {
1483
+ screen: findThresholds.screen ?? 0.01, // Default: 1% pixel diff allowed
1484
+ element: findThresholds.element ?? 0.8, // Default: 80% OpenCV correlation
1485
+ },
1486
+ assert: thresholds.assert ?? 0.05, // Default: 5% pixel diff for assertions
1487
+ },
1488
+ };
1489
+
1490
+ // Legacy cacheThresholds - keep for backwards compatibility
1446
1491
  this.cacheThresholds = {
1447
- find: options.cacheThreshold?.find ?? 0.01, // Default: 1% threshold
1448
- findAll: options.cacheThreshold?.findAll ?? 0.01,
1449
- assert: options.cacheThreshold?.assert ?? 0.05, // Default: 5% threshold for assertions
1492
+ find: options.cacheThreshold?.find ?? this.cacheConfig.thresholds.find.screen,
1493
+ findAll: options.cacheThreshold?.findAll ?? this.cacheConfig.thresholds.find.screen,
1494
+ assert: options.cacheThreshold?.assert ?? this.cacheConfig.thresholds.assert,
1450
1495
  };
1451
1496
  }
1452
1497
 
1498
+ // AI sampling configuration
1499
+ // Supports: { ai: { temperature: 0, top: { p: 1, k: 0 } } }
1500
+ // Can be overridden per find() or assert() call
1501
+ this.aiConfig = typeof options.ai === "object" ? {
1502
+ temperature: options.ai.temperature,
1503
+ top: {
1504
+ p: options.ai.top?.p,
1505
+ k: options.ai.top?.k,
1506
+ },
1507
+ } : {};
1508
+
1453
1509
  // Redraw configuration
1454
1510
  // Supports both:
1455
1511
  // - redraw: { enabled: true, diffThreshold: 0.1, screenRedraw: true, networkMonitor: true }
@@ -1656,26 +1712,6 @@ class TestDriverSDK {
1656
1712
  guest = false,
1657
1713
  } = options;
1658
1714
 
1659
- // If dashcam is enabled, add web logs for all websites
1660
- // Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
1661
- if (this.dashcamEnabled) {
1662
- // get the domain from the url for more specific logging, e.g. "Web Logs - example.com"
1663
- let domain = url;
1664
- let protocol = "https:";
1665
- try {
1666
- const urlObj = new URL(url);
1667
- domain = urlObj.hostname;
1668
- protocol = urlObj.protocol;
1669
- } catch (err) {
1670
- // If URL parsing fails, fall back to using the full URL as the domain
1671
- }
1672
-
1673
- // the pattern should be protocol://domain* to match all subpages of the domain
1674
- const webLogPattern = `${protocol}//${domain}*`;
1675
-
1676
- await this.dashcam.addWebLog(webLogPattern, "Web Logs");
1677
- }
1678
-
1679
1715
  // Set up Chrome profile with preferences
1680
1716
  const shell = this.os === "windows" ? "pwsh" : "sh";
1681
1717
  const userDataDir =
@@ -1930,12 +1966,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
1930
1966
  );
1931
1967
  }
1932
1968
 
1933
- // If dashcam is enabled, add web logs for all websites
1934
- // Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
1935
- if (this.dashcamEnabled) {
1936
- await this.dashcam.addWebLog("**", "Web Logs");
1937
- }
1938
-
1939
1969
  // Set up Chrome profile with preferences
1940
1970
  const userDataDir =
1941
1971
  this.os === "windows"
@@ -2056,12 +2086,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
2056
2086
 
2057
2087
  const shell = this.os === "windows" ? "pwsh" : "sh";
2058
2088
 
2059
- // If dashcam is enabled, add web logs for all websites
2060
- // Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
2061
- if (this.dashcamEnabled) {
2062
- await this.dashcam.addWebLog("**", "Web Logs");
2063
- }
2064
-
2065
2089
  // Install extensions if provided
2066
2090
  for (const extension of extensions) {
2067
2091
  console.log(`[provision.vscode] Installing extension: ${extension}`);
@@ -2132,12 +2156,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
2132
2156
 
2133
2157
  const shell = this.os === "windows" ? "pwsh" : "sh";
2134
2158
 
2135
- // If dashcam is enabled, add web logs for all websites
2136
- // Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
2137
- if (this.dashcamEnabled) {
2138
- await this.dashcam.addWebLog("**", "Web Logs");
2139
- }
2140
-
2141
2159
  // Determine download directory
2142
2160
  const downloadDir =
2143
2161
  this.os === "windows" ? "C:\\Users\\testdriver\\Downloads" : "/tmp";
@@ -2272,12 +2290,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
2272
2290
 
2273
2291
  const shell = this.os === "windows" ? "pwsh" : "sh";
2274
2292
 
2275
- // If dashcam is enabled, add web logs for all websites
2276
- // Note: File log and dashcam.start() are handled by the connection promise in hooks.mjs
2277
- if (this.dashcamEnabled) {
2278
- await this.dashcam.addWebLog("**", "Web Logs");
2279
- }
2280
-
2281
2293
  const argsString = args.join(" ");
2282
2294
 
2283
2295
  if (this.os === "windows") {
@@ -2791,7 +2803,7 @@ CAPTCHA_SOLVER_EOF`,
2791
2803
  * Automatically locates the element and returns it
2792
2804
  *
2793
2805
  * @param {string} description - Description of the element to find
2794
- * @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cacheThreshold}
2806
+ * @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cache: { thresholds: { screen, element } }}
2795
2807
  * @returns {Promise<Element> & ChainableElement} Element instance that has been located, with chainable methods
2796
2808
  *
2797
2809
  * @example
@@ -2880,7 +2892,7 @@ CAPTCHA_SOLVER_EOF`,
2880
2892
  * Automatically locates all matching elements and returns them as an array
2881
2893
  *
2882
2894
  * @param {string} description - Description of the elements to find
2883
- * @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cacheThreshold}
2895
+ * @param {number | Object} [options] - Cache options: number for threshold, or object with {cacheKey, cache: { thresholds: { screen } }}
2884
2896
  * @returns {Promise<Element[]>} Array of Element instances that have been located
2885
2897
  *
2886
2898
  * @example
@@ -2936,9 +2948,10 @@ CAPTCHA_SOLVER_EOF`,
2936
2948
  try {
2937
2949
  const screenshot = await this.system.captureScreenBase64();
2938
2950
 
2939
- // Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold
2951
+ // Handle options - can be a number (cacheThreshold) or object with cacheKey/cacheThreshold/cache
2940
2952
  let cacheKey = null;
2941
2953
  let cacheThreshold = null;
2954
+ let perCommandThresholds = null; // Per-command { screen } override (findAll has no element threshold)
2942
2955
 
2943
2956
  if (typeof options === "number") {
2944
2957
  // Legacy: options is just a number threshold
@@ -2947,6 +2960,10 @@ CAPTCHA_SOLVER_EOF`,
2947
2960
  // New: options is an object with cacheKey and/or cacheThreshold
2948
2961
  cacheKey = options.cacheKey || null;
2949
2962
  cacheThreshold = options.cacheThreshold ?? null;
2963
+ // Per-command cache thresholds: { cache: { thresholds: { screen: 0.1 } } }
2964
+ if (typeof options.cache === "object" && options.cache?.thresholds) {
2965
+ perCommandThresholds = options.cache.thresholds;
2966
+ }
2950
2967
  }
2951
2968
 
2952
2969
  // Use default cacheKey from SDK constructor if not provided in findAll() options
@@ -2969,11 +2986,11 @@ CAPTCHA_SOLVER_EOF`,
2969
2986
  threshold = -1;
2970
2987
  cacheKey = null; // Clear any cacheKey to ensure cache is truly disabled
2971
2988
  } else if (cacheKey) {
2972
- // cacheKey provided - enable cache with threshold
2973
- threshold = cacheThreshold ?? this.cacheThresholds?.findAll ?? 0.01;
2989
+ // cacheKey provided - enable cache with threshold (findAll only uses screen, no element)
2990
+ threshold = perCommandThresholds?.screen ?? cacheThreshold ?? this.cacheConfig?.thresholds?.find?.screen ?? 0.01;
2974
2991
  } else if (cacheThreshold !== null) {
2975
2992
  // Explicit threshold provided without cacheKey
2976
- threshold = cacheThreshold;
2993
+ threshold = perCommandThresholds?.screen ?? cacheThreshold;
2977
2994
  } else {
2978
2995
  // No cacheKey, no explicit threshold - disable cache
2979
2996
  threshold = -1;
@@ -2994,7 +3011,7 @@ CAPTCHA_SOLVER_EOF`,
2994
3011
  }
2995
3012
 
2996
3013
  const response = await this.apiClient.req(
2997
- "/api/v7.0.0/testdriver-agent/testdriver-find-all",
3014
+ "/api/v7.0.0/testdriver/find-all",
2998
3015
  {
2999
3016
  session: this.getSessionId(),
3000
3017
  element: description,
@@ -3010,7 +3027,7 @@ CAPTCHA_SOLVER_EOF`,
3010
3027
 
3011
3028
  if (response && response.elements && response.elements.length > 0) {
3012
3029
  // Single log at the end - found elements
3013
- const formattedMessage = formatter.formatFindAllSingleLine(
3030
+ const formattedMessage = formatter.formatElementsFound(
3014
3031
  description,
3015
3032
  response.elements.length,
3016
3033
  {
@@ -3093,7 +3110,7 @@ CAPTCHA_SOLVER_EOF`,
3093
3110
  const duration = Date.now() - startTime;
3094
3111
 
3095
3112
  // Single log at the end - no elements found
3096
- const formattedMessage = formatter.formatFindAllSingleLine(
3113
+ const formattedMessage = formatter.formatElementsFound(
3097
3114
  description,
3098
3115
  0,
3099
3116
  {
@@ -3139,7 +3156,7 @@ CAPTCHA_SOLVER_EOF`,
3139
3156
  const duration = Date.now() - startTime;
3140
3157
 
3141
3158
  // Single log at the end - error
3142
- const formattedMessage = formatter.formatFindAllSingleLine(
3159
+ const formattedMessage = formatter.formatElementsFound(
3143
3160
  description,
3144
3161
  0,
3145
3162
  {
@@ -3334,16 +3351,30 @@ CAPTCHA_SOLVER_EOF`,
3334
3351
  let result;
3335
3352
  // Special handling for assert to inject SDK options (cacheKey, os, resolution, threshold)
3336
3353
  // similar to how find() handles these in the Element class
3354
+ // Note: assert does NOT use elementSimilarity (template matching not relevant for assertions)
3337
3355
  if (commandName === 'assert') {
3338
3356
  const assertion = args[0];
3339
3357
  const userOptions = args[1] || {};
3340
3358
 
3359
+ // Support per-command cache threshold override: { cache: { threshold: 0.05 } }
3360
+ const perCommandThreshold = typeof userOptions.cache === "object"
3361
+ ? userOptions.cache.threshold
3362
+ : undefined;
3363
+
3341
3364
  // Merge SDK defaults with user options (user options take precedence)
3342
3365
  const mergedOptions = {
3343
3366
  cacheKey: userOptions.cacheKey ?? sdk.options.cacheKey,
3344
3367
  os: userOptions.os ?? sdk.os,
3345
3368
  resolution: userOptions.resolution ?? sdk.resolution,
3346
- threshold: userOptions.threshold !== undefined ? userOptions.threshold : (sdk.cacheThresholds?.assert ?? -1),
3369
+ threshold: perCommandThreshold ?? userOptions.threshold ?? (sdk.cacheConfig?.thresholds?.assert ?? sdk.cacheThresholds?.assert ?? 0.05),
3370
+ ai: {
3371
+ ...sdk.aiConfig,
3372
+ ...(typeof userOptions.ai === "object" ? userOptions.ai : {}),
3373
+ top: {
3374
+ ...sdk.aiConfig?.top,
3375
+ ...(typeof userOptions.ai === "object" ? userOptions.ai?.top : {}),
3376
+ },
3377
+ },
3347
3378
  };
3348
3379
 
3349
3380
  // Note: commands.assert takes (assertion, options), shouldThrow is determined internally
@@ -3451,74 +3482,70 @@ CAPTCHA_SOLVER_EOF`,
3451
3482
  }
3452
3483
 
3453
3484
  /**
3454
- * Extract all visible text from the current screen using OCR (Tesseract)
3455
- * Returns structured data with text content, bounding boxes, and confidence scores
3485
+ * Parse the current screen using OmniParser v2 to detect all UI elements
3486
+ * Returns structured data with element types, bounding boxes, and content
3487
+ * Requires enterprise or self-hosted plan.
3456
3488
  *
3457
- * @returns {Promise<OCRResult>} OCR extraction result
3489
+ * @returns {Promise<ParseResult>} Parsed screen elements
3458
3490
  *
3459
- * @typedef {Object} OCRResult
3460
- * @property {OCRWord[]} words - Array of words with positions and confidence
3461
- * @property {string} fullText - All extracted text concatenated
3462
- * @property {number} confidence - Overall OCR confidence (0-100)
3491
+ * @typedef {Object} ParseResult
3492
+ * @property {ParsedElement[]} elements - Array of detected UI elements
3493
+ * @property {string} annotatedImageUrl - URL of the annotated screenshot
3463
3494
  * @property {number} imageWidth - Width of the analyzed image
3464
3495
  * @property {number} imageHeight - Height of the analyzed image
3465
3496
  *
3466
- * @typedef {Object} OCRWord
3467
- * @property {string} content - The text content of the word
3468
- * @property {number} confidence - Confidence score (0-100)
3469
- * @property {Object} bbox - Bounding box coordinates
3497
+ * @typedef {Object} ParsedElement
3498
+ * @property {number} index - Element index
3499
+ * @property {string} type - Element type (e.g. "text", "icon", "button")
3500
+ * @property {string} content - Text content or description
3501
+ * @property {string} interactivity - Interactivity level (e.g. "clickable", "non-interactive")
3502
+ * @property {Object} bbox - Bounding box in pixel coordinates
3470
3503
  * @property {number} bbox.x0 - Left edge X coordinate
3471
3504
  * @property {number} bbox.y0 - Top edge Y coordinate
3472
3505
  * @property {number} bbox.x1 - Right edge X coordinate
3473
3506
  * @property {number} bbox.y1 - Bottom edge Y coordinate
3507
+ * @property {Object} boundingBox - Bounding box as {left, top, width, height}
3508
+ * @property {number} boundingBox.left - Left position
3509
+ * @property {number} boundingBox.top - Top position
3510
+ * @property {number} boundingBox.width - Element width
3511
+ * @property {number} boundingBox.height - Element height
3474
3512
  *
3475
3513
  * @example
3476
- * // Get all text on screen
3477
- * const result = await testdriver.ocr();
3478
- * console.log(result.fullText);
3479
- * // "Welcome to TestDriver Sign In Email Password Submit"
3480
- *
3481
- * @example
3482
- * // Find words matching a pattern
3483
- * const result = await testdriver.ocr();
3484
- * const buttons = result.words.filter(w =>
3485
- * w.content.toLowerCase().includes('button')
3486
- * );
3514
+ * // Get all elements on screen
3515
+ * const result = await testdriver.parse();
3516
+ * console.log(`Found ${result.elements.length} elements`);
3487
3517
  *
3488
3518
  * @example
3489
- * // Get word positions for clicking
3490
- * const result = await testdriver.ocr();
3491
- * const submitWord = result.words.find(w => w.content === 'Submit');
3492
- * if (submitWord) {
3493
- * // Calculate center of the word
3494
- * const x = (submitWord.bbox.x0 + submitWord.bbox.x1) / 2;
3495
- * const y = (submitWord.bbox.y0 + submitWord.bbox.y1) / 2;
3496
- * await testdriver.click({ x, y });
3497
- * }
3519
+ * // Find clickable elements
3520
+ * const result = await testdriver.parse();
3521
+ * const clickable = result.elements.filter(e => e.interactivity === 'clickable');
3498
3522
  *
3499
3523
  * @example
3500
- * // Check if specific text exists on screen
3501
- * const result = await testdriver.ocr();
3502
- * const hasError = result.words.some(w =>
3503
- * w.content.toLowerCase().includes('error')
3504
- * );
3524
+ * // Find text content
3525
+ * const result = await testdriver.parse();
3526
+ * const textElements = result.elements.filter(e => e.type === 'text');
3527
+ * textElements.forEach(e => console.log(e.content));
3505
3528
  */
3506
- async ocr() {
3529
+ async parse() {
3507
3530
  this._ensureConnected();
3508
3531
 
3509
3532
  const { events } = require("./agent/events.js");
3510
- this.emitter.emit(events.log.log, "🔍 Running OCR text extraction...");
3533
+ this.emitter.emit(events.log.log, "🔍 Running OmniParser screen analysis...");
3511
3534
 
3512
3535
  const screenshot = await this.system.captureScreenBase64();
3513
3536
 
3514
- const response = await this.apiClient.req("ocr", {
3537
+ const response = await this.apiClient.req("parse", {
3515
3538
  session: this.getSessionId(),
3516
3539
  image: screenshot,
3517
3540
  });
3518
3541
 
3542
+ if (response.error) {
3543
+ throw new Error(response.error);
3544
+ }
3545
+
3519
3546
  this.emitter.emit(
3520
3547
  events.log.log,
3521
- `✅ OCR complete: ${response.words?.length || 0} words extracted`,
3548
+ `✅ Parse complete: ${response.elements?.length || 0} elements detected`,
3522
3549
  );
3523
3550
 
3524
3551
  return response;