testdriverai 5.3.18 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/agent.js CHANGED
@@ -526,6 +526,8 @@ const loadYML = async (file) => {
526
526
  process.env["TD_INTERPOLATION_VARS"] || "{}",
527
527
  );
528
528
 
529
+ yml = await parser.validateYAML(yml);
530
+
529
531
  // Inject environment variables into any ${VAR} strings
530
532
  yml = parser.interpolate(yml, process.env);
531
533
 
@@ -1241,6 +1243,8 @@ const start = async () => {
1241
1243
  if (thisCommand !== "init" && thisCommand !== "upload-secrets") {
1242
1244
  logger.info(chalk.dim(`Working on ${thisFile}`));
1243
1245
 
1246
+ loadYML(thisFile)
1247
+
1244
1248
  if (!config.TD_VM) {
1245
1249
  logger.info(
1246
1250
  chalk.red("Warning! ") +
package/lib/parser.js CHANGED
@@ -1,9 +1,27 @@
1
1
  // parses markdown content to find code blocks, and then extracts yaml from those code blocks
2
2
  const Parser = require("markdown-parser");
3
3
  const yaml = require("js-yaml");
4
+ const Ajv = require("ajv");
4
5
 
5
6
  let parser = new Parser();
6
7
 
8
+ const chalk = require('chalk');
9
+
10
+ function formatAjvError(error) {
11
+
12
+ return [
13
+ chalk.bold.red('❌ Validation Error'),
14
+ `${chalk.bold('Path:')} ${chalk.yellow(error.instancePath)}`,
15
+ `${chalk.bold('Schema:')} ${chalk.cyan(error.schemaPath)}`,
16
+ `${chalk.bold('Keyword:')} ${chalk.magenta(error.keyword)}`,
17
+ error.params?.missingProperty
18
+ ? `${chalk.bold('Missing:')} ${chalk.red(error.params.missingProperty)}`
19
+ : '',
20
+ `${chalk.bold('Message:')} ${chalk.white(error.message)}`,
21
+ `\n`
22
+ ].filter(Boolean).join('\n');
23
+ }
24
+
7
25
  // use markdown parser to find code blocks within AI response
8
26
  const findCodeBlocks = async function (markdownContent) {
9
27
  return new Promise((resolve, reject) => {
@@ -77,6 +95,22 @@ const parseYAML = async function (inputYaml) {
77
95
  return doc;
78
96
  };
79
97
 
98
+ // validate yaml using schema.json in root
99
+ let schema = require("../schema.json");
100
+ const validateYAML = async function (yaml) {
101
+
102
+ let ajv = new Ajv({allowUnionTypes: true});
103
+ let validate = ajv.compile(schema);
104
+ let valid = validate(await parseYAML(yaml));
105
+
106
+ if (!valid) {
107
+ validate.errors.forEach(err => console.log(formatAjvError(err)));
108
+ throw new Error("Invalid YAML");
109
+ }
110
+
111
+ return yaml;
112
+ };
113
+
80
114
  // Replace ${VAR} with the value from the vars object
81
115
  // Will skip variables that are not in the vars object
82
116
  // Will skip escaped variables like \${VAR}
@@ -96,6 +130,7 @@ module.exports = {
96
130
  findGenerativePrompts,
97
131
  getYAMLFromCodeBlock,
98
132
  interpolate,
133
+ validateYAML,
99
134
  getCommands: async function (codeBlock) {
100
135
  const yml = getYAMLFromCodeBlock(codeBlock);
101
136
  let yamlArray = await parseYAML(yml);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testdriverai",
3
- "version": "5.3.18",
3
+ "version": "5.4.0",
4
4
  "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
5
5
  "main": "index.js",
6
6
  "bin": {
@@ -19,6 +19,7 @@
19
19
  "dependencies": {
20
20
  "@e2b/desktop": "^1.6.0",
21
21
  "@electerm/strip-ansi": "^1.0.0",
22
+ "ajv": "^8.17.1",
22
23
  "arktype": "^2.1.19",
23
24
  "axios": "^1.7.7",
24
25
  "chalk": "^4.1.2",
package/schema.json ADDED
@@ -0,0 +1,588 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "properties": {
5
+ "version": {
6
+ "type": "string",
7
+ "description": "The version of the TestDriver framework."
8
+ },
9
+ "session": {
10
+ "type": "string",
11
+ "description": "A unique identifier for the test session."
12
+ },
13
+ "steps": {
14
+ "type": "array",
15
+ "description": "An array of test steps.",
16
+ "items": {
17
+ "type": "object",
18
+ "properties": {
19
+ "prompt": {
20
+ "type": "string",
21
+ "description": "A description of the step's purpose."
22
+ },
23
+ "commands": {
24
+ "type": "array",
25
+ "description": "An array of commands to execute in this step.",
26
+ "items": {
27
+ "type": "object",
28
+ "properties": {
29
+ "command": {
30
+ "type": "string",
31
+ "description": "The command to execute."
32
+ }
33
+ },
34
+ "required": [
35
+ "command"
36
+ ],
37
+ "allOf": [
38
+ {
39
+ "if": {
40
+ "properties": {
41
+ "command": {
42
+ "const": "type"
43
+ }
44
+ }
45
+ },
46
+ "then": {
47
+ "properties": {
48
+ "text": {
49
+ "type": ["string", "integer"]
50
+ },
51
+ "delay": {
52
+ "type": "integer"
53
+ }
54
+ },
55
+ "required": [
56
+ "text"
57
+ ]
58
+ }
59
+ },
60
+ {
61
+ "if": {
62
+ "properties": {
63
+ "command": {
64
+ "const": "press-keys"
65
+ }
66
+ }
67
+ },
68
+ "then": {
69
+ "properties": {
70
+ "keys": {
71
+ "type": "array",
72
+ "items": {
73
+ "type": "string"
74
+ },
75
+ "enum": [
76
+ [
77
+ "backspace", "delete", "enter", "tab", "escape", "up", "down", "right", "left",
78
+ "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
79
+ "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19",
80
+ "f20", "f21", "f22", "f23", "f24", "capslock", "command", "alt", "right_alt",
81
+ "control", "left_control", "right_control", "shift", "right_shift", "space",
82
+ "printscreen", "insert", "menu", "audio_mute", "audio_vol_down", "audio_vol_up",
83
+ "audio_play", "audio_stop", "audio_pause", "audio_prev", "audio_next",
84
+ "audio_rewind", "audio_forward", "audio_repeat", "audio_random", "numpad_lock",
85
+ "numpad_0", "numpad_1", "numpad_2", "numpad_3", "numpad_4", "numpad_5",
86
+ "numpad_6", "numpad_7", "numpad_8", "numpad_9", "numpad_+", "numpad_-",
87
+ "numpad_*", "numpad_/", "numpad_.", "lights_mon_up", "lights_mon_down",
88
+ "lights_kbd_toggle", "lights_kbd_up", "lights_kbd_down", "a", "b", "c", "d", "e",
89
+ "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u",
90
+ "v", "w", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
91
+ ]
92
+ ]
93
+ }
94
+ },
95
+ "required": [
96
+ "keys"
97
+ ]
98
+ }
99
+ },
100
+ {
101
+ "if": {
102
+ "properties": {
103
+ "command": {
104
+ "const": "scroll"
105
+ }
106
+ }
107
+ },
108
+ "then": {
109
+ "properties": {
110
+ "direction": {
111
+ "type": "string",
112
+ "enum": [
113
+ "up",
114
+ "down",
115
+ "left",
116
+ "right"
117
+ ]
118
+ },
119
+ "amount": {
120
+ "type": "integer"
121
+ },
122
+ "method": {
123
+ "type": "string",
124
+ "enum": [
125
+ "keyboard",
126
+ "mouse"
127
+ ]
128
+ }
129
+ },
130
+ "required": [
131
+ "direction"
132
+ ]
133
+ }
134
+ },
135
+ {
136
+ "if": {
137
+ "properties": {
138
+ "command": {
139
+ "const": "wait"
140
+ }
141
+ }
142
+ },
143
+ "then": {
144
+ "properties": {
145
+ "timeout": {
146
+ "type": "integer"
147
+ }
148
+ },
149
+ "required": [
150
+ "timeout"
151
+ ]
152
+ }
153
+ },
154
+ {
155
+ "if": {
156
+ "properties": {
157
+ "command": {
158
+ "const": "click"
159
+ }
160
+ }
161
+ },
162
+ "then": {
163
+ "properties": {
164
+ "x": {
165
+ "type": "integer"
166
+ },
167
+ "y": {
168
+ "type": "integer"
169
+ },
170
+ "action": {
171
+ "type": "string",
172
+ "enum": [
173
+ "click",
174
+ "right-click",
175
+ "double-click",
176
+ "hover",
177
+ "drag-start",
178
+ "drag-end"
179
+ ]
180
+ }
181
+ },
182
+ "required": [
183
+ "x",
184
+ "y",
185
+ "action"
186
+ ]
187
+ }
188
+ },
189
+ {
190
+ "if": {
191
+ "properties": {
192
+ "command": {
193
+ "const": "hover"
194
+ }
195
+ }
196
+ },
197
+ "then": {
198
+ "properties": {
199
+ "x": {
200
+ "type": "integer"
201
+ },
202
+ "y": {
203
+ "type": "integer"
204
+ }
205
+ },
206
+ "required": [
207
+ "x",
208
+ "y"
209
+ ]
210
+ }
211
+ },
212
+ {
213
+ "if": {
214
+ "properties": {
215
+ "command": {
216
+ "const": "drag"
217
+ }
218
+ }
219
+ },
220
+ "then": {
221
+ "properties": {
222
+ "x": {
223
+ "type": "integer"
224
+ },
225
+ "y": {
226
+ "type": "integer"
227
+ }
228
+ },
229
+ "required": [
230
+ "x",
231
+ "y"
232
+ ]
233
+ }
234
+ },
235
+ {
236
+ "if": {
237
+ "properties": {
238
+ "command": {
239
+ "const": "hover-text"
240
+ }
241
+ }
242
+ },
243
+ "then": {
244
+ "properties": {
245
+ "text": {
246
+ "type": "string"
247
+ },
248
+ "description": {
249
+ "type": "string"
250
+ },
251
+ "action": {
252
+ "type": "string",
253
+ "enum": [
254
+ "click",
255
+ "right-click",
256
+ "double-click",
257
+ "hover",
258
+ "drag-start",
259
+ "drag-end"
260
+ ]
261
+ },
262
+ "method": {
263
+ "type": "string",
264
+ "enum": [
265
+ "ai",
266
+ "turbo"
267
+ ]
268
+ }
269
+ },
270
+ "required": [
271
+ "text",
272
+ "description",
273
+ "action"
274
+ ]
275
+ }
276
+ },
277
+ {
278
+ "if": {
279
+ "properties": {
280
+ "command": {
281
+ "const": "hover-image"
282
+ }
283
+ }
284
+ },
285
+ "then": {
286
+ "properties": {
287
+ "description": {
288
+ "type": "string"
289
+ },
290
+ "action": {
291
+ "type": "string",
292
+ "enum": [
293
+ "click",
294
+ "right-click",
295
+ "double-click",
296
+ "hover",
297
+ "drag-start",
298
+ "drag-end"
299
+ ]
300
+ }
301
+ },
302
+ "required": [
303
+ "description",
304
+ "action"
305
+ ]
306
+ }
307
+ },
308
+ {
309
+ "if": {
310
+ "properties": {
311
+ "command": {
312
+ "const": "match-image"
313
+ }
314
+ }
315
+ },
316
+ "then": {
317
+ "properties": {
318
+ "path": {
319
+ "type": "string"
320
+ },
321
+ "action": {
322
+ "type": "string",
323
+ "enum": [
324
+ "click",
325
+ "right-click",
326
+ "double-click",
327
+ "hover",
328
+ "drag-start",
329
+ "drag-end"
330
+ ]
331
+ }
332
+ },
333
+ "required": [
334
+ "path",
335
+ "action"
336
+ ]
337
+ }
338
+ },
339
+ {
340
+ "if": {
341
+ "properties": {
342
+ "command": {
343
+ "const": "wait-for-image"
344
+ }
345
+ }
346
+ },
347
+ "then": {
348
+ "properties": {
349
+ "description": {
350
+ "type": "string"
351
+ },
352
+ "timeout": {
353
+ "type": "integer"
354
+ }
355
+ },
356
+ "required": [
357
+ "description"
358
+ ]
359
+ }
360
+ },
361
+ {
362
+ "if": {
363
+ "properties": {
364
+ "command": {
365
+ "const": "wait-for-text"
366
+ }
367
+ }
368
+ },
369
+ "then": {
370
+ "properties": {
371
+ "text": {
372
+ "type": "string"
373
+ },
374
+ "timeout": {
375
+ "type": "integer"
376
+ },
377
+ "method": {
378
+ "type": "string",
379
+ "enum": [
380
+ "ai",
381
+ "turbo"
382
+ ]
383
+ }
384
+ },
385
+ "required": [
386
+ "text"
387
+ ]
388
+ }
389
+ },
390
+ {
391
+ "if": {
392
+ "properties": {
393
+ "command": {
394
+ "const": "scroll-until-text"
395
+ }
396
+ }
397
+ },
398
+ "then": {
399
+ "properties": {
400
+ "text": {
401
+ "type": "string"
402
+ },
403
+ "direction": {
404
+ "type": "string",
405
+ "enum": [
406
+ "up",
407
+ "down",
408
+ "left",
409
+ "right"
410
+ ]
411
+ },
412
+ "distance": {
413
+ "type": "integer"
414
+ },
415
+ "method": {
416
+ "type": "string",
417
+ "enum": [
418
+ "ai",
419
+ "turbo"
420
+ ]
421
+ }
422
+ },
423
+ "required": [
424
+ "text",
425
+ "direction"
426
+ ]
427
+ }
428
+ },
429
+ {
430
+ "if": {
431
+ "properties": {
432
+ "command": {
433
+ "const": "scroll-until-image"
434
+ }
435
+ }
436
+ },
437
+ "then": {
438
+ "properties": {
439
+ "description": {
440
+ "type": "string"
441
+ },
442
+ "direction": {
443
+ "type": "string",
444
+ "enum": [
445
+ "up",
446
+ "down",
447
+ "left",
448
+ "right"
449
+ ]
450
+ },
451
+ "distance": {
452
+ "type": "integer"
453
+ }
454
+ },
455
+ "required": [
456
+ "description",
457
+ "direction"
458
+ ]
459
+ }
460
+ },
461
+ {
462
+ "if": {
463
+ "properties": {
464
+ "command": {
465
+ "const": "focus-application"
466
+ }
467
+ }
468
+ },
469
+ "then": {
470
+ "properties": {
471
+ "name": {
472
+ "type": "string"
473
+ }
474
+ },
475
+ "required": [
476
+ "name"
477
+ ]
478
+ }
479
+ },
480
+ {
481
+ "if": {
482
+ "properties": {
483
+ "command": {
484
+ "const": "remember"
485
+ }
486
+ }
487
+ },
488
+ "then": {
489
+ "properties": {
490
+ "description": {
491
+ "type": "string"
492
+ },
493
+ "value": {
494
+ "type": "string"
495
+ }
496
+ },
497
+ "required": [
498
+ "description",
499
+ "value"
500
+ ]
501
+ }
502
+ },
503
+ {
504
+ "if": {
505
+ "properties": {
506
+ "command": {
507
+ "const": "assert"
508
+ }
509
+ }
510
+ },
511
+ "then": {
512
+ "properties": {
513
+ "expect": {
514
+ "type": "string"
515
+ },
516
+ "async": {
517
+ "type": "boolean"
518
+ }
519
+ },
520
+ "required": [
521
+ "expect"
522
+ ]
523
+ }
524
+ },
525
+ {
526
+ "if": {
527
+ "properties": {
528
+ "command": {
529
+ "const": "exec"
530
+ }
531
+ }
532
+ },
533
+ "then": {
534
+ "properties": {
535
+ "lang": {
536
+ "type": "string",
537
+ "enum": ["js", "shell"]
538
+ },
539
+ "mac": {
540
+ "type": "string"
541
+ },
542
+ "windows": {
543
+ "type": "string"
544
+ },
545
+ "linux": {
546
+ "type": "string"
547
+ },
548
+ "silent": {
549
+ "type": "boolean"
550
+ },
551
+ "output": {
552
+ "type": "string"
553
+ }
554
+ },
555
+ "anyOf": [
556
+ {
557
+ "required": [
558
+ "mac"
559
+ ]
560
+ },
561
+ {
562
+ "required": [
563
+ "windows"
564
+ ]
565
+ },
566
+ {
567
+ "required": [
568
+ "linux"
569
+ ]
570
+ }
571
+ ]
572
+ }
573
+ }
574
+ ]
575
+ }
576
+ }
577
+ },
578
+ "required": [
579
+ "commands"
580
+ ]
581
+ }
582
+ }
583
+ },
584
+ "required": [
585
+ "version",
586
+ "steps"
587
+ ]
588
+ }
@@ -1,6 +1,13 @@
1
- version: 5.3.13
2
- session: 680126994a12078ae2a74ee5
1
+ version: 5.1.1
2
+ session: 67f00511acbd9ccac373edf7
3
3
  steps:
4
- - prompt: /try log in
5
- commands: []
6
- A
4
+ - prompt: launch chrome
5
+ commands:
6
+ - command: exec
7
+ lang: shell
8
+ linux: |
9
+ jumpapp google-chrome --disable-fre --no-default-browser-check --no-first-run "${TD_WEBSITE}" &
10
+ exit
11
+ - command: wait-for-text
12
+ text: "Google Chrome"
13
+ timeout: 30000