testdriverai 7.3.43 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/lib/commands.js +172 -232
- package/agent/lib/debugger-server.js +33 -0
- package/agent/lib/sandbox.js +9 -1
- package/agent/lib/sdk.js +67 -0
- package/examples/hover-image.test.mjs +11 -0
- package/lib/vitest/hooks.mjs +0 -52
- package/package.json +1 -1
package/agent/lib/commands.js
CHANGED
|
@@ -137,6 +137,54 @@ const createCommands = (
|
|
|
137
137
|
|
|
138
138
|
const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
|
|
139
139
|
|
|
140
|
+
/**
|
|
141
|
+
* Track an interaction via HTTP API (fire-and-forget)
|
|
142
|
+
* @param {Object} data - Interaction data
|
|
143
|
+
* @param {string} data.interactionType - Type of interaction (click, type, assert, etc.)
|
|
144
|
+
* @param {string} [data.prompt] - Description/prompt for the interaction
|
|
145
|
+
* @param {Object} [data.input] - Input data (varies by interaction type)
|
|
146
|
+
* @param {Object} [data.coordinates] - Coordinates {x, y}
|
|
147
|
+
* @param {number} data.timestamp - Absolute epoch timestamp
|
|
148
|
+
* @param {number} [data.duration] - Duration in ms
|
|
149
|
+
* @param {boolean} data.success - Whether the interaction succeeded
|
|
150
|
+
* @param {string} [data.error] - Error message if failed
|
|
151
|
+
* @param {boolean} [data.cacheHit] - Whether cache was used
|
|
152
|
+
* @param {string} [data.selector] - Selector ID
|
|
153
|
+
* @param {boolean} [data.selectorUsed] - Whether selector was used
|
|
154
|
+
* @param {number} [data.confidence] - AI confidence score
|
|
155
|
+
* @param {string} [data.reasoning] - AI reasoning
|
|
156
|
+
* @param {number} [data.similarity] - Cache similarity score
|
|
157
|
+
* @param {string} [data.screenshotUrl] - S3 key for screenshot
|
|
158
|
+
* @param {boolean} [data.isSecret] - Whether interaction contains sensitive data
|
|
159
|
+
*/
|
|
160
|
+
const trackInteraction = (data) => {
|
|
161
|
+
const sessionId = sessionInstance?.get();
|
|
162
|
+
if (!sessionId) return;
|
|
163
|
+
|
|
164
|
+
sdk.req("/api/v7.0.0/testdriver/interaction/track", {
|
|
165
|
+
session: sessionId,
|
|
166
|
+
type: data.interactionType,
|
|
167
|
+
coordinates: data.coordinates,
|
|
168
|
+
input: data.input,
|
|
169
|
+
prompt: data.prompt,
|
|
170
|
+
selectorUsed: data.selectorUsed,
|
|
171
|
+
selector: data.selector,
|
|
172
|
+
cacheHit: data.cacheHit,
|
|
173
|
+
status: "completed",
|
|
174
|
+
success: data.success,
|
|
175
|
+
error: data.error,
|
|
176
|
+
duration: data.duration,
|
|
177
|
+
timestamp: data.timestamp,
|
|
178
|
+
isSecret: data.isSecret,
|
|
179
|
+
confidence: data.confidence,
|
|
180
|
+
reasoning: data.reasoning,
|
|
181
|
+
similarity: data.similarity,
|
|
182
|
+
screenshotUrl: data.screenshotUrl,
|
|
183
|
+
}).catch((err) => {
|
|
184
|
+
console.warn(`Failed to track ${data.interactionType} interaction:`, err.message);
|
|
185
|
+
});
|
|
186
|
+
};
|
|
187
|
+
|
|
140
188
|
const findImageOnScreen = async (
|
|
141
189
|
relativePath,
|
|
142
190
|
haystack,
|
|
@@ -296,26 +344,19 @@ const createCommands = (
|
|
|
296
344
|
emitter.emit(events.log.narration, formatter.formatAssertResult(passed, responseText, assertDuration, cacheHit), true);
|
|
297
345
|
|
|
298
346
|
// Track interaction with success/failure (fire-and-forget)
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
reasoning: reasoning,
|
|
313
|
-
similarity: similarity,
|
|
314
|
-
screenshotUrl: response?.screenshotKey ?? null,
|
|
315
|
-
}).catch((err) => {
|
|
316
|
-
console.warn("Failed to track assert interaction:", err.message);
|
|
317
|
-
});
|
|
318
|
-
}
|
|
347
|
+
trackInteraction({
|
|
348
|
+
interactionType: "assert",
|
|
349
|
+
prompt: assertion,
|
|
350
|
+
timestamp: assertTimestamp,
|
|
351
|
+
duration: assertDuration,
|
|
352
|
+
success: passed,
|
|
353
|
+
error: passed ? undefined : responseText,
|
|
354
|
+
cacheHit: cacheHit,
|
|
355
|
+
confidence: confidence,
|
|
356
|
+
reasoning: reasoning,
|
|
357
|
+
similarity: similarity,
|
|
358
|
+
screenshotUrl: response?.screenshotKey ?? null,
|
|
359
|
+
});
|
|
319
360
|
|
|
320
361
|
if (passed) {
|
|
321
362
|
return true;
|
|
@@ -430,40 +471,26 @@ const createCommands = (
|
|
|
430
471
|
);
|
|
431
472
|
|
|
432
473
|
// Track interaction success (fire-and-forget)
|
|
433
|
-
const
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
duration: scrollDuration,
|
|
443
|
-
success: scrollSuccess,
|
|
444
|
-
error: scrollError,
|
|
445
|
-
}).catch((err) => {
|
|
446
|
-
console.warn("Failed to track scroll interaction:", err.message);
|
|
447
|
-
});
|
|
448
|
-
}
|
|
474
|
+
const scrollDuration = Date.now() - scrollStartTime;
|
|
475
|
+
trackInteraction({
|
|
476
|
+
interactionType: "scroll",
|
|
477
|
+
input: { direction, amount },
|
|
478
|
+
timestamp: scrollTimestamp,
|
|
479
|
+
duration: scrollDuration,
|
|
480
|
+
success: scrollSuccess,
|
|
481
|
+
error: scrollError,
|
|
482
|
+
});
|
|
449
483
|
} catch (error) {
|
|
450
484
|
// Track interaction failure (fire-and-forget)
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
duration: scrollDuration,
|
|
461
|
-
success: false,
|
|
462
|
-
error: error.message,
|
|
463
|
-
}).catch((err) => {
|
|
464
|
-
console.warn("Failed to track scroll interaction:", err.message);
|
|
465
|
-
});
|
|
466
|
-
}
|
|
485
|
+
const scrollDuration = Date.now() - scrollStartTime;
|
|
486
|
+
trackInteraction({
|
|
487
|
+
interactionType: "scroll",
|
|
488
|
+
input: { direction, amount },
|
|
489
|
+
timestamp: scrollTimestamp,
|
|
490
|
+
duration: scrollDuration,
|
|
491
|
+
success: false,
|
|
492
|
+
error: error.message,
|
|
493
|
+
});
|
|
467
494
|
throw error;
|
|
468
495
|
}
|
|
469
496
|
};
|
|
@@ -575,15 +602,12 @@ const createCommands = (
|
|
|
575
602
|
const actionDuration = actionEndTime - clickStartTime;
|
|
576
603
|
|
|
577
604
|
// Track interaction (fire-and-forget)
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
sandbox.send({
|
|
581
|
-
type: "trackInteraction",
|
|
605
|
+
if (elementData.prompt) {
|
|
606
|
+
trackInteraction({
|
|
582
607
|
interactionType: "click",
|
|
583
|
-
session: sessionId,
|
|
584
608
|
prompt: elementData.prompt,
|
|
585
609
|
input: { x, y, action },
|
|
586
|
-
timestamp: clickTimestamp,
|
|
610
|
+
timestamp: clickTimestamp,
|
|
587
611
|
duration: actionDuration,
|
|
588
612
|
success: true,
|
|
589
613
|
cacheHit: elementData.cacheHit,
|
|
@@ -593,8 +617,6 @@ const createCommands = (
|
|
|
593
617
|
reasoning: elementData.reasoning ?? null,
|
|
594
618
|
similarity: elementData.similarity ?? null,
|
|
595
619
|
screenshotUrl: elementData.screenshotUrl ?? null,
|
|
596
|
-
}).catch((err) => {
|
|
597
|
-
console.warn("Failed to track click interaction:", err.message);
|
|
598
620
|
});
|
|
599
621
|
}
|
|
600
622
|
|
|
@@ -627,16 +649,13 @@ const createCommands = (
|
|
|
627
649
|
return;
|
|
628
650
|
} catch (error) {
|
|
629
651
|
// Track interaction failure (fire-and-forget)
|
|
630
|
-
|
|
631
|
-
if (sessionId && elementData.prompt) {
|
|
652
|
+
if (elementData.prompt) {
|
|
632
653
|
const clickDuration = Date.now() - clickStartTime;
|
|
633
|
-
|
|
634
|
-
type: "trackInteraction",
|
|
654
|
+
trackInteraction({
|
|
635
655
|
interactionType: "click",
|
|
636
|
-
session: sessionId,
|
|
637
656
|
prompt: elementData.prompt,
|
|
638
657
|
input: { x, y, action },
|
|
639
|
-
timestamp: clickTimestamp,
|
|
658
|
+
timestamp: clickTimestamp,
|
|
640
659
|
duration: clickDuration,
|
|
641
660
|
success: false,
|
|
642
661
|
error: error.message,
|
|
@@ -646,8 +665,6 @@ const createCommands = (
|
|
|
646
665
|
confidence: elementData.confidence ?? null,
|
|
647
666
|
reasoning: elementData.reasoning ?? null,
|
|
648
667
|
similarity: elementData.similarity ?? null,
|
|
649
|
-
}).catch((err) => {
|
|
650
|
-
console.warn("Failed to track click interaction:", err.message);
|
|
651
668
|
});
|
|
652
669
|
}
|
|
653
670
|
throw error;
|
|
@@ -698,18 +715,15 @@ const createCommands = (
|
|
|
698
715
|
await sandbox.send({ type: "moveMouse", x, y, ...elementData });
|
|
699
716
|
|
|
700
717
|
// Track interaction (fire-and-forget)
|
|
701
|
-
const sessionId = sessionInstance?.get();
|
|
702
718
|
const actionEndTime = Date.now();
|
|
703
719
|
const actionDuration = actionEndTime - hoverStartTime;
|
|
704
720
|
|
|
705
|
-
if (
|
|
706
|
-
|
|
707
|
-
type: "trackInteraction",
|
|
721
|
+
if (elementData.prompt) {
|
|
722
|
+
trackInteraction({
|
|
708
723
|
interactionType: "hover",
|
|
709
|
-
session: sessionId,
|
|
710
724
|
prompt: elementData.prompt,
|
|
711
725
|
input: { x, y },
|
|
712
|
-
timestamp: hoverTimestamp,
|
|
726
|
+
timestamp: hoverTimestamp,
|
|
713
727
|
duration: actionDuration,
|
|
714
728
|
success: true,
|
|
715
729
|
cacheHit: elementData.cacheHit,
|
|
@@ -719,8 +733,6 @@ const createCommands = (
|
|
|
719
733
|
reasoning: elementData.reasoning ?? null,
|
|
720
734
|
similarity: elementData.similarity ?? null,
|
|
721
735
|
screenshotUrl: elementData.screenshotUrl ?? null,
|
|
722
|
-
}).catch((err) => {
|
|
723
|
-
console.warn("Failed to track hover interaction:", err.message);
|
|
724
736
|
});
|
|
725
737
|
}
|
|
726
738
|
|
|
@@ -741,16 +753,13 @@ const createCommands = (
|
|
|
741
753
|
return;
|
|
742
754
|
} catch (error) {
|
|
743
755
|
// Track interaction failure (fire-and-forget)
|
|
744
|
-
|
|
745
|
-
if (sessionId && elementData.prompt) {
|
|
756
|
+
if (elementData.prompt) {
|
|
746
757
|
const hoverDuration = Date.now() - hoverStartTime;
|
|
747
|
-
|
|
748
|
-
type: "trackInteraction",
|
|
758
|
+
trackInteraction({
|
|
749
759
|
interactionType: "hover",
|
|
750
|
-
session: sessionId,
|
|
751
760
|
prompt: elementData.prompt,
|
|
752
761
|
input: { x, y },
|
|
753
|
-
timestamp: hoverTimestamp,
|
|
762
|
+
timestamp: hoverTimestamp,
|
|
754
763
|
duration: hoverDuration,
|
|
755
764
|
success: false,
|
|
756
765
|
error: error.message,
|
|
@@ -761,8 +770,6 @@ const createCommands = (
|
|
|
761
770
|
reasoning: elementData.reasoning ?? null,
|
|
762
771
|
similarity: elementData.similarity ?? null,
|
|
763
772
|
screenshotUrl: elementData.screenshotUrl ?? null,
|
|
764
|
-
}).catch((err) => {
|
|
765
|
-
console.warn("Failed to track hover interaction:", err.message);
|
|
766
773
|
});
|
|
767
774
|
}
|
|
768
775
|
throw error;
|
|
@@ -963,23 +970,16 @@ const createCommands = (
|
|
|
963
970
|
emitter.emit(events.log.narration, formatter.formatTypeResult(text, secret, typeActionEndTime - typeStartTime), true);
|
|
964
971
|
|
|
965
972
|
// Track interaction (fire-and-forget)
|
|
966
|
-
const
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
duration: typeDuration,
|
|
977
|
-
success: true,
|
|
978
|
-
isSecret: secret, // Flag this interaction if it contains a secret
|
|
979
|
-
}).catch((err) => {
|
|
980
|
-
console.warn("Failed to track type interaction:", err.message);
|
|
981
|
-
});
|
|
982
|
-
}
|
|
973
|
+
const typeDuration = Date.now() - typeStartTime;
|
|
974
|
+
trackInteraction({
|
|
975
|
+
interactionType: "type",
|
|
976
|
+
// Store masked text if secret, otherwise store actual text
|
|
977
|
+
input: { text: secret ? "****" : text, delay },
|
|
978
|
+
timestamp: typeTimestamp,
|
|
979
|
+
duration: typeDuration,
|
|
980
|
+
success: true,
|
|
981
|
+
isSecret: secret, // Flag this interaction if it contains a secret
|
|
982
|
+
});
|
|
983
983
|
|
|
984
984
|
const redrawStartTime = Date.now();
|
|
985
985
|
await redraw.wait(5000, redrawOptions);
|
|
@@ -1036,21 +1036,14 @@ const createCommands = (
|
|
|
1036
1036
|
);
|
|
1037
1037
|
|
|
1038
1038
|
// Track interaction (fire-and-forget)
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
timestamp: pressKeysTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
|
|
1048
|
-
duration: pressKeysDuration,
|
|
1049
|
-
success: true,
|
|
1050
|
-
}).catch((err) => {
|
|
1051
|
-
console.warn("Failed to track pressKeys interaction:", err.message);
|
|
1052
|
-
});
|
|
1053
|
-
}
|
|
1039
|
+
const pressKeysDuration = Date.now() - pressKeysStartTime;
|
|
1040
|
+
trackInteraction({
|
|
1041
|
+
interactionType: "pressKeys",
|
|
1042
|
+
input: { keys },
|
|
1043
|
+
timestamp: pressKeysTimestamp,
|
|
1044
|
+
duration: pressKeysDuration,
|
|
1045
|
+
success: true,
|
|
1046
|
+
});
|
|
1054
1047
|
|
|
1055
1048
|
const redrawStartTime = Date.now();
|
|
1056
1049
|
await redraw.wait(5000, redrawOptions);
|
|
@@ -1079,21 +1072,14 @@ const createCommands = (
|
|
|
1079
1072
|
const result = await delay(timeout);
|
|
1080
1073
|
|
|
1081
1074
|
// Track interaction (fire-and-forget)
|
|
1082
|
-
const
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
timestamp: waitTimestamp, // Use dashcam elapsed time instead of absolute time
|
|
1091
|
-
duration: waitDuration,
|
|
1092
|
-
success: true,
|
|
1093
|
-
}).catch((err) => {
|
|
1094
|
-
console.warn("Failed to track wait interaction:", err.message);
|
|
1095
|
-
});
|
|
1096
|
-
}
|
|
1075
|
+
const waitDuration = Date.now() - waitStartTime;
|
|
1076
|
+
trackInteraction({
|
|
1077
|
+
interactionType: "wait",
|
|
1078
|
+
input: { timeout },
|
|
1079
|
+
timestamp: waitTimestamp,
|
|
1080
|
+
duration: waitDuration,
|
|
1081
|
+
success: true,
|
|
1082
|
+
});
|
|
1097
1083
|
|
|
1098
1084
|
return result;
|
|
1099
1085
|
},
|
|
@@ -1158,44 +1144,30 @@ const createCommands = (
|
|
|
1158
1144
|
);
|
|
1159
1145
|
|
|
1160
1146
|
// Track interaction success (fire-and-forget)
|
|
1161
|
-
const
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
timestamp: waitForImageTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
|
|
1171
|
-
duration: waitForImageDuration,
|
|
1172
|
-
success: true,
|
|
1173
|
-
}).catch((err) => {
|
|
1174
|
-
console.warn("Failed to track waitForImage interaction:", err.message);
|
|
1175
|
-
});
|
|
1176
|
-
}
|
|
1147
|
+
const waitForImageDuration = Date.now() - startTime;
|
|
1148
|
+
trackInteraction({
|
|
1149
|
+
interactionType: "waitForImage",
|
|
1150
|
+
prompt: description,
|
|
1151
|
+
input: { timeout },
|
|
1152
|
+
timestamp: waitForImageTimestamp,
|
|
1153
|
+
duration: waitForImageDuration,
|
|
1154
|
+
success: true,
|
|
1155
|
+
});
|
|
1177
1156
|
|
|
1178
1157
|
return;
|
|
1179
1158
|
} else {
|
|
1180
1159
|
// Track interaction failure (fire-and-forget)
|
|
1181
|
-
const sessionId = sessionInstance?.get();
|
|
1182
1160
|
const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for an image matching the description \"${description}\"`;
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
success: false,
|
|
1194
|
-
error: errorMsg,
|
|
1195
|
-
}).catch((err) => {
|
|
1196
|
-
console.warn("Failed to track waitForImage interaction:", err.message);
|
|
1197
|
-
});
|
|
1198
|
-
}
|
|
1161
|
+
const waitForImageDuration = Date.now() - startTime;
|
|
1162
|
+
trackInteraction({
|
|
1163
|
+
interactionType: "waitForImage",
|
|
1164
|
+
prompt: description,
|
|
1165
|
+
input: { timeout },
|
|
1166
|
+
timestamp: waitForImageTimestamp,
|
|
1167
|
+
duration: waitForImageDuration,
|
|
1168
|
+
success: false,
|
|
1169
|
+
error: errorMsg,
|
|
1170
|
+
});
|
|
1199
1171
|
|
|
1200
1172
|
throw new MatchError(errorMsg);
|
|
1201
1173
|
}
|
|
@@ -1267,44 +1239,30 @@ const createCommands = (
|
|
|
1267
1239
|
emitter.emit(events.log.narration, theme.dim(`"${text}" found!`), true);
|
|
1268
1240
|
|
|
1269
1241
|
// Track interaction success (fire-and-forget)
|
|
1270
|
-
const
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
timestamp: waitForTextTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
|
|
1280
|
-
duration: waitForTextDuration,
|
|
1281
|
-
success: true,
|
|
1282
|
-
}).catch((err) => {
|
|
1283
|
-
console.warn("Failed to track waitForText interaction:", err.message);
|
|
1284
|
-
});
|
|
1285
|
-
}
|
|
1242
|
+
const waitForTextDuration = Date.now() - startTime;
|
|
1243
|
+
trackInteraction({
|
|
1244
|
+
interactionType: "waitForText",
|
|
1245
|
+
prompt: text,
|
|
1246
|
+
input: { timeout },
|
|
1247
|
+
timestamp: waitForTextTimestamp,
|
|
1248
|
+
duration: waitForTextDuration,
|
|
1249
|
+
success: true,
|
|
1250
|
+
});
|
|
1286
1251
|
|
|
1287
1252
|
return;
|
|
1288
1253
|
} else {
|
|
1289
1254
|
// Track interaction failure (fire-and-forget)
|
|
1290
|
-
const sessionId = sessionInstance?.get();
|
|
1291
1255
|
const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for "${text}"`;
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
success: false,
|
|
1303
|
-
error: errorMsg,
|
|
1304
|
-
}).catch((err) => {
|
|
1305
|
-
console.warn("Failed to track waitForText interaction:", err.message);
|
|
1306
|
-
});
|
|
1307
|
-
}
|
|
1256
|
+
const waitForTextDuration = Date.now() - startTime;
|
|
1257
|
+
trackInteraction({
|
|
1258
|
+
interactionType: "waitForText",
|
|
1259
|
+
prompt: text,
|
|
1260
|
+
input: { timeout },
|
|
1261
|
+
timestamp: waitForTextTimestamp,
|
|
1262
|
+
duration: waitForTextDuration,
|
|
1263
|
+
success: false,
|
|
1264
|
+
error: errorMsg,
|
|
1265
|
+
});
|
|
1308
1266
|
|
|
1309
1267
|
throw new MatchError(errorMsg);
|
|
1310
1268
|
}
|
|
@@ -1514,45 +1472,27 @@ const createCommands = (
|
|
|
1514
1472
|
});
|
|
1515
1473
|
|
|
1516
1474
|
// Track interaction success
|
|
1517
|
-
const
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
prompt: description,
|
|
1526
|
-
timestamp: rememberTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
|
|
1527
|
-
duration: rememberDuration,
|
|
1528
|
-
success: true,
|
|
1529
|
-
});
|
|
1530
|
-
} catch (err) {
|
|
1531
|
-
console.warn("Failed to track extract interaction:", err.message);
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1475
|
+
const rememberDuration = Date.now() - rememberStartTime;
|
|
1476
|
+
trackInteraction({
|
|
1477
|
+
interactionType: "extract",
|
|
1478
|
+
prompt: description,
|
|
1479
|
+
timestamp: rememberTimestamp,
|
|
1480
|
+
duration: rememberDuration,
|
|
1481
|
+
success: true,
|
|
1482
|
+
});
|
|
1534
1483
|
|
|
1535
1484
|
return result.data;
|
|
1536
1485
|
} catch (error) {
|
|
1537
1486
|
// Track interaction failure
|
|
1538
|
-
const
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
timestamp: rememberTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
|
|
1548
|
-
duration: rememberDuration,
|
|
1549
|
-
success: false,
|
|
1550
|
-
error: error.message,
|
|
1551
|
-
});
|
|
1552
|
-
} catch (err) {
|
|
1553
|
-
console.warn("Failed to track extract interaction:", err.message);
|
|
1554
|
-
}
|
|
1555
|
-
}
|
|
1487
|
+
const rememberDuration = Date.now() - rememberStartTime;
|
|
1488
|
+
trackInteraction({
|
|
1489
|
+
interactionType: "extract",
|
|
1490
|
+
prompt: description,
|
|
1491
|
+
timestamp: rememberTimestamp,
|
|
1492
|
+
duration: rememberDuration,
|
|
1493
|
+
success: false,
|
|
1494
|
+
error: error.message,
|
|
1495
|
+
});
|
|
1556
1496
|
throw error;
|
|
1557
1497
|
}
|
|
1558
1498
|
},
|
|
@@ -84,6 +84,13 @@ function createDebuggerServer(config = {}) {
|
|
|
84
84
|
return;
|
|
85
85
|
}
|
|
86
86
|
const actualPort = address.port;
|
|
87
|
+
|
|
88
|
+
// Don't let the debugger server prevent Node process from exiting
|
|
89
|
+
// This ensures tests can exit even if debugger cleanup fails
|
|
90
|
+
if (server.unref) {
|
|
91
|
+
server.unref();
|
|
92
|
+
}
|
|
93
|
+
|
|
87
94
|
resolve({ port: actualPort, server, wss });
|
|
88
95
|
});
|
|
89
96
|
|
|
@@ -110,6 +117,9 @@ function broadcastEvent(event, data) {
|
|
|
110
117
|
|
|
111
118
|
async function startDebugger(config = {}, emitter) {
|
|
112
119
|
try {
|
|
120
|
+
// Register exit handler to ensure cleanup on process exit
|
|
121
|
+
registerExitHandler();
|
|
122
|
+
|
|
113
123
|
const { port } = await createDebuggerServer(config);
|
|
114
124
|
const url = `http://localhost:${port}`;
|
|
115
125
|
|
|
@@ -190,6 +200,29 @@ function stopDebugger() {
|
|
|
190
200
|
forceStopDebugger();
|
|
191
201
|
}
|
|
192
202
|
|
|
203
|
+
// Ensure debugger server is cleaned up on process exit
|
|
204
|
+
// This prevents zombie processes when tests crash or cleanup fails
|
|
205
|
+
let exitHandlerRegistered = false;
|
|
206
|
+
function registerExitHandler() {
|
|
207
|
+
if (exitHandlerRegistered) return;
|
|
208
|
+
exitHandlerRegistered = true;
|
|
209
|
+
|
|
210
|
+
process.on("exit", () => {
|
|
211
|
+
if (server || wss) {
|
|
212
|
+
// Synchronous cleanup - can't await in exit handler
|
|
213
|
+
if (wss) {
|
|
214
|
+
try { wss.close(); } catch (e) { /* ignore */ }
|
|
215
|
+
wss = null;
|
|
216
|
+
}
|
|
217
|
+
if (server) {
|
|
218
|
+
try { server.close(); } catch (e) { /* ignore */ }
|
|
219
|
+
server = null;
|
|
220
|
+
}
|
|
221
|
+
clients.clear();
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
193
226
|
module.exports = {
|
|
194
227
|
startDebugger,
|
|
195
228
|
stopDebugger,
|
package/agent/lib/sandbox.js
CHANGED
|
@@ -128,6 +128,10 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
|
|
|
128
128
|
);
|
|
129
129
|
}
|
|
130
130
|
}, timeout);
|
|
131
|
+
// Don't let pending timeouts prevent Node process from exiting
|
|
132
|
+
if (timeoutId.unref) {
|
|
133
|
+
timeoutId.unref();
|
|
134
|
+
}
|
|
131
135
|
|
|
132
136
|
// Track timeout so close() can clear it
|
|
133
137
|
this.pendingTimeouts.set(requestId, timeoutId);
|
|
@@ -150,7 +154,7 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
|
|
|
150
154
|
|
|
151
155
|
// Fire-and-forget message types: attach .catch() to prevent
|
|
152
156
|
// unhandled promise rejections if nobody awaits the result
|
|
153
|
-
const fireAndForgetTypes = ["output"
|
|
157
|
+
const fireAndForgetTypes = ["output"];
|
|
154
158
|
if (fireAndForgetTypes.includes(message.type)) {
|
|
155
159
|
p.catch(() => {});
|
|
156
160
|
}
|
|
@@ -360,6 +364,10 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
|
|
|
360
364
|
this.reconnecting = false;
|
|
361
365
|
}
|
|
362
366
|
}, delay);
|
|
367
|
+
// Don't let reconnect timer prevent Node process from exiting
|
|
368
|
+
if (this.reconnectTimer.unref) {
|
|
369
|
+
this.reconnectTimer.unref();
|
|
370
|
+
}
|
|
363
371
|
}
|
|
364
372
|
|
|
365
373
|
/**
|
package/agent/lib/sdk.js
CHANGED
|
@@ -346,6 +346,73 @@ const createSDK = (emitter, config, sessionInstance) => {
|
|
|
346
346
|
}
|
|
347
347
|
}
|
|
348
348
|
|
|
349
|
+
// ── S3 upload: replace large inline base64 images with S3 keys ──────
|
|
350
|
+
// If data.image is a large base64 string (>50KB), upload the raw PNG
|
|
351
|
+
// to S3 via a presigned URL and send only the imageKey instead.
|
|
352
|
+
// This reduces JSON body size from ~1.3MB to ~60 bytes.
|
|
353
|
+
const MIN_IMAGE_SIZE = 50_000; // 50KB base64 chars
|
|
354
|
+
if (
|
|
355
|
+
data &&
|
|
356
|
+
typeof data.image === "string" &&
|
|
357
|
+
data.image.length > MIN_IMAGE_SIZE
|
|
358
|
+
) {
|
|
359
|
+
try {
|
|
360
|
+
const apiRoot = config["TD_API_ROOT"];
|
|
361
|
+
const uploadUrlEndpoint = [apiRoot, "api", version, "testdriver", "upload-url"].join("/");
|
|
362
|
+
|
|
363
|
+
// Step 1: Get presigned upload URL from API
|
|
364
|
+
const uploadRes = await axios(uploadUrlEndpoint, {
|
|
365
|
+
method: "post",
|
|
366
|
+
headers: {
|
|
367
|
+
"Content-Type": "application/json",
|
|
368
|
+
...(token && { Authorization: `Bearer ${token}` }),
|
|
369
|
+
},
|
|
370
|
+
timeout: 15000,
|
|
371
|
+
data: {
|
|
372
|
+
session: sessionInstance.get(),
|
|
373
|
+
contentType: "image/png",
|
|
374
|
+
},
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
const { uploadUrl, imageKey } = uploadRes.data;
|
|
378
|
+
|
|
379
|
+
if (uploadUrl && imageKey) {
|
|
380
|
+
// Step 2: Upload raw PNG bytes to S3 via presigned PUT URL
|
|
381
|
+
const base64Data = data.image.replace(/^data:image\/\w+;base64,/, "");
|
|
382
|
+
const pngBuffer = Buffer.from(base64Data, "base64");
|
|
383
|
+
|
|
384
|
+
await axios(uploadUrl, {
|
|
385
|
+
method: "put",
|
|
386
|
+
headers: {
|
|
387
|
+
"Content-Type": "image/png",
|
|
388
|
+
"Content-Length": pngBuffer.length,
|
|
389
|
+
},
|
|
390
|
+
data: pngBuffer,
|
|
391
|
+
timeout: 30000,
|
|
392
|
+
maxBodyLength: Infinity,
|
|
393
|
+
maxContentLength: Infinity,
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
// Step 3: Replace image with imageKey in the request data
|
|
397
|
+
const savedKB = (data.image.length / 1024).toFixed(0);
|
|
398
|
+
delete data.image;
|
|
399
|
+
data.imageKey = imageKey;
|
|
400
|
+
emitter.emit(events.log?.debug || events.sdk.request, {
|
|
401
|
+
path,
|
|
402
|
+
message: `[sdk] uploaded screenshot to S3 (saved ${savedKB}KB inline), imageKey=${imageKey}`,
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
} catch (uploadErr) {
|
|
406
|
+
// Non-fatal: fall back to sending base64 inline
|
|
407
|
+
// This ensures old API servers without the upload-url endpoint still work
|
|
408
|
+
emitter.emit(events.log?.debug || events.sdk.request, {
|
|
409
|
+
path,
|
|
410
|
+
message: `[sdk] S3 upload failed, falling back to inline base64: ${uploadErr.message}`,
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
// ── End S3 upload ───────────────────────────────────────────────────
|
|
415
|
+
|
|
349
416
|
emitter.emit(events.sdk.request, {
|
|
350
417
|
path,
|
|
351
418
|
});
|
|
@@ -13,11 +13,20 @@ import { getDefaults } from "./config.mjs";
|
|
|
13
13
|
* @param {string} username - Username (default: 'standard_user')
|
|
14
14
|
*/
|
|
15
15
|
async function performLogin(client, username = "standard_user") {
|
|
16
|
+
|
|
17
|
+
console.log('Performing login with username:', username);
|
|
16
18
|
await client.focusApplication("Google Chrome");
|
|
19
|
+
|
|
20
|
+
console.log('Extracting password from page');
|
|
17
21
|
const password = await client.extract("the password");
|
|
22
|
+
|
|
23
|
+
console.log('Password extracted:', password ? '***' : 'not found');
|
|
24
|
+
|
|
18
25
|
const usernameField = await client.find(
|
|
19
26
|
"username input",
|
|
20
27
|
);
|
|
28
|
+
|
|
29
|
+
console.log('Clicking on username field and entering credentials');
|
|
21
30
|
await usernameField.click();
|
|
22
31
|
await client.type(username);
|
|
23
32
|
await client.pressKeys(["tab"]);
|
|
@@ -35,6 +44,8 @@ describe("Hover Image Test", () => {
|
|
|
35
44
|
url: 'http://testdriver-sandbox.vercel.app/login'
|
|
36
45
|
});
|
|
37
46
|
|
|
47
|
+
console.log('starting login')
|
|
48
|
+
|
|
38
49
|
// Perform login first
|
|
39
50
|
await performLogin(testdriver);
|
|
40
51
|
|
package/lib/vitest/hooks.mjs
CHANGED
|
@@ -121,7 +121,6 @@ function bufferConsoleToClients(args, level) {
|
|
|
121
121
|
line: message,
|
|
122
122
|
level: level || "log",
|
|
123
123
|
source: "console",
|
|
124
|
-
logFile: "console",
|
|
125
124
|
});
|
|
126
125
|
}
|
|
127
126
|
}
|
|
@@ -217,47 +216,6 @@ function cleanupConsoleSpy(client) {
|
|
|
217
216
|
const testDriverInstances = new WeakMap();
|
|
218
217
|
const lifecycleHandlers = new WeakMap();
|
|
219
218
|
|
|
220
|
-
/**
|
|
221
|
-
* Known log file paths on the Windows test runner.
|
|
222
|
-
* These are written by pyautogui-cli.py and related services.
|
|
223
|
-
*/
|
|
224
|
-
const WINDOWS_RUNNER_LOG_PATHS = [
|
|
225
|
-
"C:\\Windows\\Temp\\pyautogui-cli.log",
|
|
226
|
-
];
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* Fetch log files from the Windows test runner via exec() and save them locally.
|
|
230
|
-
* This runs before disconnect so we can still communicate with the sandbox.
|
|
231
|
-
*
|
|
232
|
-
* @param {import('../../sdk.js').default} client - TestDriver SDK instance
|
|
233
|
-
* @param {string} testName - Test file name (used for local directory naming)
|
|
234
|
-
*/
|
|
235
|
-
async function fetchRunnerLogs(client, testName) {
|
|
236
|
-
if (!client.connected) return;
|
|
237
|
-
if (client.os !== "windows") return;
|
|
238
|
-
|
|
239
|
-
for (const remotePath of WINDOWS_RUNNER_LOG_PATHS) {
|
|
240
|
-
const localName = path.basename(remotePath);
|
|
241
|
-
try {
|
|
242
|
-
// Read the log file contents via PowerShell
|
|
243
|
-
const cmd = `if (Test-Path '${remotePath}') { Get-Content -Path '${remotePath}' -Raw } else { Write-Output '__FILE_NOT_FOUND__' }`;
|
|
244
|
-
const result = await Promise.race([
|
|
245
|
-
client.exec("pwsh", cmd, 10000, true),
|
|
246
|
-
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 15000)),
|
|
247
|
-
]);
|
|
248
|
-
|
|
249
|
-
if (result && typeof result === "string" && !result.includes("__FILE_NOT_FOUND__")) {
|
|
250
|
-
console.log(`\n[TestDriver] === Runner Log: ${localName} ===`);
|
|
251
|
-
console.log(result);
|
|
252
|
-
console.log(`[TestDriver] === End Runner Log ===\n`);
|
|
253
|
-
}
|
|
254
|
-
} catch (err) {
|
|
255
|
-
// Fire-and-forget — don't let log retrieval break cleanup
|
|
256
|
-
console.warn(`[TestDriver] Could not fetch runner log ${remotePath}: ${err.message}`);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
|
|
261
219
|
/**
|
|
262
220
|
* Upload buffered SDK + console logs directly to S3 via the existing Log system.
|
|
263
221
|
* Extracts the replayId from the dashcam URL, calls POST /api/v1/logs to create
|
|
@@ -673,16 +631,6 @@ export function TestDriver(context, options = {}) {
|
|
|
673
631
|
context.task.meta.dashcamUrl = null;
|
|
674
632
|
}
|
|
675
633
|
|
|
676
|
-
// Fetch runner logs before disconnecting (Windows only)
|
|
677
|
-
// This grabs pyautogui-cli logs from the runner while we still have a connection
|
|
678
|
-
try {
|
|
679
|
-
const logTestName = testFile.replace(/[/\\]/g, "_").replace(/\.test\.m?js$/, "");
|
|
680
|
-
await fetchRunnerLogs(currentInstance, logTestName);
|
|
681
|
-
} catch (err) {
|
|
682
|
-
// Never let log retrieval block cleanup
|
|
683
|
-
console.warn("[TestDriver] Runner log fetch failed:", err.message);
|
|
684
|
-
}
|
|
685
|
-
|
|
686
634
|
// Clean up console spies
|
|
687
635
|
cleanupConsoleSpy(currentInstance);
|
|
688
636
|
|