testdriverai 5.3.5 → 5.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/testdriver.yml +48 -8
- package/agent.js +84 -62
- package/docs/docs.json +2 -4
- package/electron/overlay.html +0 -2
- package/lib/init.js +38 -35
- package/lib/logger.js +7 -15
- package/package.json +2 -1
- package/testdriver/testdriver_2025-04-14T17-43-37-760Z.yaml +0 -10
- package/testdriver/testdriver_2025-04-14T17-45-00-565Z.yaml +0 -13
- package/testdriver/testdriver_2025-04-14T23-14-46-822Z.yaml +0 -0
- package/testdriver/testdriver_2025-04-14T23-14-51-653Z.yaml +0 -0
- package/testdriver/testdriver_2025-04-15T00-34-42-776Z.yaml +0 -0
- package/testdriver/testdriver_2025-04-15T00-35-22-776Z.yaml +0 -4
- package/testdriver/testdriver_2025-04-15T00-36-47-767Z.yaml +0 -0
|
@@ -1,28 +1,68 @@
|
|
|
1
|
-
name: TestDriver.ai
|
|
1
|
+
name: TestDriver.ai / Run / Regressions
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
push:
|
|
5
5
|
branches: ["main"]
|
|
6
6
|
pull_request:
|
|
7
7
|
workflow_dispatch:
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: '0 13 * * *' # Every day at 1 PM UTC (adjust if needed for timezone)
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
actions: read
|
|
13
|
+
contents: read
|
|
14
|
+
statuses: write
|
|
15
|
+
pull-requests: write
|
|
8
16
|
|
|
9
17
|
jobs:
|
|
10
|
-
test:
|
|
11
|
-
name:
|
|
18
|
+
gather-test-files:
|
|
19
|
+
name: Setup Test Matrix (./testdriver/*.yml)
|
|
12
20
|
runs-on: ubuntu-latest
|
|
21
|
+
outputs:
|
|
22
|
+
test_files: ${{ steps.test_list.outputs.files }}
|
|
13
23
|
steps:
|
|
24
|
+
- name: Check out repository
|
|
25
|
+
uses: actions/checkout@v2
|
|
26
|
+
with:
|
|
27
|
+
ref: ${{ github.event.ref }}
|
|
28
|
+
- name: Find all test files and extract filenames
|
|
29
|
+
id: test_list
|
|
30
|
+
run: |
|
|
31
|
+
FILES=$(ls ./testdriver/*.yml)
|
|
32
|
+
FILENAMES=$(basename -a $FILES)
|
|
33
|
+
FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]')
|
|
34
|
+
echo "::set-output name=files::$FILES_JSON"
|
|
35
|
+
|
|
36
|
+
test:
|
|
37
|
+
needs: gather-test-files
|
|
38
|
+
runs-on: ${{ matrix.os }}
|
|
39
|
+
strategy:
|
|
40
|
+
matrix:
|
|
41
|
+
test: ${{ fromJson(needs.gather-test-files.outputs.test_files) }}
|
|
42
|
+
os: ['linux', 'windows']
|
|
43
|
+
fail-fast: false
|
|
44
|
+
name: ${{ matrix.test }} (${{ matrix.os }})
|
|
45
|
+
steps:
|
|
46
|
+
- name: Check out repository
|
|
47
|
+
uses: actions/checkout@v2
|
|
48
|
+
with:
|
|
49
|
+
ref: ${{ github.event.ref }}
|
|
50
|
+
|
|
51
|
+
- name: Display filename being tested
|
|
52
|
+
run: echo "Running job for file: ${{ matrix.test }} on ${{ matrix.os }}"
|
|
53
|
+
|
|
14
54
|
- uses: testdriverai/action@main
|
|
15
55
|
with:
|
|
16
|
-
key: ${{secrets.TESTDRIVER_API_KEY}}
|
|
17
|
-
|
|
18
|
-
|
|
56
|
+
key: ${{ secrets.TESTDRIVER_API_KEY }}
|
|
57
|
+
os: ${{ matrix.os }}
|
|
58
|
+
version: "5.2.3"
|
|
59
|
+
prompt: 1. /run testdriver/${{ matrix.test }}
|
|
19
60
|
prerun: |
|
|
20
61
|
cd $env:TEMP
|
|
21
62
|
npm init -y
|
|
22
63
|
npm install dashcam-chrome
|
|
23
|
-
Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "${{ env.
|
|
64
|
+
Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "${{ env.TD_WEBSITE }}"
|
|
24
65
|
exit
|
|
25
66
|
env:
|
|
26
67
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
27
68
|
FORCE_COLOR: "3"
|
|
28
|
-
WEBSITE_URL: "https://example.com" # Define the website URL here
|
package/agent.js
CHANGED
|
@@ -137,8 +137,6 @@ let thisFile = a.file;
|
|
|
137
137
|
const thisCommand = a.command;
|
|
138
138
|
|
|
139
139
|
logger.info(chalk.green(`Howdy! I'm TestDriver v${package.version}`));
|
|
140
|
-
logger.info(chalk.dim(`Working on ${thisFile}`));
|
|
141
|
-
logger.info("");
|
|
142
140
|
logger.info(`This is beta software!`);
|
|
143
141
|
logger.info("");
|
|
144
142
|
logger.info(chalk.yellow(`Join our Discord for help`));
|
|
@@ -179,9 +177,10 @@ function fileCompleter(line) {
|
|
|
179
177
|
}
|
|
180
178
|
|
|
181
179
|
function completer(line) {
|
|
182
|
-
let completions =
|
|
183
|
-
" "
|
|
184
|
-
|
|
180
|
+
let completions =
|
|
181
|
+
"/summarize /save /run /quit /assert /undo /manual /yml /js /exec".split(
|
|
182
|
+
" ",
|
|
183
|
+
);
|
|
185
184
|
if (line.startsWith("/run ") || line.startsWith("/explore ")) {
|
|
186
185
|
return fileCompleter(line);
|
|
187
186
|
} else {
|
|
@@ -390,7 +389,12 @@ const runCommand = async (command, depth) => {
|
|
|
390
389
|
let lastCommand = new Date().getTime();
|
|
391
390
|
let csv = [["command,time"]];
|
|
392
391
|
|
|
393
|
-
const executeCommands = async (
|
|
392
|
+
const executeCommands = async (
|
|
393
|
+
commands,
|
|
394
|
+
depth,
|
|
395
|
+
pushToHistory = false,
|
|
396
|
+
dry = false,
|
|
397
|
+
) => {
|
|
394
398
|
if (commands?.length) {
|
|
395
399
|
for (const command of commands) {
|
|
396
400
|
if (pushToHistory) {
|
|
@@ -412,7 +416,12 @@ const executeCommands = async (commands, depth, pushToHistory = false, dry = fal
|
|
|
412
416
|
|
|
413
417
|
// note that commands are run in a recursive loop, so that the AI can respond to the output of the commands
|
|
414
418
|
// like `click-image` and `click-text` for example
|
|
415
|
-
const executeCodeBlocks = async (
|
|
419
|
+
const executeCodeBlocks = async (
|
|
420
|
+
codeblocks,
|
|
421
|
+
depth,
|
|
422
|
+
pushToHistory = false,
|
|
423
|
+
dry = false,
|
|
424
|
+
) => {
|
|
416
425
|
depth = depth + 1;
|
|
417
426
|
|
|
418
427
|
logger.debug("%j", { message: "execute code blocks", depth });
|
|
@@ -441,7 +450,7 @@ const aiExecute = async (message, validateAndLoop = false, dry = false) => {
|
|
|
441
450
|
executionHistory.push({ prompt: lastPrompt, commands: [] });
|
|
442
451
|
|
|
443
452
|
logger.debug("kicking off exploratory loop");
|
|
444
|
-
|
|
453
|
+
|
|
445
454
|
// kick everything off
|
|
446
455
|
await actOnMarkdown(message, 0, true, dry);
|
|
447
456
|
|
|
@@ -461,7 +470,7 @@ const aiExecute = async (message, validateAndLoop = false, dry = false) => {
|
|
|
461
470
|
|
|
462
471
|
if (checkCodeblocks.length > 0) {
|
|
463
472
|
logger.debug("check thinks more needs to be done");
|
|
464
|
-
|
|
473
|
+
|
|
465
474
|
logger.info(chalk.dim("not done yet!"), "testdriver");
|
|
466
475
|
logger.info("");
|
|
467
476
|
|
|
@@ -549,7 +558,11 @@ commands:
|
|
|
549
558
|
|
|
550
559
|
// this function responds to the result of `promptUser()` which is the user input
|
|
551
560
|
// it kicks off the exploratory loop, which is the main function that interacts with the AI
|
|
552
|
-
const exploratoryLoop = async (
|
|
561
|
+
const exploratoryLoop = async (
|
|
562
|
+
currentTask,
|
|
563
|
+
dry = false,
|
|
564
|
+
validateAndLoop = false,
|
|
565
|
+
) => {
|
|
553
566
|
lastPrompt = currentTask;
|
|
554
567
|
checkCount = 0;
|
|
555
568
|
|
|
@@ -586,7 +599,7 @@ const exploratoryLoop = async (currentTask, dry = false, validateAndLoop = false
|
|
|
586
599
|
await aiExecute(message.data, validateAndLoop, dry);
|
|
587
600
|
logger.debug("showing prompt from exploratoryLoop response check");
|
|
588
601
|
}
|
|
589
|
-
|
|
602
|
+
|
|
590
603
|
await save({ silent: false });
|
|
591
604
|
|
|
592
605
|
return;
|
|
@@ -628,7 +641,6 @@ const generate = async (type, count, baseYaml, skipYaml = false) => {
|
|
|
628
641
|
|
|
629
642
|
// for each testPrompt
|
|
630
643
|
for (const testPrompt of testPrompts) {
|
|
631
|
-
|
|
632
644
|
// with the contents of the testPrompt
|
|
633
645
|
let fileName =
|
|
634
646
|
sanitizeFilename(testPrompt.name)
|
|
@@ -647,14 +659,16 @@ const generate = async (type, count, baseYaml, skipYaml = false) => {
|
|
|
647
659
|
let list = testPrompt.steps;
|
|
648
660
|
|
|
649
661
|
if (baseYaml && fs.existsSync(baseYaml)) {
|
|
650
|
-
list.unshift({
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
662
|
+
list.unshift({
|
|
663
|
+
step: {
|
|
664
|
+
command: "run",
|
|
665
|
+
file: baseYaml,
|
|
666
|
+
},
|
|
667
|
+
});
|
|
654
668
|
}
|
|
655
|
-
let contents = yaml.dump({
|
|
669
|
+
let contents = yaml.dump({
|
|
656
670
|
version: package.version,
|
|
657
|
-
steps: list
|
|
671
|
+
steps: list,
|
|
658
672
|
});
|
|
659
673
|
fs.writeFileSync(path1, contents);
|
|
660
674
|
}
|
|
@@ -699,7 +713,12 @@ ${yml}
|
|
|
699
713
|
};
|
|
700
714
|
|
|
701
715
|
// this function is responsible for starting the recursive process of executing codeblocks
|
|
702
|
-
const actOnMarkdown = async (
|
|
716
|
+
const actOnMarkdown = async (
|
|
717
|
+
content,
|
|
718
|
+
depth,
|
|
719
|
+
pushToHistory = false,
|
|
720
|
+
dry = false,
|
|
721
|
+
) => {
|
|
703
722
|
logger.debug("%j", {
|
|
704
723
|
message: "actOnMarkdown called",
|
|
705
724
|
depth,
|
|
@@ -714,7 +733,12 @@ const actOnMarkdown = async (content, depth, pushToHistory = false, dry = false)
|
|
|
714
733
|
}
|
|
715
734
|
|
|
716
735
|
if (codeblocks.length) {
|
|
717
|
-
let executions = await executeCodeBlocks(
|
|
736
|
+
let executions = await executeCodeBlocks(
|
|
737
|
+
codeblocks,
|
|
738
|
+
depth,
|
|
739
|
+
pushToHistory,
|
|
740
|
+
dry,
|
|
741
|
+
);
|
|
718
742
|
return executions;
|
|
719
743
|
} else {
|
|
720
744
|
return true;
|
|
@@ -722,7 +746,6 @@ const actOnMarkdown = async (content, depth, pushToHistory = false, dry = false)
|
|
|
722
746
|
};
|
|
723
747
|
|
|
724
748
|
const ensureMacScreenPerms = async () => {
|
|
725
|
-
|
|
726
749
|
// if os is mac, check for screen capture permissions
|
|
727
750
|
if (
|
|
728
751
|
!config.TD_VM &&
|
|
@@ -739,12 +762,11 @@ const ensureMacScreenPerms = async () => {
|
|
|
739
762
|
analytics.track("noMacPermissions");
|
|
740
763
|
return exit();
|
|
741
764
|
}
|
|
742
|
-
}
|
|
765
|
+
};
|
|
743
766
|
|
|
744
767
|
// simple function to backfill the chat history with a prompt and
|
|
745
768
|
// then call `promptUser()` to get the user input
|
|
746
769
|
const firstPrompt = async () => {
|
|
747
|
-
|
|
748
770
|
// readline is what allows us to get user input
|
|
749
771
|
rl = readline.createInterface({
|
|
750
772
|
terminal: true,
|
|
@@ -765,7 +787,6 @@ const firstPrompt = async () => {
|
|
|
765
787
|
// this is how we parse user input
|
|
766
788
|
// notice that the AI is only called if the input is not a command
|
|
767
789
|
const handleInput = async (input) => {
|
|
768
|
-
|
|
769
790
|
if (!isInteractive) return;
|
|
770
791
|
if (!input.trim().length) return promptUser();
|
|
771
792
|
|
|
@@ -779,8 +800,10 @@ const firstPrompt = async () => {
|
|
|
779
800
|
analytics.track("input", { input });
|
|
780
801
|
|
|
781
802
|
logger.info(""); // adds a nice break between submissions
|
|
782
|
-
|
|
783
|
-
let interpolationVars = JSON.parse(
|
|
803
|
+
|
|
804
|
+
let interpolationVars = JSON.parse(
|
|
805
|
+
process.env["TD_INTERPOLATION_VARS"] || "{}",
|
|
806
|
+
);
|
|
784
807
|
|
|
785
808
|
// Inject environment variables into any ${VAR} strings
|
|
786
809
|
input = parser.interpolate(input, process.env);
|
|
@@ -814,7 +837,6 @@ const firstPrompt = async () => {
|
|
|
814
837
|
let shouldExit = flags.includes("--exit") ? true : false;
|
|
815
838
|
|
|
816
839
|
await run(file, shouldSave, shouldExit);
|
|
817
|
-
|
|
818
840
|
} else if (input.indexOf("/explore") == 0) {
|
|
819
841
|
const file = commands[1];
|
|
820
842
|
await run(file, true, true);
|
|
@@ -822,7 +844,7 @@ const firstPrompt = async () => {
|
|
|
822
844
|
const skipYaml = commands[4] === "--skip-yaml";
|
|
823
845
|
await generate(commands[1], commands[2], commands[3], skipYaml);
|
|
824
846
|
} else if (input.indexOf("/dry") == 0) {
|
|
825
|
-
await exploratoryLoop(input.replace(
|
|
847
|
+
await exploratoryLoop(input.replace("/dry", ""), true, false);
|
|
826
848
|
} else if (input.indexOf("/yaml") == 0) {
|
|
827
849
|
await runRawYML(commands[1]);
|
|
828
850
|
} else if (input.indexOf("/js") == 0) {
|
|
@@ -988,7 +1010,7 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
|
|
|
988
1010
|
try {
|
|
989
1011
|
fs.writeFileSync(filepath, regression);
|
|
990
1012
|
} catch (e) {
|
|
991
|
-
console.log(e)
|
|
1013
|
+
console.log(e);
|
|
992
1014
|
logger.error(e.message);
|
|
993
1015
|
logger.error("%s", e);
|
|
994
1016
|
}
|
|
@@ -1012,7 +1034,6 @@ ${regression}
|
|
|
1012
1034
|
};
|
|
1013
1035
|
|
|
1014
1036
|
let runRawYML = async (yml) => {
|
|
1015
|
-
|
|
1016
1037
|
const tmp = require("tmp");
|
|
1017
1038
|
let tmpobj = tmp.fileSync();
|
|
1018
1039
|
|
|
@@ -1035,17 +1056,15 @@ let runRawYML = async (yml) => {
|
|
|
1035
1056
|
|
|
1036
1057
|
// write the yaml to a file
|
|
1037
1058
|
fs.writeFileSync(tmpobj.name, yaml.dump(ymlObj));
|
|
1038
|
-
|
|
1059
|
+
|
|
1039
1060
|
// and run it with run()
|
|
1040
1061
|
await run(tmpobj.name, false, true);
|
|
1041
|
-
|
|
1042
|
-
}
|
|
1062
|
+
};
|
|
1043
1063
|
|
|
1044
1064
|
// this will load a regression test from a file location
|
|
1045
1065
|
// it parses the markdown file and executes the codeblocks exactly as if they were
|
|
1046
1066
|
// generated by the AI in a single prompt
|
|
1047
1067
|
let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
|
|
1048
|
-
|
|
1049
1068
|
setTerminalWindowTransparency(true);
|
|
1050
1069
|
emitter.emit(events.interactive, false);
|
|
1051
1070
|
|
|
@@ -1080,12 +1099,10 @@ let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
|
|
|
1080
1099
|
}
|
|
1081
1100
|
|
|
1082
1101
|
if (shouldSave) {
|
|
1083
|
-
|
|
1084
1102
|
executionHistory.push({
|
|
1085
1103
|
prompt: step.prompt,
|
|
1086
1104
|
commands: [], // run will overwrite the commands
|
|
1087
1105
|
});
|
|
1088
|
-
|
|
1089
1106
|
}
|
|
1090
1107
|
|
|
1091
1108
|
let markdown = `\`\`\`yaml
|
|
@@ -1193,24 +1210,25 @@ const start = async () => {
|
|
|
1193
1210
|
// }
|
|
1194
1211
|
|
|
1195
1212
|
// await sdk.auth();
|
|
1196
|
-
|
|
1197
1213
|
if (thisCommand !== "run") {
|
|
1198
1214
|
speak("Howdy! I am TestDriver version " + package.version);
|
|
1199
1215
|
}
|
|
1200
1216
|
|
|
1201
|
-
if (thisCommand !== "init"
|
|
1217
|
+
if (thisCommand !== "init" && thisCommand !== "upload-secrets") {
|
|
1218
|
+
logger.info(chalk.dim(`Working on ${thisFile}`));
|
|
1202
1219
|
|
|
1203
1220
|
if (!config.TD_VM) {
|
|
1204
1221
|
logger.info(
|
|
1205
|
-
chalk.red("Warning! "
|
|
1206
|
-
chalk.dim(
|
|
1222
|
+
chalk.red("Warning! ") +
|
|
1223
|
+
chalk.dim(
|
|
1224
|
+
"Local mode sends screenshots of the desktop to our API. Set `TD_VM=true` to run in a secure VM.",
|
|
1225
|
+
),
|
|
1207
1226
|
);
|
|
1208
1227
|
logger.info(
|
|
1209
1228
|
chalk.dim("https://docs.testdriver.ai/security-and-privacy/agent"),
|
|
1210
1229
|
);
|
|
1211
1230
|
logger.info("");
|
|
1212
1231
|
}
|
|
1213
|
-
|
|
1214
1232
|
}
|
|
1215
1233
|
|
|
1216
1234
|
analytics.track("command", { command: thisCommand, file: thisFile });
|
|
@@ -1231,48 +1249,47 @@ const start = async () => {
|
|
|
1231
1249
|
};
|
|
1232
1250
|
|
|
1233
1251
|
const makeSandbox = async () => {
|
|
1234
|
-
|
|
1235
1252
|
if (config.TD_VM) {
|
|
1236
|
-
|
|
1237
1253
|
try {
|
|
1238
|
-
|
|
1239
1254
|
logger.info(chalk.gray(`- creating sandbox...`));
|
|
1240
1255
|
server.broadcast("status", `Creating new sandbox...`);
|
|
1241
1256
|
await sandbox.boot();
|
|
1242
1257
|
logger.info(chalk.gray(`- authenticating...`));
|
|
1243
1258
|
server.broadcast("status", `Authenticating...`);
|
|
1244
|
-
await sandbox.send({
|
|
1259
|
+
await sandbox.send({
|
|
1260
|
+
type: "authenticate",
|
|
1261
|
+
apiKey: config.TD_API_KEY,
|
|
1262
|
+
secret: config.TD_SECRET,
|
|
1263
|
+
});
|
|
1245
1264
|
logger.info(chalk.gray(`- configuring...`));
|
|
1246
1265
|
server.broadcast("status", `Configuring...`);
|
|
1247
|
-
await sandbox.send({
|
|
1266
|
+
await sandbox.send({
|
|
1267
|
+
type: "create",
|
|
1268
|
+
resolution: config.TD_VM_RESOLUTION,
|
|
1269
|
+
});
|
|
1248
1270
|
logger.info(chalk.gray(`- starting stream...`));
|
|
1249
1271
|
server.broadcast("status", `Starting stream...`);
|
|
1250
|
-
await sandbox.send({type:
|
|
1251
|
-
let {url} = await sandbox.send({type:
|
|
1272
|
+
await sandbox.send({ type: "stream.start" });
|
|
1273
|
+
let { url } = await sandbox.send({ type: "stream.getUrl" });
|
|
1252
1274
|
logger.info(chalk.gray(`- rendering...`));
|
|
1253
1275
|
server.broadcast("status", `Rendering...`);
|
|
1254
|
-
await sandbox.send({type:
|
|
1255
|
-
emitter.emit(events.vm.show, {url});
|
|
1276
|
+
await sandbox.send({ type: "ready" });
|
|
1277
|
+
emitter.emit(events.vm.show, { url });
|
|
1256
1278
|
logger.info(chalk.gray(`- booting...`));
|
|
1257
1279
|
server.broadcast("status", `Starting...`);
|
|
1258
|
-
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
1280
|
+
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
1259
1281
|
logger.info(chalk.green(``));
|
|
1260
1282
|
logger.info(chalk.green(`sandbox runner ready!`));
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
logger.error(e)
|
|
1283
|
+
} catch (e) {
|
|
1284
|
+
logger.error(e);
|
|
1264
1285
|
logger.error(chalk.red(`sandbox runner failed to start`));
|
|
1265
1286
|
process.exit(1);
|
|
1266
1287
|
}
|
|
1267
|
-
|
|
1268
1288
|
}
|
|
1269
1289
|
|
|
1270
1290
|
emitter.emit(events.interactive, false);
|
|
1271
|
-
emitter.emit(events.showWindow)
|
|
1272
|
-
|
|
1273
|
-
}
|
|
1274
|
-
|
|
1275
|
-
|
|
1291
|
+
emitter.emit(events.showWindow);
|
|
1292
|
+
};
|
|
1276
1293
|
|
|
1277
1294
|
const newSession = async () => {
|
|
1278
1295
|
// should be start of new session
|
|
@@ -1286,11 +1303,16 @@ const newSession = async () => {
|
|
|
1286
1303
|
};
|
|
1287
1304
|
|
|
1288
1305
|
const runPrerun = async () => {
|
|
1289
|
-
const prerunFile = path.join(
|
|
1306
|
+
const prerunFile = path.join(
|
|
1307
|
+
workingDir,
|
|
1308
|
+
"testdriver",
|
|
1309
|
+
"lifecycle",
|
|
1310
|
+
"prerun.yaml",
|
|
1311
|
+
);
|
|
1290
1312
|
if (fs.existsSync(prerunFile)) {
|
|
1291
1313
|
await run(prerunFile, false, false);
|
|
1292
1314
|
}
|
|
1293
|
-
}
|
|
1315
|
+
};
|
|
1294
1316
|
|
|
1295
1317
|
process.on("uncaughtException", async (err) => {
|
|
1296
1318
|
analytics.track("uncaughtException", { err });
|
package/docs/docs.json
CHANGED
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"pages": [
|
|
28
28
|
"/getting-started/vscode",
|
|
29
29
|
"/getting-started/setup",
|
|
30
|
-
"/getting-started/generating",
|
|
31
30
|
"/getting-started/writing",
|
|
31
|
+
"/getting-started/generating",
|
|
32
32
|
"/getting-started/running",
|
|
33
33
|
"/getting-started/ci",
|
|
34
34
|
"/features/auto-healing",
|
|
@@ -105,9 +105,7 @@
|
|
|
105
105
|
},
|
|
106
106
|
{
|
|
107
107
|
"group": "CLI",
|
|
108
|
-
"pages": [
|
|
109
|
-
"/cli/overview"
|
|
110
|
-
]
|
|
108
|
+
"pages": ["/cli/overview"]
|
|
111
109
|
}
|
|
112
110
|
]
|
|
113
111
|
}
|
package/electron/overlay.html
CHANGED
package/lib/init.js
CHANGED
|
@@ -7,6 +7,7 @@ const os = require("os");
|
|
|
7
7
|
const chalk = require("chalk");
|
|
8
8
|
const { Readable } = require("stream");
|
|
9
9
|
const { logger } = require("./logger");
|
|
10
|
+
const Table = require("cli-table3");
|
|
10
11
|
|
|
11
12
|
async function getLatestRelease(owner, repo) {
|
|
12
13
|
try {
|
|
@@ -59,47 +60,51 @@ async function getLatestRelease(owner, repo) {
|
|
|
59
60
|
}
|
|
60
61
|
|
|
61
62
|
module.exports = async () => {
|
|
62
|
-
logger.info(
|
|
63
|
-
logger.info("");
|
|
64
|
-
logger.info(chalk.dim("Choose a runner:"));
|
|
63
|
+
logger.info("------");
|
|
65
64
|
logger.info("");
|
|
66
|
-
logger.info(chalk.green("
|
|
67
|
-
logger.info(chalk.dim("Uses your computer to execute tests."));
|
|
65
|
+
logger.info(chalk.green("Welcome to the Testdriver Setup!"));
|
|
68
66
|
logger.info("");
|
|
69
|
-
logger.info(
|
|
70
|
-
logger.info(
|
|
71
|
-
logger.info(chalk.
|
|
72
|
-
logger.info(chalk.dim("
|
|
73
|
-
logger.info(
|
|
74
|
-
logger.info(chalk.
|
|
67
|
+
logger.info("Choose a Runner");
|
|
68
|
+
logger.info("│");
|
|
69
|
+
logger.info("├── Local Runner:" + chalk.green(" Free"));
|
|
70
|
+
logger.info("│ └── " + chalk.dim("Run tests on your computer"));
|
|
71
|
+
logger.info("│");
|
|
72
|
+
logger.info("└── Sandbox Runner:" + chalk.green(" 14 Day Trial then $~3/hr"));
|
|
73
|
+
logger.info(
|
|
74
|
+
" └── " + chalk.dim("Run tests on private hosted ephemeral VMs"),
|
|
75
|
+
);
|
|
76
|
+
logger.info(" ├── ✅ " + "Added Privacy");
|
|
77
|
+
logger.info(" ├── ✅ " + "Faster Execution");
|
|
78
|
+
logger.info(" ├── ✅ " + "Parallel Execution");
|
|
79
|
+
logger.info(" └── ✅ " + "Better DX");
|
|
75
80
|
logger.info("");
|
|
76
81
|
|
|
77
82
|
const response = await prompts([
|
|
78
83
|
{
|
|
79
84
|
type: "confirm",
|
|
80
85
|
name: "TD_VM",
|
|
81
|
-
message: "Use
|
|
86
|
+
message: "Use Sandbox Runners? (Recommended)",
|
|
82
87
|
initial: true,
|
|
83
88
|
},
|
|
84
89
|
{
|
|
85
|
-
type: prev => (prev ? "password" : null),
|
|
86
|
-
name:
|
|
87
|
-
message:
|
|
90
|
+
type: (prev) => (prev ? "password" : null),
|
|
91
|
+
name: "TD_API_KEY",
|
|
92
|
+
message: "API KEY (from https://app.testdriver.ai/team)",
|
|
88
93
|
},
|
|
89
94
|
{
|
|
90
|
-
type: prev => (prev ? null : "confirm"),
|
|
95
|
+
type: (prev) => (prev ? null : "confirm"),
|
|
91
96
|
name: "TD_MINIMIZE",
|
|
92
97
|
message: "Minimize terminal app?",
|
|
93
98
|
initial: false,
|
|
94
99
|
},
|
|
95
100
|
{
|
|
96
|
-
type: prev => (prev ? null : "confirm"),
|
|
101
|
+
type: (prev) => (prev ? null : "confirm"),
|
|
97
102
|
name: "TD_NOTIFY",
|
|
98
103
|
message: "Enable desktop notifications?",
|
|
99
104
|
initial: true,
|
|
100
105
|
},
|
|
101
106
|
{
|
|
102
|
-
type: prev => (prev ? null : "confirm"),
|
|
107
|
+
type: (prev) => (prev ? null : "confirm"),
|
|
103
108
|
name: "TD_SPEAK",
|
|
104
109
|
message: "Enable text to speech narration?",
|
|
105
110
|
initial: true,
|
|
@@ -127,7 +132,7 @@ module.exports = async () => {
|
|
|
127
132
|
name: "TD_ANALYTICS",
|
|
128
133
|
message: "Send anonymous analytics?",
|
|
129
134
|
initial: true,
|
|
130
|
-
}
|
|
135
|
+
},
|
|
131
136
|
]);
|
|
132
137
|
|
|
133
138
|
logger.info("");
|
|
@@ -136,10 +141,12 @@ module.exports = async () => {
|
|
|
136
141
|
logger.info(`Downloading latest workflow files...`);
|
|
137
142
|
logger.info("");
|
|
138
143
|
|
|
139
|
-
const append = path.join(process.cwd(),
|
|
140
|
-
const existingEnv = fs.existsSync(append)
|
|
141
|
-
|
|
142
|
-
|
|
144
|
+
const append = path.join(process.cwd(), ".env");
|
|
145
|
+
const existingEnv = fs.existsSync(append)
|
|
146
|
+
? fs.readFileSync(append, "utf8")
|
|
147
|
+
: "";
|
|
148
|
+
const envMap = existingEnv.split("\n").reduce((acc, line) => {
|
|
149
|
+
const [key, value] = line.split("=");
|
|
143
150
|
if (key) acc[key] = value;
|
|
144
151
|
return acc;
|
|
145
152
|
}, {});
|
|
@@ -150,7 +157,7 @@ module.exports = async () => {
|
|
|
150
157
|
|
|
151
158
|
const updatedEnv = Object.entries(envMap)
|
|
152
159
|
.map(([key, value]) => `${key}=${value}`)
|
|
153
|
-
.join(
|
|
160
|
+
.join("\n");
|
|
154
161
|
|
|
155
162
|
await fs.writeFileSync(append, updatedEnv);
|
|
156
163
|
|
|
@@ -160,9 +167,6 @@ module.exports = async () => {
|
|
|
160
167
|
await decompress(resolvedPath, process.cwd(), {
|
|
161
168
|
strip: 1,
|
|
162
169
|
filter: (file) => {
|
|
163
|
-
|
|
164
|
-
console.log(file.path);
|
|
165
|
-
|
|
166
170
|
let pass =
|
|
167
171
|
file.path.startsWith("testdriver") || file.path.startsWith(".github");
|
|
168
172
|
if (pass) {
|
|
@@ -187,18 +191,15 @@ module.exports = async () => {
|
|
|
187
191
|
fs.mkdirSync(testdriverGenerateFolder);
|
|
188
192
|
}
|
|
189
193
|
|
|
190
|
-
const tdScreen = path.join(
|
|
191
|
-
process.cwd(),
|
|
192
|
-
"testdriver",
|
|
193
|
-
"screenshots"
|
|
194
|
-
);
|
|
194
|
+
const tdScreen = path.join(process.cwd(), "testdriver", "screenshots");
|
|
195
195
|
if (!fs.existsSync(tdScreen)) {
|
|
196
196
|
fs.mkdirSync(tdScreen);
|
|
197
197
|
}
|
|
198
198
|
const tdScreenMac = path.join(
|
|
199
199
|
process.cwd(),
|
|
200
200
|
"testdriver",
|
|
201
|
-
"screenshots",
|
|
201
|
+
"screenshots",
|
|
202
|
+
"mac",
|
|
202
203
|
);
|
|
203
204
|
if (!fs.existsSync(tdScreenMac)) {
|
|
204
205
|
fs.mkdirSync(tdScreenMac);
|
|
@@ -206,7 +207,8 @@ module.exports = async () => {
|
|
|
206
207
|
const tdScreenWindows = path.join(
|
|
207
208
|
process.cwd(),
|
|
208
209
|
"testdriver",
|
|
209
|
-
"screenshots",
|
|
210
|
+
"screenshots",
|
|
211
|
+
"windows",
|
|
210
212
|
);
|
|
211
213
|
if (!fs.existsSync(tdScreenWindows)) {
|
|
212
214
|
fs.mkdirSync(tdScreenWindows);
|
|
@@ -214,7 +216,8 @@ module.exports = async () => {
|
|
|
214
216
|
const tdScreenLinux = path.join(
|
|
215
217
|
process.cwd(),
|
|
216
218
|
"testdriver",
|
|
217
|
-
"screenshots",
|
|
219
|
+
"screenshots",
|
|
220
|
+
"linux",
|
|
218
221
|
);
|
|
219
222
|
if (!fs.existsSync(tdScreenLinux)) {
|
|
220
223
|
fs.mkdirSync(tdScreenLinux);
|
package/lib/logger.js
CHANGED
|
@@ -17,7 +17,7 @@ const logFormat = printf(({ message }) => {
|
|
|
17
17
|
return `${message}`;
|
|
18
18
|
});
|
|
19
19
|
|
|
20
|
-
let interpolationVars = JSON.parse(process.env.TD_INTERPOLATION_VARS ||
|
|
20
|
+
let interpolationVars = JSON.parse(process.env.TD_INTERPOLATION_VARS || "{}");
|
|
21
21
|
|
|
22
22
|
// this handles local `TD_*` variables
|
|
23
23
|
for (const [key, value] of Object.entries(process.env)) {
|
|
@@ -28,31 +28,27 @@ for (const [key, value] of Object.entries(process.env)) {
|
|
|
28
28
|
|
|
29
29
|
const censorSensitiveData = (message) => {
|
|
30
30
|
for (let value of Object.values(interpolationVars)) {
|
|
31
|
-
|
|
32
31
|
// Avoid replacing vars that are 0 or 1 character
|
|
33
32
|
if (value.length >= 2) {
|
|
34
33
|
message = message.replaceAll(value, "****");
|
|
35
34
|
}
|
|
36
35
|
}
|
|
37
36
|
return message;
|
|
38
|
-
}
|
|
37
|
+
};
|
|
39
38
|
|
|
40
39
|
const logger = winston.createLogger({
|
|
41
|
-
level: shouldLog ?
|
|
40
|
+
level: shouldLog ? "debug" : "info",
|
|
42
41
|
format: winston.format.combine(
|
|
43
42
|
winston.format.splat(),
|
|
44
43
|
winston.format((info) => {
|
|
45
44
|
info.message = censorSensitiveData(info.message);
|
|
46
45
|
return info;
|
|
47
46
|
})(),
|
|
48
|
-
logFormat
|
|
47
|
+
logFormat,
|
|
49
48
|
),
|
|
50
|
-
transports: [
|
|
51
|
-
new winston.transports.Console(),
|
|
52
|
-
],
|
|
49
|
+
transports: [new winston.transports.Console()],
|
|
53
50
|
});
|
|
54
51
|
|
|
55
|
-
|
|
56
52
|
// marked is a markdown parser
|
|
57
53
|
// markedTerminal allows us to render markdown in CLI
|
|
58
54
|
marked.use(
|
|
@@ -97,11 +93,8 @@ const createMarkdownStreamLogger = () => {
|
|
|
97
93
|
diff = censorSensitiveData(diff);
|
|
98
94
|
process.stdout.write(diff);
|
|
99
95
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
96
|
},
|
|
103
97
|
end() {
|
|
104
|
-
|
|
105
98
|
const previousConsoleOutput = markedParsePartial(buffer, 0, -1);
|
|
106
99
|
const consoleOutput = markedParsePartial(buffer, 0, Infinity);
|
|
107
100
|
let diff = consoleOutput.replace(previousConsoleOutput, "");
|
|
@@ -126,8 +119,8 @@ const prettyMarkdown = (markdown) => {
|
|
|
126
119
|
let consoleOutput = marked.parse(markdown);
|
|
127
120
|
|
|
128
121
|
// strip newlines at end of consoleOutput
|
|
129
|
-
consoleOutput = consoleOutput.replace(/\n$/, "");
|
|
130
|
-
consoleOutput = consoleOutput.replace(/^/gm, spaceChar);
|
|
122
|
+
// consoleOutput = consoleOutput.replace(/\n$/, "");
|
|
123
|
+
// consoleOutput = consoleOutput.replace(/^/gm, spaceChar);
|
|
131
124
|
|
|
132
125
|
logger.info(consoleOutput);
|
|
133
126
|
};
|
|
@@ -151,4 +144,3 @@ module.exports = {
|
|
|
151
144
|
prettyMarkdown,
|
|
152
145
|
createMarkdownStreamLogger,
|
|
153
146
|
};
|
|
154
|
-
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testdriverai",
|
|
3
|
-
"version": "5.3.
|
|
3
|
+
"version": "5.3.6",
|
|
4
4
|
"description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"bin": {
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
"axios": "^1.7.7",
|
|
23
23
|
"chalk": "^4.1.2",
|
|
24
24
|
"cli-progress": "^3.12.0",
|
|
25
|
+
"cli-table3": "^0.6.5",
|
|
25
26
|
"datadog-winston": "^1.6.0",
|
|
26
27
|
"decompress": "^4.2.1",
|
|
27
28
|
"dotenv": "^16.4.5",
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
version: 5.3.0
|
|
2
|
-
session: 67fd4b4fd04f2b16ef43c86d
|
|
3
|
-
steps:
|
|
4
|
-
- prompt: /try navigate to airbnb.com
|
|
5
|
-
commands:
|
|
6
|
-
- command: press-keys
|
|
7
|
-
keys:
|
|
8
|
-
- command
|
|
9
|
-
- space
|
|
10
|
-
- prompt: /try navigate to airbnb.com
|
|
11
|
-
commands:
|
|
12
|
-
- command: focus-application
|
|
13
|
-
name: Google Chrome
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|