testronaut 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -2
- package/TODO.md +9 -0
- package/bin/cli.js +265 -35
- package/bin/init.js +5 -2
- package/core/missionDiscovery.js +3 -3
- package/core/missionLoader.js +16 -0
- package/core/turnLoop.js +17 -14
- package/package.json +2 -1
- package/runner/testronaut.js +4 -1
- package/tests/binTests/cli.helpers.test.js +75 -2
- package/tests/binTests/cli.integration.test.js +77 -0
- package/tests/binTests/init.test.js +9 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +3 -0
- package/tests/coreTests/missionDiscovery.test.js +2 -2
- package/tests/coreTests/missionLoader.test.js +52 -0
- package/tests/coreTests/turnLoop.test.js +40 -0
- package/tests/toolsTests/generateHtmlReport.test.js +4 -0
- package/tools/chromeBrowser.js +14 -4
- package/tools/generateHtmlReport.js +9 -5
- package/tools/mfaCode.js +4 -1
package/README.md
CHANGED
|
@@ -58,12 +58,18 @@ testronaut --init
|
|
|
58
58
|
Run the sample mission:
|
|
59
59
|
```bash
|
|
60
60
|
testronaut welcome.mission.js
|
|
61
|
+
|
|
62
|
+
# Paths work too, which enables shell tab completion
|
|
63
|
+
testronaut missions/welcome.mission.js
|
|
61
64
|
```
|
|
62
65
|
|
|
63
66
|
**One-off / no install** (use `npx` to run without installing):
|
|
64
67
|
```bash
|
|
65
68
|
npx testronaut --init
|
|
66
69
|
npx testronaut welcome.mission.js
|
|
70
|
+
|
|
71
|
+
# Paths work too, which enables shell tab completion
|
|
72
|
+
npx testronaut missions/welcome.mission.js
|
|
67
73
|
```
|
|
68
74
|
|
|
69
75
|
---
|
|
@@ -74,9 +80,13 @@ npx testronaut welcome.mission.js
|
|
|
74
80
|
missions/
|
|
75
81
|
├── login.mission.js
|
|
76
82
|
├── logout.mission.js
|
|
77
|
-
└── dashboard.mission.
|
|
83
|
+
└── dashboard.mission.ts
|
|
78
84
|
```
|
|
79
85
|
|
|
86
|
+
Mission files can use standard `import`/`export` syntax in `.js` or `.ts` files.
|
|
87
|
+
Testronaut loads them independently of your project's module format, so your
|
|
88
|
+
`package.json` does not need `"type": "module"`.
|
|
89
|
+
|
|
80
90
|
Each mission exports a string or function and calls `runMissions`.
|
|
81
91
|
|
|
82
92
|
---
|
|
@@ -153,6 +163,20 @@ Run a specific mission:
|
|
|
153
163
|
testronaut login.mission.js
|
|
154
164
|
```
|
|
155
165
|
|
|
166
|
+
Preview or inspect without launching a browser:
|
|
167
|
+
```bash
|
|
168
|
+
testronaut list
|
|
169
|
+
testronaut --dry-run
|
|
170
|
+
testronaut config
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Upload the latest or a selected report:
|
|
174
|
+
```bash
|
|
175
|
+
testronaut upload
|
|
176
|
+
testronaut upload run_1788745106444
|
|
177
|
+
testronaut upload run_1788745106444.json --no-upload-screenshots
|
|
178
|
+
```
|
|
179
|
+
|
|
156
180
|
Chain missions together:
|
|
157
181
|
```js
|
|
158
182
|
await runMissions({
|
|
@@ -243,7 +267,7 @@ Notes:
|
|
|
243
267
|
|
|
244
268
|
## 📋 Reports
|
|
245
269
|
|
|
246
|
-
Testronaut generates
|
|
270
|
+
Testronaut generates JSON, HTML, and run-specific screenshots under the configured `outputDir` (default):
|
|
247
271
|
|
|
248
272
|
```
|
|
249
273
|
missions/mission_reports/
|
package/TODO.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# TODO
|
|
2
2
|
|
|
3
|
+
## Quality-of-Life Follow-Ups
|
|
4
|
+
|
|
5
|
+
- [ ] Add a versioned report schema shared with Mission Control so future report changes can be validated and migrated explicitly.
|
|
6
|
+
- [ ] Add resumable, retryable, and idempotent report uploads so interrupted screenshot transfers do not require starting over or create duplicate reports.
|
|
7
|
+
- [ ] Add `testronaut doctor` to check Node.js, browser installation, configuration, authentication, output-directory access, and API connectivity without running a mission.
|
|
8
|
+
- [ ] Add optional shell completion for commands, flags, mission files, tags, and local report IDs.
|
|
9
|
+
- [ ] Treat `--json` output as a stable automation contract with documented schemas and tests for every supported command.
|
|
10
|
+
- [ ] If authentication ever moves outside project config, continue reading existing `sessionToken` values and provide an opt-in migration path before deprecating legacy storage.
|
|
11
|
+
|
|
3
12
|
## Automated MFA Follow-Ups
|
|
4
13
|
|
|
5
14
|
- [ ] Add an integration smoke test against a local or staging API fixture for `get_mfa_code`.
|
package/bin/cli.js
CHANGED
|
@@ -34,6 +34,9 @@
|
|
|
34
34
|
* See tests in tests/toolsTests/cli.helpers.test.js
|
|
35
35
|
*/
|
|
36
36
|
|
|
37
|
+
// Missions often interpolate project .env values while their modules load.
|
|
38
|
+
// Load those values before discovery imports any user-authored mission code.
|
|
39
|
+
import 'dotenv/config';
|
|
37
40
|
import fs from 'fs';
|
|
38
41
|
import path from 'path';
|
|
39
42
|
import { fileURLToPath } from 'url';
|
|
@@ -51,6 +54,7 @@ const exec = promisify(execCmd);
|
|
|
51
54
|
import url from 'url';
|
|
52
55
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
53
56
|
import { discoverMissionFiles } from '../core/missionDiscovery.js';
|
|
57
|
+
import { loadMissionModule } from '../core/missionLoader.js';
|
|
54
58
|
import { loadConfig } from '../core/config.js';
|
|
55
59
|
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../core/tags.js';
|
|
56
60
|
|
|
@@ -59,12 +63,32 @@ process.env.PLAYWRIGHT_BROWSERS_PATH = process.env.PLAYWRIGHT_BROWSERS_PATH || '
|
|
|
59
63
|
|
|
60
64
|
const TMP_DIR = path.resolve('./missions/tmp');
|
|
61
65
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
66
|
+
const CLI_VERSION = JSON.parse(
|
|
67
|
+
fs.readFileSync(path.resolve(__dirname, '../package.json'), 'utf8')
|
|
68
|
+
).version;
|
|
62
69
|
|
|
63
70
|
const DEFAULT_API_BASE = 'http://api.testronaut.app';
|
|
64
71
|
const DEV_API_BASE = 'https://staging.api.testronaut.app';
|
|
65
72
|
|
|
66
73
|
let args = process.argv.slice(2);
|
|
67
74
|
|
|
75
|
+
const jsonOutput = args.includes('--json');
|
|
76
|
+
const quietOutput = args.includes('--quiet') || jsonOutput;
|
|
77
|
+
args = args.filter(arg => arg !== '--json' && arg !== '--quiet');
|
|
78
|
+
const writeOutput = value => process.stdout.write(`${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}\n`);
|
|
79
|
+
if (quietOutput) {
|
|
80
|
+
console.log = () => {};
|
|
81
|
+
console.warn = () => {};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const dryRun = args.includes('--dry-run');
|
|
85
|
+
args = args.filter(arg => arg !== '--dry-run');
|
|
86
|
+
const screenshotsEnabled = !args.includes('--no-screenshots');
|
|
87
|
+
args = args.filter(arg => arg !== '--no-screenshots');
|
|
88
|
+
const uploadScreenshots = !args.includes('--no-upload-screenshots');
|
|
89
|
+
args = args.filter(arg => arg !== '--no-upload-screenshots');
|
|
90
|
+
process.env.TESTRONAUT_SCREENSHOTS = screenshotsEnabled ? '1' : '0';
|
|
91
|
+
|
|
68
92
|
// Detect how the CLI was invoked so help text matches the actual command
|
|
69
93
|
function detectCliName(npmCommand = process.env.npm_command, argv1 = process.argv[1]) {
|
|
70
94
|
// npx sets npm_command to 'exec'
|
|
@@ -311,8 +335,97 @@ export const __test__ = {
|
|
|
311
335
|
parseRunOptionsArgs,
|
|
312
336
|
detectCliName,
|
|
313
337
|
isDirectInvocation,
|
|
338
|
+
resolveMissionPath,
|
|
339
|
+
collectReportScreenshotNames,
|
|
340
|
+
collectReportScreenshotPaths,
|
|
341
|
+
resolveReportDir,
|
|
342
|
+
closestMissionMatch,
|
|
343
|
+
resolveReportFile,
|
|
344
|
+
buildEffectiveConfig,
|
|
314
345
|
};
|
|
315
346
|
|
|
347
|
+
function resolveReportDir(config = {}, cwd = process.cwd()) {
|
|
348
|
+
return path.resolve(cwd, config.outputDir || 'missions/mission_reports');
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function resolveReportFile(requested, reportDir, cwd = process.cwd()) {
|
|
352
|
+
if (!requested) return null;
|
|
353
|
+
const names = path.extname(requested) ? [requested] : [requested, `${requested}.json`];
|
|
354
|
+
for (const name of names) {
|
|
355
|
+
const direct = path.resolve(cwd, name);
|
|
356
|
+
if (fs.existsSync(direct)) return direct;
|
|
357
|
+
const inReportDir = path.resolve(reportDir, name);
|
|
358
|
+
if (fs.existsSync(inReportDir)) return inReportDir;
|
|
359
|
+
}
|
|
360
|
+
return path.resolve(reportDir, names.at(-1));
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function buildEffectiveConfig(config = {}, cwd = process.cwd()) {
|
|
364
|
+
const resolved = resolveProviderModel({ cwd });
|
|
365
|
+
const source = key => process.env[key] ? 'environment' : undefined;
|
|
366
|
+
return {
|
|
367
|
+
configFile: path.resolve(cwd, 'testronaut-config.json'),
|
|
368
|
+
provider: { value: resolved.provider, source: source('TESTRONAUT_PROVIDER') || (config.provider ? 'config' : 'default') },
|
|
369
|
+
model: { value: resolved.model, source: source('TESTRONAUT_MODEL') || (config.model ? 'config' : 'default') },
|
|
370
|
+
outputDir: { value: resolveReportDir(config, cwd), source: config.outputDir ? 'config' : 'default' },
|
|
371
|
+
maxTurns: { value: Number(process.env.TESTRONAUT_TURNS || config.maxTurns || 20), source: source('TESTRONAUT_TURNS') || (config.maxTurns != null ? 'config' : 'default') },
|
|
372
|
+
tags: config.tags || [],
|
|
373
|
+
tagMatch: config.tagMatch || 'any',
|
|
374
|
+
addTags: normalizeTags([...(config.addTags || []), ...cliAddTags]),
|
|
375
|
+
screenshots: screenshotsEnabled,
|
|
376
|
+
authenticated: Boolean(config.sessionToken),
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function closestMissionMatch(requested, candidates = []) {
|
|
381
|
+
const target = path.basename(requested).toLowerCase();
|
|
382
|
+
const distance = (a, b) => {
|
|
383
|
+
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
384
|
+
for (let i = 1; i <= a.length; i++) {
|
|
385
|
+
let previous = row[0];
|
|
386
|
+
row[0] = i;
|
|
387
|
+
for (let j = 1; j <= b.length; j++) {
|
|
388
|
+
const saved = row[j];
|
|
389
|
+
row[j] = Math.min(row[j] + 1, row[j - 1] + 1, previous + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
390
|
+
previous = saved;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return row[b.length];
|
|
394
|
+
};
|
|
395
|
+
return candidates
|
|
396
|
+
.map(file => ({ file, distance: distance(target, path.basename(file).toLowerCase()) }))
|
|
397
|
+
.sort((a, b) => a.distance - b.distance || a.file.localeCompare(b.file))[0]?.file || null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function resolveMissionPath(filePath, { cwd = process.cwd(), missionsRoot } = {}) {
|
|
401
|
+
if (path.isAbsolute(filePath)) return path.normalize(filePath);
|
|
402
|
+
|
|
403
|
+
const directPath = path.resolve(cwd, filePath);
|
|
404
|
+
const hasPathSegments = path.dirname(filePath) !== '.';
|
|
405
|
+
if (hasPathSegments || fs.existsSync(directPath)) return directPath;
|
|
406
|
+
|
|
407
|
+
return path.resolve(missionsRoot || path.join(cwd, 'missions'), filePath);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function collectReportScreenshotNames(report) {
|
|
411
|
+
return collectReportScreenshotPaths(report).map(file => path.basename(file));
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function collectReportScreenshotPaths(report) {
|
|
415
|
+
const names = [];
|
|
416
|
+
const seen = new Set();
|
|
417
|
+
for (const mission of Array.isArray(report?.missions) ? report.missions : []) {
|
|
418
|
+
for (const step of Array.isArray(mission?.steps) ? mission.steps : []) {
|
|
419
|
+
if (typeof step?.screenshotPath !== 'string') continue;
|
|
420
|
+
const name = step.screenshotPath.trim().replace(/^\.\//, '').replaceAll('\\', '/');
|
|
421
|
+
if (!name || seen.has(name)) continue;
|
|
422
|
+
seen.add(name);
|
|
423
|
+
names.push(name);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return names;
|
|
427
|
+
}
|
|
428
|
+
|
|
316
429
|
function parseTagArgs(argsList) {
|
|
317
430
|
const nextArgs = [];
|
|
318
431
|
const tagValues = [];
|
|
@@ -603,9 +716,11 @@ const HELP_TEXT = `
|
|
|
603
716
|
|
|
604
717
|
Usage:
|
|
605
718
|
${cliName} Run all missions in the ./missions directory
|
|
606
|
-
${cliName} <file> Run a
|
|
719
|
+
${cliName} <file> Run a .mission.js or .mission.ts file by name or path
|
|
607
720
|
${cliName} login Log in and store session token
|
|
608
|
-
${cliName}
|
|
721
|
+
${cliName} list List discovered missions and tags without running them
|
|
722
|
+
${cliName} config Show effective configuration and value sources
|
|
723
|
+
${cliName} upload [file] Upload the latest report or a selected report file/run ID
|
|
609
724
|
${cliName} serve Serve & open the most recent HTML report (read-only)
|
|
610
725
|
${cliName} view Alias of 'serve'
|
|
611
726
|
|
|
@@ -621,6 +736,11 @@ Options:
|
|
|
621
736
|
--no-human-input Disable human-in-the-loop verification prompts for automated runs
|
|
622
737
|
--human-input-timeout=<s> Override human input wait timeout in seconds (default: 60)
|
|
623
738
|
--help Show this help message
|
|
739
|
+
--dry-run Show the resolved mission plan without launching a browser
|
|
740
|
+
--no-screenshots Remove the screenshot tool for this run
|
|
741
|
+
--no-upload-screenshots Upload report JSON without its screenshots
|
|
742
|
+
--json Emit machine-readable command/run output
|
|
743
|
+
--quiet Suppress informational logs
|
|
624
744
|
--retry_limit=<n> Override agent turn retry limits (minimum 1, maximum 10)
|
|
625
745
|
--tag=<tag> Run missions matching a tag; repeat for multiple tags
|
|
626
746
|
--tags=<tag,...> Compact comma-list form (OR/any by default)
|
|
@@ -631,6 +751,7 @@ Options:
|
|
|
631
751
|
Examples:
|
|
632
752
|
${cliName}
|
|
633
753
|
${cliName} login
|
|
754
|
+
${cliName} list
|
|
634
755
|
${cliName} upload
|
|
635
756
|
${cliName} serve
|
|
636
757
|
${cliName} --init
|
|
@@ -689,8 +810,8 @@ if (args.includes('login')) {
|
|
|
689
810
|
}
|
|
690
811
|
|
|
691
812
|
// Handle the upload command
|
|
692
|
-
if (args
|
|
693
|
-
await uploadReport();
|
|
813
|
+
if (args[0] === 'upload') {
|
|
814
|
+
await uploadReport(args[1], { uploadScreenshots });
|
|
694
815
|
process.exit(0);
|
|
695
816
|
}
|
|
696
817
|
|
|
@@ -704,6 +825,37 @@ if (args.includes('serve') || args.includes('view')) {
|
|
|
704
825
|
|
|
705
826
|
const { root: missionsRoot, files: discoveredMissions } = await discoverMissionFiles({ cwd: process.cwd() });
|
|
706
827
|
const tagConfig = await loadConfig(process.cwd());
|
|
828
|
+
const outputDir = resolveReportDir(tagConfig);
|
|
829
|
+
process.env.TESTRONAUT_OUTPUT_DIR = outputDir;
|
|
830
|
+
process.env.TESTRONAUT_RUN_ID = runId;
|
|
831
|
+
|
|
832
|
+
if (args.length === 1 && args[0] === 'config') {
|
|
833
|
+
writeOutput(buildEffectiveConfig(tagConfig));
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
if (args.length === 1 && args[0] === 'list') {
|
|
838
|
+
if (!discoveredMissions.length) {
|
|
839
|
+
writeOutput(jsonOutput ? { missions: [], root: missionsRoot } : `No missions found in ${path.relative(process.cwd(), missionsRoot) || '.'}.`);
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
const listedMissions = [];
|
|
843
|
+
if (!jsonOutput) console.log(`Missions in ${path.relative(process.cwd(), missionsRoot) || '.'}:`);
|
|
844
|
+
for (const file of discoveredMissions) {
|
|
845
|
+
try {
|
|
846
|
+
const mission = await loadMissionModule(path.resolve(missionsRoot, file));
|
|
847
|
+
const tags = normalizeTags(mission.tags);
|
|
848
|
+
listedMissions.push({ file, tags });
|
|
849
|
+
if (!jsonOutput) console.log(` ${file}${tags.length ? ` [${tags.join(', ')}]` : ''}`);
|
|
850
|
+
} catch (error) {
|
|
851
|
+
console.log(` ${file} [could not load: ${error.message}]`);
|
|
852
|
+
process.exitCode = 1;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
if (jsonOutput) writeOutput({ root: missionsRoot, missions: listedMissions });
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
|
|
707
859
|
const explicitFiles = args.length > 0;
|
|
708
860
|
let requestedTags = [];
|
|
709
861
|
let tagMatch = 'any';
|
|
@@ -719,32 +871,71 @@ try {
|
|
|
719
871
|
process.exit(1);
|
|
720
872
|
}
|
|
721
873
|
|
|
722
|
-
if (!fs.existsSync(missionsRoot)) {
|
|
874
|
+
if (!explicitFiles && !fs.existsSync(missionsRoot)) {
|
|
723
875
|
console.error(`❌ Missions directory not found: ${path.relative(process.cwd(), missionsRoot)}`);
|
|
724
876
|
process.exit(1);
|
|
725
877
|
}
|
|
726
878
|
|
|
879
|
+
if (dryRun) {
|
|
880
|
+
const candidates = explicitFiles ? args : discoveredMissions;
|
|
881
|
+
const missions = [];
|
|
882
|
+
let invalid = false;
|
|
883
|
+
for (const file of candidates) {
|
|
884
|
+
const modulePath = resolveMissionPath(file, { cwd: process.cwd(), missionsRoot });
|
|
885
|
+
if (!fs.existsSync(modulePath)) {
|
|
886
|
+
missions.push({ file, status: 'missing', suggestion: closestMissionMatch(file, discoveredMissions) });
|
|
887
|
+
invalid = true;
|
|
888
|
+
continue;
|
|
889
|
+
}
|
|
890
|
+
try {
|
|
891
|
+
const mission = await loadMissionModule(modulePath);
|
|
892
|
+
const tags = normalizeTags(mission.tags);
|
|
893
|
+
const selected = explicitFiles || matchesTagFilter(tags, requestedTags, tagMatch);
|
|
894
|
+
missions.push({ file: path.relative(process.cwd(), modulePath), tags, selected, valid: typeof mission.executeMission === 'function' });
|
|
895
|
+
if (typeof mission.executeMission !== 'function') invalid = true;
|
|
896
|
+
} catch (error) {
|
|
897
|
+
missions.push({ file, status: 'load-error', error: error.message });
|
|
898
|
+
invalid = true;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
writeOutput({ dryRun: true, outputDir, provider: resolveProviderModel({ cwd: process.cwd() }), screenshots: screenshotsEnabled, missions });
|
|
902
|
+
if (invalid) process.exitCode = 1;
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
|
|
727
906
|
const runFile = async (filePath) => {
|
|
907
|
+
const modulePath = resolveMissionPath(filePath, { cwd: process.cwd(), missionsRoot });
|
|
728
908
|
try {
|
|
729
|
-
|
|
730
|
-
|
|
909
|
+
if (!fs.existsSync(modulePath)) {
|
|
910
|
+
const suggestion = closestMissionMatch(filePath, discoveredMissions);
|
|
911
|
+
console.error(`❌ Mission file not found: ${filePath}`);
|
|
912
|
+
if (suggestion) console.error(` Did you mean "${suggestion}"?`);
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
const missionsModule = await loadMissionModule(modulePath);
|
|
731
916
|
|
|
732
917
|
if (!explicitFiles) {
|
|
733
918
|
const moduleTags = normalizeTags(missionsModule.tags);
|
|
734
|
-
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return
|
|
919
|
+
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return null;
|
|
735
920
|
}
|
|
736
921
|
|
|
737
922
|
if (typeof missionsModule.executeMission === 'function') {
|
|
738
923
|
const result = await missionsModule.executeMission();
|
|
924
|
+
if (result == null) {
|
|
925
|
+
console.error(`❌ Mission did not return a result: ${filePath}`);
|
|
926
|
+
return false;
|
|
927
|
+
}
|
|
739
928
|
allResults.push({
|
|
740
|
-
file:
|
|
929
|
+
file: path.relative(process.cwd(), modulePath) || path.basename(modulePath),
|
|
741
930
|
result
|
|
742
931
|
});
|
|
743
|
-
|
|
932
|
+
const missionResults = Array.isArray(result) ? result : [result];
|
|
933
|
+
return !missionResults.some(item => item?.status === 'failed');
|
|
744
934
|
}
|
|
935
|
+
console.error(`❌ Mission does not export executeMission(): ${filePath}`);
|
|
745
936
|
} catch (err) {
|
|
746
937
|
console.error(`❌ Error running mission: ${filePath}`);
|
|
747
|
-
console.error(err);
|
|
938
|
+
console.error(` ${err?.message || err}`);
|
|
748
939
|
}
|
|
749
940
|
return false;
|
|
750
941
|
};
|
|
@@ -752,20 +943,27 @@ const runFile = async (filePath) => {
|
|
|
752
943
|
if (args.length > 0) {
|
|
753
944
|
// Run specific file(s)
|
|
754
945
|
for (const file of args) {
|
|
755
|
-
await runFile(file);
|
|
946
|
+
if (!await runFile(file)) process.exitCode = 1;
|
|
756
947
|
}
|
|
757
948
|
} else {
|
|
758
949
|
// Run missions discovered from config (or default behavior)
|
|
759
950
|
for (const file of discoveredMissions) {
|
|
760
|
-
await runFile(file);
|
|
951
|
+
if (await runFile(file) === false) process.exitCode = 1;
|
|
761
952
|
}
|
|
762
953
|
}
|
|
763
954
|
|
|
955
|
+
if (!explicitFiles && discoveredMissions.length === 0) {
|
|
956
|
+
console.error(`❌ No mission files found in ${path.relative(process.cwd(), missionsRoot) || '.'}.`);
|
|
957
|
+
process.exit(1);
|
|
958
|
+
}
|
|
959
|
+
|
|
764
960
|
if (!explicitFiles && requestedTags.length && allResults.length === 0) {
|
|
765
961
|
console.error(`❌ No missions matched ${tagMatch === 'all' ? 'all' : 'any'} of: ${requestedTags.join(', ')}`);
|
|
766
962
|
process.exit(1);
|
|
767
963
|
}
|
|
768
964
|
|
|
965
|
+
if (allResults.length === 0) return;
|
|
966
|
+
|
|
769
967
|
const endTime = new Date();
|
|
770
968
|
|
|
771
969
|
const flatMissions = allResults.flatMap(entry => {
|
|
@@ -794,6 +992,7 @@ const { provider: llmProvider, model: llmModel } = resolveProviderModel({ cwd: p
|
|
|
794
992
|
|
|
795
993
|
const report = {
|
|
796
994
|
runId,
|
|
995
|
+
cli: { version: CLI_VERSION },
|
|
797
996
|
startTime: startTime.toISOString(),
|
|
798
997
|
endTime: endTime.toISOString(),
|
|
799
998
|
llm: {
|
|
@@ -811,10 +1010,10 @@ report.tags = normalizeTags(flatMissions.flatMap(m =>
|
|
|
811
1010
|
m.submissionType === 'mission' ? (m.tags ?? []) : []
|
|
812
1011
|
));
|
|
813
1012
|
|
|
814
|
-
const outputDir = './missions/mission_reports';
|
|
815
1013
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
816
|
-
fs.writeFileSync(`${
|
|
817
|
-
generateHtmlReport(report, `${
|
|
1014
|
+
fs.writeFileSync(path.join(outputDir, `${runId}.json`), JSON.stringify(report, null, 2));
|
|
1015
|
+
generateHtmlReport(report, path.join(outputDir, `${runId}.html`));
|
|
1016
|
+
if (jsonOutput) writeOutput(report);
|
|
818
1017
|
|
|
819
1018
|
try {
|
|
820
1019
|
if (!process.env.TN_KEEP_TMP && fs.existsSync(TMP_DIR)) {
|
|
@@ -1010,7 +1209,7 @@ async function handleLogin() {
|
|
|
1010
1209
|
}
|
|
1011
1210
|
|
|
1012
1211
|
// Upload the most recent report
|
|
1013
|
-
async function uploadReport() {
|
|
1212
|
+
async function uploadReport(requestedReport, { uploadScreenshots: shouldUploadScreenshots = true } = {}) {
|
|
1014
1213
|
const configPath = path.resolve(process.cwd(), 'testronaut-config.json');
|
|
1015
1214
|
if (!fs.existsSync(configPath)) {
|
|
1016
1215
|
console.error('❌ Configuration file not found.');
|
|
@@ -1025,16 +1224,29 @@ async function uploadReport() {
|
|
|
1025
1224
|
|
|
1026
1225
|
|
|
1027
1226
|
// 1) Find latest report JSON
|
|
1028
|
-
const reportDir =
|
|
1029
|
-
|
|
1030
|
-
|
|
1227
|
+
const reportDir = resolveReportDir(config);
|
|
1228
|
+
if (!requestedReport && !fs.existsSync(reportDir)) {
|
|
1229
|
+
console.error(`❌ Report directory not found: ${path.relative(process.cwd(), reportDir) || reportDir}`);
|
|
1230
|
+
process.exit(1);
|
|
1231
|
+
}
|
|
1232
|
+
const files = fs.existsSync(reportDir) ? fs.readdirSync(reportDir).filter(f => f.endsWith('.json')) : [];
|
|
1233
|
+
if (!requestedReport && files.length === 0) {
|
|
1031
1234
|
console.error('❌ No report files found.');
|
|
1032
1235
|
process.exit(1);
|
|
1033
1236
|
}
|
|
1034
1237
|
files.sort((a, b) => parseInt(b.split('_')[1]) - parseInt(a.split('_')[1]));
|
|
1035
|
-
const
|
|
1036
|
-
|
|
1238
|
+
const selectedReportPath = requestedReport
|
|
1239
|
+
? resolveReportFile(requestedReport, reportDir)
|
|
1240
|
+
: path.join(reportDir, files[0]);
|
|
1241
|
+
if (!fs.existsSync(selectedReportPath)) {
|
|
1242
|
+
console.error(`❌ Report file not found: ${requestedReport}`);
|
|
1243
|
+
process.exit(1);
|
|
1244
|
+
}
|
|
1245
|
+
const latestReportPath = selectedReportPath;
|
|
1246
|
+
const latestReportFile = path.basename(selectedReportPath);
|
|
1037
1247
|
const reportJson = fs.readFileSync(latestReportPath, 'utf8');
|
|
1248
|
+
const report = JSON.parse(reportJson);
|
|
1249
|
+
const selectedReportDir = path.dirname(selectedReportPath);
|
|
1038
1250
|
|
|
1039
1251
|
// 2) Upload report FIRST and capture its ID
|
|
1040
1252
|
console.log(`🛰️ Uploading report: ${latestReportFile}`);
|
|
@@ -1061,10 +1273,17 @@ async function uploadReport() {
|
|
|
1061
1273
|
process.exit(1);
|
|
1062
1274
|
}
|
|
1063
1275
|
|
|
1276
|
+
if (!shouldUploadScreenshots) {
|
|
1277
|
+
console.log('ℹ️ Screenshot upload disabled. Report JSON upload complete.');
|
|
1278
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1064
1282
|
// 3) Now find the fixed 'screenshots' folder next to JSON
|
|
1065
|
-
const screenshotsDir = path.join(
|
|
1283
|
+
const screenshotsDir = path.join(selectedReportDir, 'screenshots');
|
|
1066
1284
|
if (!fs.existsSync(screenshotsDir) || !fs.statSync(screenshotsDir).isDirectory()) {
|
|
1067
1285
|
console.log('ℹ️ No screenshots directory found. Done.');
|
|
1286
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1068
1287
|
return;
|
|
1069
1288
|
}
|
|
1070
1289
|
|
|
@@ -1076,13 +1295,22 @@ async function uploadReport() {
|
|
|
1076
1295
|
return Date.UTC(+Y, +M - 1, +D, +h, +mnt, +s, +ms);
|
|
1077
1296
|
};
|
|
1078
1297
|
|
|
1079
|
-
|
|
1080
|
-
.
|
|
1081
|
-
|
|
1082
|
-
|
|
1298
|
+
const imageFiles = collectReportScreenshotPaths(report)
|
|
1299
|
+
.map(relativePath => ({
|
|
1300
|
+
relativePath,
|
|
1301
|
+
filePath: path.resolve(selectedReportDir, relativePath),
|
|
1302
|
+
}))
|
|
1303
|
+
.filter(({ filePath }) =>
|
|
1304
|
+
filePath.startsWith(`${path.resolve(screenshotsDir)}${path.sep}`) &&
|
|
1305
|
+
fs.existsSync(filePath) &&
|
|
1306
|
+
fs.statSync(filePath).isFile() &&
|
|
1307
|
+
/\.(png|jpg|jpeg|webp|gif)$/i.test(filePath)
|
|
1308
|
+
)
|
|
1309
|
+
.sort((a, b) => parseScreenshotTimestamp(path.basename(a.filePath)) - parseScreenshotTimestamp(path.basename(b.filePath)));
|
|
1083
1310
|
|
|
1084
1311
|
if (!imageFiles.length) {
|
|
1085
|
-
console.log('ℹ️
|
|
1312
|
+
console.log('ℹ️ This report does not reference any available screenshots. Done.');
|
|
1313
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1086
1314
|
return;
|
|
1087
1315
|
}
|
|
1088
1316
|
|
|
@@ -1101,11 +1329,10 @@ async function uploadReport() {
|
|
|
1101
1329
|
|
|
1102
1330
|
const failures = [];
|
|
1103
1331
|
for (let i = 0; i < imageFiles.length; i++) {
|
|
1104
|
-
const f = imageFiles[i];
|
|
1105
|
-
const filePath = path.join(screenshotsDir, f);
|
|
1332
|
+
const { filePath, relativePath: f } = imageFiles[i];
|
|
1106
1333
|
const stepIndex = i;
|
|
1107
1334
|
|
|
1108
|
-
process.stdout.write(` ➜ ${f} (step ${stepIndex}) … `);
|
|
1335
|
+
if (!quietOutput) process.stdout.write(` ➜ ${f} (step ${stepIndex}) … `);
|
|
1109
1336
|
try {
|
|
1110
1337
|
const buf = fs.readFileSync(filePath);
|
|
1111
1338
|
const stat = fs.statSync(filePath);
|
|
@@ -1154,9 +1381,9 @@ async function uploadReport() {
|
|
|
1154
1381
|
const finish = await parseJsonSafe(finishRes, 'uploads/finish');
|
|
1155
1382
|
if (!finish.ok) throw new Error(`uploads/finish responded ok=false`);
|
|
1156
1383
|
|
|
1157
|
-
process.stdout.write('ok\n');
|
|
1384
|
+
if (!quietOutput) process.stdout.write('ok\n');
|
|
1158
1385
|
} catch (err) {
|
|
1159
|
-
process.stdout.write('FAIL\n');
|
|
1386
|
+
if (!quietOutput) process.stdout.write('FAIL\n');
|
|
1160
1387
|
failures.push({ file: f, error: err.message || String(err) });
|
|
1161
1388
|
}
|
|
1162
1389
|
}
|
|
@@ -1165,8 +1392,10 @@ async function uploadReport() {
|
|
|
1165
1392
|
console.log('\n⚠️ Some screenshots failed to upload:');
|
|
1166
1393
|
for (const f of failures) console.log(` - ${f.file}: ${f.error}`);
|
|
1167
1394
|
process.exitCode = 1;
|
|
1395
|
+
if (jsonOutput) writeOutput({ ok: false, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: imageFiles.length - failures.length, failures });
|
|
1168
1396
|
} else {
|
|
1169
1397
|
console.log('✅ All screenshots uploaded.');
|
|
1398
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: imageFiles.length });
|
|
1170
1399
|
}
|
|
1171
1400
|
}
|
|
1172
1401
|
|
|
@@ -1248,11 +1477,12 @@ function findLatestReportPair(reportDir) {
|
|
|
1248
1477
|
}
|
|
1249
1478
|
|
|
1250
1479
|
async function serveLatestReport() {
|
|
1251
|
-
const
|
|
1480
|
+
const config = await loadConfig(process.cwd());
|
|
1481
|
+
const reportDir = resolveReportDir(config);
|
|
1252
1482
|
|
|
1253
1483
|
const latest = findLatestReportPair(reportDir);
|
|
1254
1484
|
if (!latest) {
|
|
1255
|
-
console.error(
|
|
1485
|
+
console.error(`❌ No HTML reports found in ${path.relative(process.cwd(), reportDir) || reportDir}.`);
|
|
1256
1486
|
process.exit(1);
|
|
1257
1487
|
}
|
|
1258
1488
|
|
package/bin/init.js
CHANGED
|
@@ -43,7 +43,6 @@ export async function initializeTestronautProject() {
|
|
|
43
43
|
// ─────────────────────────────────────────────
|
|
44
44
|
const root = process.cwd();
|
|
45
45
|
const missionsDir = path.join(root, 'missions');
|
|
46
|
-
const reportsDir = path.join(missionsDir, 'mission_reports');
|
|
47
46
|
const configPath = path.join(root, 'testronaut-config.json');
|
|
48
47
|
const envPath = path.join(root, '.env');
|
|
49
48
|
|
|
@@ -139,6 +138,9 @@ export async function initializeTestronautProject() {
|
|
|
139
138
|
config.outputDir = config.outputDir || defaults.outputDir;
|
|
140
139
|
config.projectName = config.projectName || defaults.projectName;
|
|
141
140
|
config.maxTurns = config.maxTurns ?? defaults.maxTurns;
|
|
141
|
+
config.tags = config.tags ?? defaults.tags;
|
|
142
|
+
config.tagMatch = config.tagMatch ?? defaults.tagMatch;
|
|
143
|
+
config.addTags = config.addTags ?? defaults.addTags;
|
|
142
144
|
config.dom = { ...defaults.dom, ...(config.dom || {}) };
|
|
143
145
|
config.resourceGuard = { ...defaults.resourceGuard, ...(config.resourceGuard || {}) };
|
|
144
146
|
config.humanInput = { ...defaults.humanInput, ...(config.humanInput || {}) };
|
|
@@ -197,12 +199,13 @@ export async function initializeTestronautProject() {
|
|
|
197
199
|
// ─────────────────────────────────────────────
|
|
198
200
|
// STEP 5: Ensure folder structure exists (safe to call every run)
|
|
199
201
|
// ─────────────────────────────────────────────
|
|
202
|
+
const reportsDir = path.resolve(root, config.outputDir || 'missions/mission_reports');
|
|
200
203
|
if (!fs.existsSync(missionsDir)) {
|
|
201
204
|
fs.mkdirSync(missionsDir);
|
|
202
205
|
console.log('📁 Created missions/');
|
|
203
206
|
}
|
|
204
207
|
if (!fs.existsSync(reportsDir)) {
|
|
205
208
|
fs.mkdirSync(reportsDir);
|
|
206
|
-
console.log(
|
|
209
|
+
console.log(`📁 Created ${path.relative(root, reportsDir) || reportsDir}/`);
|
|
207
210
|
}
|
|
208
211
|
}
|
package/core/missionDiscovery.js
CHANGED
|
@@ -9,7 +9,7 @@ const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
|
|
|
9
9
|
/**
|
|
10
10
|
* Discover mission files based on testronaut-config.json.
|
|
11
11
|
* - When no missions block exists, preserve legacy behavior:
|
|
12
|
-
* read the ./missions folder (non-recursive) for
|
|
12
|
+
* read the ./missions folder (non-recursive) for JavaScript/TypeScript missions.
|
|
13
13
|
* - When missions is defined, apply root/include/exclude globs relative to root.
|
|
14
14
|
*
|
|
15
15
|
* @param {object} opts
|
|
@@ -21,13 +21,13 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
|
|
|
21
21
|
const cfg = await loadConfig(cwd);
|
|
22
22
|
const missionsCfg = cfg?.missions;
|
|
23
23
|
|
|
24
|
-
// Legacy behavior
|
|
24
|
+
// Legacy behavior remains non-recursive, now with first-class TypeScript support.
|
|
25
25
|
if (!missionsCfg) {
|
|
26
26
|
if (!fs.existsSync(defaultRoot)) {
|
|
27
27
|
return { root: defaultRoot, files: [] };
|
|
28
28
|
}
|
|
29
29
|
const files = fs.readdirSync(defaultRoot)
|
|
30
|
-
.filter(f => f.endsWith('.mission.js'))
|
|
30
|
+
.filter(f => f.endsWith('.mission.js') || f.endsWith('.mission.ts'))
|
|
31
31
|
.sort();
|
|
32
32
|
return { root: defaultRoot, files };
|
|
33
33
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { createJiti } from 'jiti';
|
|
2
|
+
|
|
3
|
+
// Mission files are user-authored configuration/code, so Testronaut owns their
|
|
4
|
+
// module loading semantics instead of inheriting the host project's package type.
|
|
5
|
+
const jiti = createJiti(import.meta.url, {
|
|
6
|
+
interopDefault: true,
|
|
7
|
+
sourceMaps: true,
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Load an ESM-style JavaScript or TypeScript mission regardless of whether the
|
|
12
|
+
* consuming project is ESM, CommonJS, or has no package.json `type` field.
|
|
13
|
+
*/
|
|
14
|
+
export async function loadMissionModule(modulePath) {
|
|
15
|
+
return jiti.import(modulePath);
|
|
16
|
+
}
|
package/core/turnLoop.js
CHANGED
|
@@ -156,6 +156,8 @@ const FIRE_AND_FORGET_TOOLS = new Set([
|
|
|
156
156
|
'switch_to_page',
|
|
157
157
|
'close_current_page',
|
|
158
158
|
'request_human_input',
|
|
159
|
+
'set_ground_control_state',
|
|
160
|
+
'record_mission_telemetry',
|
|
159
161
|
]);
|
|
160
162
|
|
|
161
163
|
/**
|
|
@@ -252,9 +254,10 @@ export const turnLoop = async (
|
|
|
252
254
|
const maxAttempts = retryLimitClamped + 1; // includes initial attempt
|
|
253
255
|
ctx.groundControl = groundControl;
|
|
254
256
|
const humanInput = ctx.humanInput || { enabled: true, timeoutSeconds: 60 };
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
257
|
+
const disabledTools = new Set();
|
|
258
|
+
if (humanInput.enabled === false) disabledTools.add('request_human_input');
|
|
259
|
+
if (process.env.TESTRONAUT_SCREENSHOTS === '0') disabledTools.add('screenshot');
|
|
260
|
+
const activeToolsSchema = toolsSchema.filter(t => !disabledTools.has(t?.function?.name));
|
|
258
261
|
let agentMemory = { lastMenuExpanded: false, humanInput };
|
|
259
262
|
ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
260
263
|
let turnRetries = 0;
|
|
@@ -433,7 +436,17 @@ export const turnLoop = async (
|
|
|
433
436
|
let result;
|
|
434
437
|
let errorMessage = null;
|
|
435
438
|
try {
|
|
436
|
-
|
|
439
|
+
if (fnName === 'set_ground_control_state') {
|
|
440
|
+
applyGroundControlUpdate(groundControl, args);
|
|
441
|
+
result = { ok: true, groundControl };
|
|
442
|
+
} else if (fnName === 'record_mission_telemetry') {
|
|
443
|
+
const recorded = recordGroundTelemetry(groundControl, args, { turn });
|
|
444
|
+
result = { ok: true, recorded };
|
|
445
|
+
} else {
|
|
446
|
+
const toolHandler = CHROME_TOOL_MAP[fnName];
|
|
447
|
+
if (typeof toolHandler !== 'function') throw new Error(`Unknown tool: ${fnName}`);
|
|
448
|
+
result = await toolHandler(browser, args, agentMemory);
|
|
449
|
+
}
|
|
437
450
|
if (typeof result !== 'string') result = JSON.stringify(result ?? '');
|
|
438
451
|
} catch (e) {
|
|
439
452
|
errorMessage = `ERROR: ${e.message}`;
|
|
@@ -649,16 +662,6 @@ export const turnLoop = async (
|
|
|
649
662
|
}
|
|
650
663
|
}
|
|
651
664
|
|
|
652
|
-
if (fnName === 'set_ground_control_state') {
|
|
653
|
-
applyGroundControlUpdate(groundControl, args);
|
|
654
|
-
result = JSON.stringify({ ok: true, groundControl });
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
if (fnName === 'record_mission_telemetry') {
|
|
658
|
-
const recorded = recordGroundTelemetry(groundControl, args, { turn });
|
|
659
|
-
result = JSON.stringify({ ok: true, recorded });
|
|
660
|
-
}
|
|
661
|
-
|
|
662
665
|
// Decide what to send back to the LLM for this tool.
|
|
663
666
|
// - For "fire-and-forget" tools, send a tiny stub (OK/error) to avoid
|
|
664
667
|
// bloating context with large payloads (file JSON, etc.).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.6.
|
|
3
|
+
"version": "1.6.2",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"testronaut": "bin/cli.js"
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"dotenv": "^17.2.0",
|
|
31
31
|
"fast-glob": "^3.3.3",
|
|
32
32
|
"inquirer": "^12.9.1",
|
|
33
|
+
"jiti": "^2.7.0",
|
|
33
34
|
"node-fetch": "^3.3.2",
|
|
34
35
|
"openai": "^6.39.0",
|
|
35
36
|
"playwright": "^1.54.1",
|
package/runner/testronaut.js
CHANGED
|
@@ -23,9 +23,11 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
// Mission modules commonly interpolate project .env values at module load time.
|
|
27
|
+
// Keep env loading eager even though the browser/agent import below is lazy.
|
|
28
|
+
import 'dotenv/config';
|
|
26
29
|
import fs from 'fs';
|
|
27
30
|
import path from 'path';
|
|
28
|
-
import { runAgent } from '../core/agent.js';
|
|
29
31
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
32
|
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
33
|
import { normalizeTags } from '../core/tags.js';
|
|
@@ -202,6 +204,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
202
204
|
);
|
|
203
205
|
|
|
204
206
|
// 3) Execute
|
|
207
|
+
const { runAgent } = await import('../core/agent.js');
|
|
205
208
|
const success = await runAgent(
|
|
206
209
|
goals,
|
|
207
210
|
missionName,
|
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
1
|
+
import { afterEach, describe, it, expect } from 'vitest';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import fs from 'fs';
|
|
4
|
+
import os from 'os';
|
|
4
5
|
|
|
5
6
|
// Import helper exports from cli.js
|
|
6
7
|
import { __test__ } from '../../bin/cli.js';
|
|
7
8
|
|
|
9
|
+
const tempDirs = [];
|
|
10
|
+
const makeTempDir = (prefix) => {
|
|
11
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
12
|
+
tempDirs.push(dir);
|
|
13
|
+
return dir;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
18
|
+
});
|
|
19
|
+
|
|
8
20
|
describe('cli helpers', () => {
|
|
9
21
|
it('guessMimeType returns expected types', () => {
|
|
10
22
|
const { guessMimeType } = __test__;
|
|
@@ -22,7 +34,7 @@ describe('cli helpers', () => {
|
|
|
22
34
|
|
|
23
35
|
it('findLatestReportPair finds latest html/json', () => {
|
|
24
36
|
const { findLatestReportPair } = __test__;
|
|
25
|
-
const tmp =
|
|
37
|
+
const tmp = makeTempDir('testronaut-report-');
|
|
26
38
|
const r1 = path.join(tmp, 'run_100.html');
|
|
27
39
|
const r2 = path.join(tmp, 'run_200.html');
|
|
28
40
|
fs.writeFileSync(r1, 'a');
|
|
@@ -34,6 +46,67 @@ describe('cli helpers', () => {
|
|
|
34
46
|
expect(latest?.jsonFile).toBe('run_200.json');
|
|
35
47
|
});
|
|
36
48
|
|
|
49
|
+
it('resolves bare mission names from the missions root and direct paths from cwd', () => {
|
|
50
|
+
const { resolveMissionPath } = __test__;
|
|
51
|
+
const cwd = '/tmp/project';
|
|
52
|
+
const missionsRoot = '/tmp/project/missions';
|
|
53
|
+
expect(resolveMissionPath('login.mission.js', { cwd, missionsRoot }))
|
|
54
|
+
.toBe('/tmp/project/missions/login.mission.js');
|
|
55
|
+
expect(resolveMissionPath('missions/login.mission.js', { cwd, missionsRoot }))
|
|
56
|
+
.toBe('/tmp/project/missions/login.mission.js');
|
|
57
|
+
expect(resolveMissionPath('../shared/login.mission.js', { cwd, missionsRoot }))
|
|
58
|
+
.toBe('/tmp/shared/login.mission.js');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('collects only unique screenshots referenced by a report', () => {
|
|
62
|
+
const { collectReportScreenshotNames, collectReportScreenshotPaths } = __test__;
|
|
63
|
+
const report = { missions: [
|
|
64
|
+
{ steps: [{ screenshotPath: './screenshots/one.png' }, { screenshotPath: './screenshots/two.jpg' }] },
|
|
65
|
+
{ steps: [{ screenshotPath: './screenshots/one.png' }, { events: [] }] },
|
|
66
|
+
] };
|
|
67
|
+
expect(collectReportScreenshotNames(report)).toEqual(['one.png', 'two.jpg']);
|
|
68
|
+
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/one.png', 'screenshots/two.jpg']);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('preserves run-specific screenshot paths', () => {
|
|
72
|
+
const { collectReportScreenshotPaths } = __test__;
|
|
73
|
+
const report = { missions: [{ steps: [
|
|
74
|
+
{ screenshotPath: './screenshots/run_123/one.png' },
|
|
75
|
+
] }] };
|
|
76
|
+
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/run_123/one.png']);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('resolves configured report directories and suggests close mission names', () => {
|
|
80
|
+
const { resolveReportDir, closestMissionMatch } = __test__;
|
|
81
|
+
expect(resolveReportDir({ outputDir: 'artifacts/reports' }, '/tmp/project'))
|
|
82
|
+
.toBe('/tmp/project/artifacts/reports');
|
|
83
|
+
expect(closestMissionMatch('logn.mission.js', ['checkout.mission.js', 'login.mission.js']))
|
|
84
|
+
.toBe('login.mission.js');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('resolves selected reports by run id, filename, and direct path', () => {
|
|
88
|
+
const { resolveReportFile } = __test__;
|
|
89
|
+
const tmp = makeTempDir('testronaut-selected-report-');
|
|
90
|
+
const reportDir = path.join(tmp, 'reports');
|
|
91
|
+
fs.mkdirSync(reportDir);
|
|
92
|
+
const report = path.join(reportDir, 'run_123.json');
|
|
93
|
+
fs.writeFileSync(report, '{}');
|
|
94
|
+
|
|
95
|
+
expect(resolveReportFile('run_123', reportDir, tmp)).toBe(report);
|
|
96
|
+
expect(resolveReportFile('run_123.json', reportDir, tmp)).toBe(report);
|
|
97
|
+
expect(resolveReportFile(report, reportDir, tmp)).toBe(report);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('builds a redacted effective config with source information', () => {
|
|
101
|
+
const { buildEffectiveConfig } = __test__;
|
|
102
|
+
const effective = buildEffectiveConfig({
|
|
103
|
+
provider: 'openai', model: 'gpt-5.6', outputDir: 'artifacts', sessionToken: 'secret',
|
|
104
|
+
}, '/tmp/project');
|
|
105
|
+
expect(effective.outputDir).toEqual({ value: '/tmp/project/artifacts', source: 'config' });
|
|
106
|
+
expect(effective.authenticated).toBe(true);
|
|
107
|
+
expect(JSON.stringify(effective)).not.toContain('secret');
|
|
108
|
+
});
|
|
109
|
+
|
|
37
110
|
it('parses booleans in a tolerant way', () => {
|
|
38
111
|
const { parseBool } = __test__;
|
|
39
112
|
expect(parseBool('true')).toBe(true);
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
|
|
8
|
+
const cliPath = fileURLToPath(new URL('../../bin/cli.js', import.meta.url));
|
|
9
|
+
const tempDirs = [];
|
|
10
|
+
|
|
11
|
+
function makeProject() {
|
|
12
|
+
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-cli-integration-'));
|
|
13
|
+
tempDirs.push(cwd);
|
|
14
|
+
fs.mkdirSync(path.join(cwd, 'missions'));
|
|
15
|
+
fs.writeFileSync(path.join(cwd, 'missions', 'login.mission.js'), [
|
|
16
|
+
'export const tags = ["authentication", "smoke"];',
|
|
17
|
+
'export function executeMission() { throw new Error("must not execute"); }',
|
|
18
|
+
].join('\n'));
|
|
19
|
+
fs.writeFileSync(path.join(cwd, 'testronaut-config.json'), JSON.stringify({
|
|
20
|
+
provider: 'openai', model: 'gpt-5.6', outputDir: 'artifacts/reports', sessionToken: 'private',
|
|
21
|
+
}));
|
|
22
|
+
return cwd;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function run(cwd, args) {
|
|
26
|
+
const result = spawnSync(process.execPath, [cliPath, ...args], { cwd, encoding: 'utf8' });
|
|
27
|
+
if (result.error) throw result.error;
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe('CLI inspection commands', () => {
|
|
36
|
+
it('lists mission tags without executing the mission', () => {
|
|
37
|
+
const result = run(makeProject(), ['list', '--json']);
|
|
38
|
+
expect(result.status).toBe(0);
|
|
39
|
+
expect(JSON.parse(result.stdout).missions).toEqual([
|
|
40
|
+
{ file: 'login.mission.js', tags: ['authentication', 'smoke'] },
|
|
41
|
+
]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('reports effective config without exposing the session token', () => {
|
|
45
|
+
const result = run(makeProject(), ['config', '--json']);
|
|
46
|
+
expect(result.status).toBe(0);
|
|
47
|
+
const output = JSON.parse(result.stdout);
|
|
48
|
+
expect(output.outputDir.value).toMatch(/artifacts[/\\]reports$/);
|
|
49
|
+
expect(output.authenticated).toBe(true);
|
|
50
|
+
expect(result.stdout).not.toContain('private');
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('suggests a close mission name and exits nonzero during dry-run', () => {
|
|
54
|
+
const result = run(makeProject(), ['logn.mission.js', '--dry-run', '--json']);
|
|
55
|
+
expect(result.status).toBe(1);
|
|
56
|
+
expect(JSON.parse(result.stdout).missions[0]).toMatchObject({
|
|
57
|
+
status: 'missing', suggestion: 'login.mission.js',
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('loads project .env values before evaluating mission modules', () => {
|
|
62
|
+
const cwd = makeProject();
|
|
63
|
+
fs.writeFileSync(path.join(cwd, '.env'), 'MISSION_DESTINATION=https://example.test\n');
|
|
64
|
+
fs.writeFileSync(path.join(cwd, 'missions', 'login.mission.js'), [
|
|
65
|
+
'const destination = process.env.MISSION_DESTINATION;',
|
|
66
|
+
'export const tags = ["authentication"];',
|
|
67
|
+
'export async function executeMission() {',
|
|
68
|
+
' return { missionName: destination, status: "passed", steps: [] };',
|
|
69
|
+
'}',
|
|
70
|
+
].join('\n'));
|
|
71
|
+
|
|
72
|
+
const result = run(cwd, ['--tags', 'authentication', '--json', '--quiet']);
|
|
73
|
+
|
|
74
|
+
expect(result.status).toBe(0);
|
|
75
|
+
expect(JSON.parse(result.stdout).missions[0].missionName).toBe('https://example.test');
|
|
76
|
+
});
|
|
77
|
+
});
|
|
@@ -77,6 +77,9 @@ describe('initializeTestronautProject', () => {
|
|
|
77
77
|
expect(cfg.initialized).toBe(true);
|
|
78
78
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
79
79
|
expect(typeof cfg.maxTurns).toBe('number');
|
|
80
|
+
expect(cfg.tags).toEqual([]);
|
|
81
|
+
expect(cfg.tagMatch).toBe('any');
|
|
82
|
+
expect(cfg.addTags).toEqual([]);
|
|
80
83
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
81
84
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
82
85
|
expect(cfg.humanInput?.enabled).toBe(true);
|
|
@@ -95,6 +98,9 @@ describe('initializeTestronautProject', () => {
|
|
|
95
98
|
// Welcome mission (name depends on your implementation; we check presence in missions/)
|
|
96
99
|
const files = fs.readdirSync(path.join(temp, 'missions'));
|
|
97
100
|
expect(files.some(f => /welcome/i.test(f))).toBe(true);
|
|
101
|
+
const welcome = fs.readFileSync(path.join(temp, 'missions', 'welcome.mission.js'), 'utf8');
|
|
102
|
+
expect(welcome).toContain('export const tags = [];');
|
|
103
|
+
expect(welcome).toContain('tags\n },');
|
|
98
104
|
|
|
99
105
|
// Exactly two prompts were asked
|
|
100
106
|
expect(shared.calls).toBe(2);
|
|
@@ -112,6 +118,9 @@ describe('initializeTestronautProject', () => {
|
|
|
112
118
|
const cfg = readJson(path.join(temp, 'testronaut-config.json'));
|
|
113
119
|
expect(cfg.provider).toBe('gemini');
|
|
114
120
|
expect(cfg.model).toBe('gemini-2.5-flash');
|
|
121
|
+
expect(cfg.tags).toEqual([]);
|
|
122
|
+
expect(cfg.tagMatch).toBe('any');
|
|
123
|
+
expect(cfg.addTags).toEqual([]);
|
|
115
124
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
116
125
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
117
126
|
expect(cfg.humanInput?.enabled).toBe(true);
|
|
@@ -8,6 +8,9 @@ describe('defaultConfig', () => {
|
|
|
8
8
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
9
9
|
expect(cfg.projectName).toBe('my-app');
|
|
10
10
|
expect(cfg.maxTurns).toBe(20);
|
|
11
|
+
expect(cfg.tags).toEqual([]);
|
|
12
|
+
expect(cfg.tagMatch).toBe('any');
|
|
13
|
+
expect(cfg.addTags).toEqual([]);
|
|
11
14
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
12
15
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
13
16
|
expect(cfg.resourceGuard?.hrefIncludes).toContain('/document/');
|
|
@@ -31,11 +31,11 @@ describe('discoverMissionFiles', () => {
|
|
|
31
31
|
fs.writeFileSync(path.join(missionsDir, 'beta.txt'), '// ignore');
|
|
32
32
|
fs.mkdirSync(path.join(missionsDir, 'nested'), { recursive: true });
|
|
33
33
|
fs.writeFileSync(path.join(missionsDir, 'nested', 'gamma.mission.js'), '// nested');
|
|
34
|
-
fs.writeFileSync(path.join(missionsDir, 'delta.mission.ts'), '// ts
|
|
34
|
+
fs.writeFileSync(path.join(missionsDir, 'delta.mission.ts'), '// ts');
|
|
35
35
|
|
|
36
36
|
const res = await discoverMissionFiles({ cwd: tmp });
|
|
37
37
|
expect(res.root).toBe(path.resolve(tmp, 'missions'));
|
|
38
|
-
expect(res.files).toEqual(['alpha.mission.js']);
|
|
38
|
+
expect(res.files).toEqual(['alpha.mission.js', 'delta.mission.ts']);
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it('respects missions.root with include patterns', async () => {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { loadMissionModule } from '../../core/missionLoader.js';
|
|
6
|
+
|
|
7
|
+
const tempDirs = [];
|
|
8
|
+
|
|
9
|
+
function makeProject(packageJson) {
|
|
10
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-loader-'));
|
|
11
|
+
tempDirs.push(dir);
|
|
12
|
+
if (packageJson) {
|
|
13
|
+
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(packageJson));
|
|
14
|
+
}
|
|
15
|
+
return dir;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
describe('loadMissionModule', () => {
|
|
23
|
+
it.each([
|
|
24
|
+
['without a package type', undefined],
|
|
25
|
+
['inside a CommonJS project', { type: 'commonjs' }],
|
|
26
|
+
])('loads ESM-style .mission.js %s', async (_label, packageJson) => {
|
|
27
|
+
const dir = makeProject(packageJson);
|
|
28
|
+
const missionPath = path.join(dir, 'sample.mission.js');
|
|
29
|
+
fs.writeFileSync(missionPath, 'export const tags = ["smoke"]; export async function executeMission() { return "ok"; }');
|
|
30
|
+
|
|
31
|
+
const mission = await loadMissionModule(missionPath);
|
|
32
|
+
|
|
33
|
+
expect(mission.tags).toEqual(['smoke']);
|
|
34
|
+
await expect(mission.executeMission()).resolves.toBe('ok');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('loads a typed .mission.ts file with relative imports', async () => {
|
|
38
|
+
const dir = makeProject({ type: 'commonjs' });
|
|
39
|
+
fs.writeFileSync(path.join(dir, 'goal.ts'), 'export const goal: string = "TypeScript works";');
|
|
40
|
+
const missionPath = path.join(dir, 'sample.mission.ts');
|
|
41
|
+
fs.writeFileSync(missionPath, [
|
|
42
|
+
'import { goal } from "./goal.ts";',
|
|
43
|
+
'export const tags: string[] = ["typescript"];',
|
|
44
|
+
'export async function executeMission(): Promise<string> { return goal; }',
|
|
45
|
+
].join('\n'));
|
|
46
|
+
|
|
47
|
+
const mission = await loadMissionModule(missionPath);
|
|
48
|
+
|
|
49
|
+
expect(mission.tags).toEqual(['typescript']);
|
|
50
|
+
await expect(mission.executeMission()).resolves.toBe('TypeScript works');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -78,6 +78,7 @@ vi.mock('../../tools/toolSchema.js', () => ({
|
|
|
78
78
|
{ type: 'function', function: { name: 'screenshot', description: 'take a screenshot', parameters: {} } },
|
|
79
79
|
{ type: 'function', function: { name: 'list_local_files', description: 'list files', parameters: {} } },
|
|
80
80
|
{ type: 'function', function: { name: 'get_mfa_code', description: 'get mfa', parameters: {} } },
|
|
81
|
+
{ type: 'function', function: { name: 'set_ground_control_state', description: 'set state', parameters: {} } },
|
|
81
82
|
],
|
|
82
83
|
}));
|
|
83
84
|
|
|
@@ -126,6 +127,7 @@ describe('turnLoop', () => {
|
|
|
126
127
|
let browser;
|
|
127
128
|
|
|
128
129
|
beforeEach(() => {
|
|
130
|
+
delete process.env.TESTRONAUT_SCREENSHOTS;
|
|
129
131
|
browser = {};
|
|
130
132
|
// reset spies and chat mock
|
|
131
133
|
shared.chromeToolSpies.fill.mockClear();
|
|
@@ -136,6 +138,19 @@ describe('turnLoop', () => {
|
|
|
136
138
|
if (shared.chatMock) shared.chatMock.mockReset();
|
|
137
139
|
});
|
|
138
140
|
|
|
141
|
+
it('does not expose the screenshot tool when screenshots are disabled', async () => {
|
|
142
|
+
process.env.TESTRONAUT_SCREENSHOTS = '0';
|
|
143
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
144
|
+
message: { role: 'assistant', content: 'FINAL: done' },
|
|
145
|
+
usage: { total_tokens: 2 },
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
await turnLoop(browser, baseMessages(), 1, 0, 0, {}, { steps: [], missionName: 'demo' });
|
|
149
|
+
|
|
150
|
+
const tools = shared.chatMock.mock.calls[0][0].tools;
|
|
151
|
+
expect(tools.map(tool => tool.function.name)).not.toContain('screenshot');
|
|
152
|
+
});
|
|
153
|
+
|
|
139
154
|
it('handles a final response in a single turn (no tools)', async () => {
|
|
140
155
|
shared.chatMock.mockResolvedValueOnce({
|
|
141
156
|
message: { role: 'assistant', content: 'FINAL: success' },
|
|
@@ -193,6 +208,31 @@ describe('turnLoop', () => {
|
|
|
193
208
|
expect(step2.result).toMatch(/Success/);
|
|
194
209
|
});
|
|
195
210
|
|
|
211
|
+
it('handles Ground Control state locally without dispatching it as a browser tool', async () => {
|
|
212
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
213
|
+
message: {
|
|
214
|
+
role: 'assistant', content: '', tool_calls: [{
|
|
215
|
+
id: 'tool_gc', type: 'function', function: {
|
|
216
|
+
name: 'set_ground_control_state',
|
|
217
|
+
arguments: JSON.stringify({ app: { baseUrl: 'https://example.com' }, session: { loggedIn: false } }),
|
|
218
|
+
},
|
|
219
|
+
}],
|
|
220
|
+
},
|
|
221
|
+
usage: { total_tokens: 4 },
|
|
222
|
+
});
|
|
223
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
224
|
+
message: { role: 'assistant', content: 'FINAL: done' }, usage: { total_tokens: 2 },
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
const ctx = { steps: [], missionName: 'demo' };
|
|
228
|
+
const res = await turnLoop(browser, baseMessages(), 2, 0, 0, {}, ctx);
|
|
229
|
+
|
|
230
|
+
expect(res.success).toBe(true);
|
|
231
|
+
expect(ctx.groundControl.app.baseUrl).toBe('https://example.com');
|
|
232
|
+
expect(ctx.groundControl.session.loggedIn).toBe(false);
|
|
233
|
+
expect(res.steps[0].events).toContain('[tool ] ← set_ground_control_state result: ✅ Success');
|
|
234
|
+
});
|
|
235
|
+
|
|
196
236
|
it('retries a turn on tool error before marking issues', async () => {
|
|
197
237
|
// First turn: tool call → click_text will throw
|
|
198
238
|
shared.chatMock.mockResolvedValueOnce({
|
|
@@ -21,11 +21,13 @@ describe('generateHtmlReport', () => {
|
|
|
21
21
|
it('writes an HTML report containing mission and retry metadata', () => {
|
|
22
22
|
const report = {
|
|
23
23
|
runId: 'run_test',
|
|
24
|
+
cli: { version: '1.7.0' },
|
|
24
25
|
missions: [
|
|
25
26
|
{
|
|
26
27
|
missionName: 'Mission A',
|
|
27
28
|
submissionType: 'mission',
|
|
28
29
|
submissionName: 'Main',
|
|
30
|
+
file: 'missions/login.mission.ts',
|
|
29
31
|
status: 'passed',
|
|
30
32
|
tags: ['smoke'],
|
|
31
33
|
steps: [
|
|
@@ -57,6 +59,8 @@ describe('generateHtmlReport', () => {
|
|
|
57
59
|
expect(html).toContain('⚠️ Turn Issues');
|
|
58
60
|
expect(html).toContain('data-tag="smoke"');
|
|
59
61
|
expect(html).toContain('data-tags="smoke"');
|
|
62
|
+
expect(html).toContain('CLI version:</strong> 1.7.0');
|
|
63
|
+
expect(html).toContain('missions/login.mission.ts');
|
|
60
64
|
expect(html).toContain('data-tag="untagged"');
|
|
61
65
|
expect(html).toContain('id="match-count"');
|
|
62
66
|
expect(html).toContain('No missions match these tags');
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -26,7 +26,7 @@ import { requestHumanInput } from './humanInput.js';
|
|
|
26
26
|
import { getMfaCode } from './mfaCode.js';
|
|
27
27
|
|
|
28
28
|
const FILES_DIR = path.join('missions', 'files');
|
|
29
|
-
const REPORTS_DIR = path.join('missions', 'mission_reports');
|
|
29
|
+
const REPORTS_DIR = process.env.TESTRONAUT_OUTPUT_DIR || path.join('missions', 'mission_reports');
|
|
30
30
|
const FILES_LOG = path.join(REPORTS_DIR, 'files.jsonl');
|
|
31
31
|
|
|
32
32
|
function ensureDir(p) {
|
|
@@ -752,7 +752,11 @@ export class ChromeBrowser {
|
|
|
752
752
|
}
|
|
753
753
|
if (!chosen) {
|
|
754
754
|
await this._debugClickableSnapshot({ scope }).catch(()=>{});
|
|
755
|
-
|
|
755
|
+
if (process.env.TESTRONAUT_SCREENSHOTS !== '0') {
|
|
756
|
+
const failurePath = path.join(REPORTS_DIR, 'screenshots', process.env.TESTRONAUT_RUN_ID || '', 'click_fail.png');
|
|
757
|
+
ensureDir(path.dirname(failurePath));
|
|
758
|
+
await this.page.screenshot({ path: failurePath }).catch(()=>{});
|
|
759
|
+
}
|
|
756
760
|
throw new Error('No clickable locator became ready');
|
|
757
761
|
}
|
|
758
762
|
// do the click (standard → hover+click → JS click)
|
|
@@ -1183,13 +1187,19 @@ export class ChromeBrowser {
|
|
|
1183
1187
|
}
|
|
1184
1188
|
|
|
1185
1189
|
async screenshot({ name = 'screenshot' }) {
|
|
1190
|
+
if (process.env.TESTRONAUT_SCREENSHOTS === '0') {
|
|
1191
|
+
throw new Error('Screenshots are disabled for this run (--no-screenshots).');
|
|
1192
|
+
}
|
|
1186
1193
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
1187
|
-
const
|
|
1194
|
+
const screenshotDir = path.join(REPORTS_DIR, 'screenshots', process.env.TESTRONAUT_RUN_ID || '');
|
|
1195
|
+
const screenshotPath = path.join(screenshotDir, `${name}_${timestamp}.png`);
|
|
1196
|
+
ensureDir(screenshotDir);
|
|
1188
1197
|
|
|
1189
1198
|
await this.page.screenshot({ path: screenshotPath });
|
|
1190
1199
|
|
|
1191
1200
|
// ✅ Return relative path for HTML report
|
|
1192
|
-
|
|
1201
|
+
const reportRelativePath = path.relative(REPORTS_DIR, screenshotPath).split(path.sep).join('/');
|
|
1202
|
+
return `Screenshot saved at: ./${reportRelativePath}`;
|
|
1193
1203
|
}
|
|
1194
1204
|
|
|
1195
1205
|
async upload_file({ selector, fileName, useChooser = false, timeoutMs = 15000 }) {
|
|
@@ -27,7 +27,7 @@ import { normalizeTags } from '../core/tags.js';
|
|
|
27
27
|
* @returns {string} absolute path to the written HTML file
|
|
28
28
|
*/
|
|
29
29
|
export function generateHtmlReport(report, outputPath) {
|
|
30
|
-
const { runId, startTime, endTime, missions = [], summary = {}, llm = {} } = report;
|
|
30
|
+
const { runId, startTime, endTime, missions = [], summary = {}, llm = {}, cli = {} } = report;
|
|
31
31
|
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
32
32
|
const durationSec =
|
|
33
33
|
(startTime && endTime)
|
|
@@ -137,10 +137,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
137
137
|
const missionGroupBlock = (missionName, subs) => {
|
|
138
138
|
const status = groupStatus(subs);
|
|
139
139
|
const totalSteps = subs.reduce((n, s) => n + (Array.isArray(s.steps) ? s.steps.length : 0), 0);
|
|
140
|
-
const
|
|
141
|
-
const
|
|
142
|
-
const
|
|
140
|
+
const startTimes = subs.map(s => s.startTime || 0).filter(Boolean);
|
|
141
|
+
const endTimes = subs.map(s => s.endTime || 0).filter(Boolean);
|
|
142
|
+
const firstStart = startTimes.length ? Math.min(...startTimes) : null;
|
|
143
|
+
const lastEnd = endTimes.length ? Math.max(...endTimes) : null;
|
|
144
|
+
const groupDur = (firstStart != null && lastEnd != null) ? ((lastEnd - firstStart) / 1000).toFixed(2) : '—';
|
|
143
145
|
const tags = normalizeTags(subs.flatMap(s => s.submissionType === 'mission' ? (s.tags ?? []) : []));
|
|
146
|
+
const sourceFiles = [...new Set(subs.map(s => s.file).filter(Boolean))];
|
|
144
147
|
|
|
145
148
|
// pre → mission → post
|
|
146
149
|
const order = { premission: 0, mission: 1, postmission: 2 };
|
|
@@ -155,7 +158,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
155
158
|
<span class="name">${esc(missionName)}</span>
|
|
156
159
|
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
157
160
|
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
158
|
-
<span class="meta"
|
|
161
|
+
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} • ` : ''}submissions: ${subs.length} • steps: ${totalSteps} • duration: ${groupDur}s</span>
|
|
159
162
|
<span class="toolbar">
|
|
160
163
|
<button class="btn-mini toggle" data-scope="mission" aria-label="Expand">▼</button>
|
|
161
164
|
</span>
|
|
@@ -350,6 +353,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
350
353
|
<div><strong>Start:</strong> ${esc(startTime ?? '—')}</div>
|
|
351
354
|
<div><strong>End:</strong> ${esc(endTime ?? '—')} • <strong>Duration:</strong> ${durationSec}s</div>
|
|
352
355
|
<div><strong>LLM:</strong> ${esc(llm.provider ?? '—')} • <strong>Model:</strong> ${esc(llm.model ?? '—')}</div>
|
|
356
|
+
<div><strong>CLI version:</strong> ${esc(cli.version ?? '—')}</div>
|
|
353
357
|
</div>
|
|
354
358
|
</div>
|
|
355
359
|
|
package/tools/mfaCode.js
CHANGED
|
@@ -9,7 +9,10 @@ export const DEV_API_BASE = 'https://staging.api.testronaut.app';
|
|
|
9
9
|
|
|
10
10
|
const MIN_REMAINING_SECONDS = 5;
|
|
11
11
|
const MAX_REMAINING_SECONDS = 25;
|
|
12
|
-
const API_DEBUG_LOG_RELATIVE_PATH =
|
|
12
|
+
const API_DEBUG_LOG_RELATIVE_PATH = path.join(
|
|
13
|
+
process.env.TESTRONAUT_OUTPUT_DIR || 'missions/mission_reports',
|
|
14
|
+
'api-debug.log'
|
|
15
|
+
);
|
|
13
16
|
const REDACTED = '<redacted>';
|
|
14
17
|
|
|
15
18
|
const SENSITIVE_LOG_KEYS = new Set([
|