testronaut 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -4
- package/TODO.md +9 -0
- package/bin/cli.js +267 -37
- package/bin/init.js +35 -5
- package/bin/initHelpers.js +16 -5
- package/core/missionDiscovery.js +3 -3
- package/core/missionLoader.js +16 -0
- package/core/turnLoop.js +18 -15
- package/llm/anthropic/anthropicProvider.js +119 -0
- package/llm/anthropic/models.js +7 -0
- package/llm/llmFactory.js +6 -1
- package/package.json +3 -1
- package/runner/testronaut.js +4 -1
- package/tests/binTests/cli.helpers.test.js +75 -2
- package/tests/binTests/cli.integration.test.js +77 -0
- package/tests/binTests/init.test.js +24 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +3 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +4 -0
- package/tests/binTests/initHelpersTests/models.test.js +3 -1
- package/tests/coreTests/missionDiscovery.test.js +2 -2
- package/tests/coreTests/missionLoader.test.js +52 -0
- package/tests/coreTests/turnLoop.test.js +40 -0
- package/tests/llmTests/anthropicProvider.test.js +41 -0
- package/tests/llmTests/llmFactory.test.js +17 -1
- package/tests/toolsTests/generateHtmlReport.test.js +18 -0
- package/tools/chromeBrowser.js +14 -4
- package/tools/generateHtmlReport.js +22 -7
- package/tools/mfaCode.js +4 -1
- package/tools/tokenControl.js +7 -1
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ Join the Discord to connect with other Testronauts, get support, and help shape
|
|
|
18
18
|
|
|
19
19
|
- Write tests in plain English — no brittle selectors
|
|
20
20
|
- Runs real browser sessions via **Playwright**
|
|
21
|
-
- Works with **multiple LLM providers** (OpenAI, Google Gemini, and
|
|
21
|
+
- Works with **multiple LLM providers** (OpenAI, Google Gemini, and Anthropic Claude)
|
|
22
22
|
- Modular tool and DOM-reasoning system
|
|
23
23
|
- Dynamic rate-limit and token-tracking logic
|
|
24
24
|
- Generates JSON + HTML reports automatically
|
|
@@ -58,12 +58,18 @@ testronaut --init
|
|
|
58
58
|
Run the sample mission:
|
|
59
59
|
```bash
|
|
60
60
|
testronaut welcome.mission.js
|
|
61
|
+
|
|
62
|
+
# Paths work too, which enables shell tab completion
|
|
63
|
+
testronaut missions/welcome.mission.js
|
|
61
64
|
```
|
|
62
65
|
|
|
63
66
|
**One-off / no install** (use `npx` to run without installing):
|
|
64
67
|
```bash
|
|
65
68
|
npx testronaut --init
|
|
66
69
|
npx testronaut welcome.mission.js
|
|
70
|
+
|
|
71
|
+
# Paths work too, which enables shell tab completion
|
|
72
|
+
npx testronaut missions/welcome.mission.js
|
|
67
73
|
```
|
|
68
74
|
|
|
69
75
|
---
|
|
@@ -74,9 +80,13 @@ npx testronaut welcome.mission.js
|
|
|
74
80
|
missions/
|
|
75
81
|
├── login.mission.js
|
|
76
82
|
├── logout.mission.js
|
|
77
|
-
└── dashboard.mission.
|
|
83
|
+
└── dashboard.mission.ts
|
|
78
84
|
```
|
|
79
85
|
|
|
86
|
+
Mission files can use standard `import`/`export` syntax in `.js` or `.ts` files.
|
|
87
|
+
Testronaut loads them independently of your project's module format, so your
|
|
88
|
+
`package.json` does not need `"type": "module"`.
|
|
89
|
+
|
|
80
90
|
Each mission exports a string or function and calls `runMissions`.
|
|
81
91
|
|
|
82
92
|
---
|
|
@@ -110,6 +120,9 @@ OPENAI_API_KEY=sk-...
|
|
|
110
120
|
# Or for Gemini
|
|
111
121
|
GEMINI_API_KEY=AIza...
|
|
112
122
|
|
|
123
|
+
# Or for Anthropic Claude
|
|
124
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
125
|
+
|
|
113
126
|
URL=https://example.com/login
|
|
114
127
|
USERNAME=example@example.com
|
|
115
128
|
PASSWORD=********
|
|
@@ -136,8 +149,9 @@ Current supported providers:
|
|
|
136
149
|
|-----------|----------------|
|
|
137
150
|
| **OpenAI** | gpt-5.6, gpt-5.6-terra, gpt-5.6-luna, gpt-5.5, and legacy GPT/o-series models |
|
|
138
151
|
| **Google Gemini** | gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-8b |
|
|
152
|
+
| **Anthropic Claude** | claude-sonnet-5, claude-opus-5, claude-haiku-4-5 |
|
|
139
153
|
|
|
140
|
-
|
|
154
|
+
The canonical Claude provider ID is `anthropic`; `claude` is also accepted as a CLI alias.
|
|
141
155
|
|
|
142
156
|
---
|
|
143
157
|
|
|
@@ -153,6 +167,20 @@ Run a specific mission:
|
|
|
153
167
|
testronaut login.mission.js
|
|
154
168
|
```
|
|
155
169
|
|
|
170
|
+
Preview or inspect without launching a browser:
|
|
171
|
+
```bash
|
|
172
|
+
testronaut list
|
|
173
|
+
testronaut --dry-run
|
|
174
|
+
testronaut config
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Upload the latest or a selected report:
|
|
178
|
+
```bash
|
|
179
|
+
testronaut upload
|
|
180
|
+
testronaut upload run_1788745106444
|
|
181
|
+
testronaut upload run_1788745106444.json --no-upload-screenshots
|
|
182
|
+
```
|
|
183
|
+
|
|
156
184
|
Chain missions together:
|
|
157
185
|
```js
|
|
158
186
|
await runMissions({
|
|
@@ -243,7 +271,7 @@ Notes:
|
|
|
243
271
|
|
|
244
272
|
## 📋 Reports
|
|
245
273
|
|
|
246
|
-
Testronaut generates
|
|
274
|
+
Testronaut generates JSON, HTML, and run-specific screenshots under the configured `outputDir` (default):
|
|
247
275
|
|
|
248
276
|
```
|
|
249
277
|
missions/mission_reports/
|
package/TODO.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# TODO
|
|
2
2
|
|
|
3
|
+
## Quality-of-Life Follow-Ups
|
|
4
|
+
|
|
5
|
+
- [ ] Add a versioned report schema shared with Mission Control so future report changes can be validated and migrated explicitly.
|
|
6
|
+
- [ ] Add resumable, retryable, and idempotent report uploads so interrupted screenshot transfers do not require starting over or create duplicate reports.
|
|
7
|
+
- [ ] Add `testronaut doctor` to check Node.js, browser installation, configuration, authentication, output-directory access, and API connectivity without running a mission.
|
|
8
|
+
- [ ] Add optional shell completion for commands, flags, mission files, tags, and local report IDs.
|
|
9
|
+
- [ ] Treat `--json` output as a stable automation contract with documented schemas and tests for every supported command.
|
|
10
|
+
- [ ] If authentication ever moves outside project config, continue reading existing `sessionToken` values and provide an opt-in migration path before deprecating legacy storage.
|
|
11
|
+
|
|
3
12
|
## Automated MFA Follow-Ups
|
|
4
13
|
|
|
5
14
|
- [ ] Add an integration smoke test against a local or staging API fixture for `get_mfa_code`.
|
package/bin/cli.js
CHANGED
|
@@ -34,6 +34,9 @@
|
|
|
34
34
|
* See tests in tests/toolsTests/cli.helpers.test.js
|
|
35
35
|
*/
|
|
36
36
|
|
|
37
|
+
// Missions often interpolate project .env values while their modules load.
|
|
38
|
+
// Load those values before discovery imports any user-authored mission code.
|
|
39
|
+
import 'dotenv/config';
|
|
37
40
|
import fs from 'fs';
|
|
38
41
|
import path from 'path';
|
|
39
42
|
import { fileURLToPath } from 'url';
|
|
@@ -51,6 +54,7 @@ const exec = promisify(execCmd);
|
|
|
51
54
|
import url from 'url';
|
|
52
55
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
53
56
|
import { discoverMissionFiles } from '../core/missionDiscovery.js';
|
|
57
|
+
import { loadMissionModule } from '../core/missionLoader.js';
|
|
54
58
|
import { loadConfig } from '../core/config.js';
|
|
55
59
|
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../core/tags.js';
|
|
56
60
|
|
|
@@ -59,12 +63,32 @@ process.env.PLAYWRIGHT_BROWSERS_PATH = process.env.PLAYWRIGHT_BROWSERS_PATH || '
|
|
|
59
63
|
|
|
60
64
|
const TMP_DIR = path.resolve('./missions/tmp');
|
|
61
65
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
66
|
+
const CLI_VERSION = JSON.parse(
|
|
67
|
+
fs.readFileSync(path.resolve(__dirname, '../package.json'), 'utf8')
|
|
68
|
+
).version;
|
|
62
69
|
|
|
63
70
|
const DEFAULT_API_BASE = 'http://api.testronaut.app';
|
|
64
71
|
const DEV_API_BASE = 'https://staging.api.testronaut.app';
|
|
65
72
|
|
|
66
73
|
let args = process.argv.slice(2);
|
|
67
74
|
|
|
75
|
+
const jsonOutput = args.includes('--json');
|
|
76
|
+
const quietOutput = args.includes('--quiet') || jsonOutput;
|
|
77
|
+
args = args.filter(arg => arg !== '--json' && arg !== '--quiet');
|
|
78
|
+
const writeOutput = value => process.stdout.write(`${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}\n`);
|
|
79
|
+
if (quietOutput) {
|
|
80
|
+
console.log = () => {};
|
|
81
|
+
console.warn = () => {};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const dryRun = args.includes('--dry-run');
|
|
85
|
+
args = args.filter(arg => arg !== '--dry-run');
|
|
86
|
+
const screenshotsEnabled = !args.includes('--no-screenshots');
|
|
87
|
+
args = args.filter(arg => arg !== '--no-screenshots');
|
|
88
|
+
const uploadScreenshots = !args.includes('--no-upload-screenshots');
|
|
89
|
+
args = args.filter(arg => arg !== '--no-upload-screenshots');
|
|
90
|
+
process.env.TESTRONAUT_SCREENSHOTS = screenshotsEnabled ? '1' : '0';
|
|
91
|
+
|
|
68
92
|
// Detect how the CLI was invoked so help text matches the actual command
|
|
69
93
|
function detectCliName(npmCommand = process.env.npm_command, argv1 = process.argv[1]) {
|
|
70
94
|
// npx sets npm_command to 'exec'
|
|
@@ -148,7 +172,7 @@ function parseProviderArgs(argsList) {
|
|
|
148
172
|
provider = provider.trim();
|
|
149
173
|
}
|
|
150
174
|
|
|
151
|
-
const supportedProviders = new Set(['openai', 'gemini']);
|
|
175
|
+
const supportedProviders = new Set(['openai', 'gemini', 'anthropic', 'claude']);
|
|
152
176
|
const isValid = (v) => !!v && supportedProviders.has(String(v).toLowerCase());
|
|
153
177
|
if (!isValid(provider)) {
|
|
154
178
|
invalid = true;
|
|
@@ -311,8 +335,97 @@ export const __test__ = {
|
|
|
311
335
|
parseRunOptionsArgs,
|
|
312
336
|
detectCliName,
|
|
313
337
|
isDirectInvocation,
|
|
338
|
+
resolveMissionPath,
|
|
339
|
+
collectReportScreenshotNames,
|
|
340
|
+
collectReportScreenshotPaths,
|
|
341
|
+
resolveReportDir,
|
|
342
|
+
closestMissionMatch,
|
|
343
|
+
resolveReportFile,
|
|
344
|
+
buildEffectiveConfig,
|
|
314
345
|
};
|
|
315
346
|
|
|
347
|
+
function resolveReportDir(config = {}, cwd = process.cwd()) {
|
|
348
|
+
return path.resolve(cwd, config.outputDir || 'missions/mission_reports');
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function resolveReportFile(requested, reportDir, cwd = process.cwd()) {
|
|
352
|
+
if (!requested) return null;
|
|
353
|
+
const names = path.extname(requested) ? [requested] : [requested, `${requested}.json`];
|
|
354
|
+
for (const name of names) {
|
|
355
|
+
const direct = path.resolve(cwd, name);
|
|
356
|
+
if (fs.existsSync(direct)) return direct;
|
|
357
|
+
const inReportDir = path.resolve(reportDir, name);
|
|
358
|
+
if (fs.existsSync(inReportDir)) return inReportDir;
|
|
359
|
+
}
|
|
360
|
+
return path.resolve(reportDir, names.at(-1));
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function buildEffectiveConfig(config = {}, cwd = process.cwd()) {
|
|
364
|
+
const resolved = resolveProviderModel({ cwd });
|
|
365
|
+
const source = key => process.env[key] ? 'environment' : undefined;
|
|
366
|
+
return {
|
|
367
|
+
configFile: path.resolve(cwd, 'testronaut-config.json'),
|
|
368
|
+
provider: { value: resolved.provider, source: source('TESTRONAUT_PROVIDER') || (config.provider ? 'config' : 'default') },
|
|
369
|
+
model: { value: resolved.model, source: source('TESTRONAUT_MODEL') || (config.model ? 'config' : 'default') },
|
|
370
|
+
outputDir: { value: resolveReportDir(config, cwd), source: config.outputDir ? 'config' : 'default' },
|
|
371
|
+
maxTurns: { value: Number(process.env.TESTRONAUT_TURNS || config.maxTurns || 20), source: source('TESTRONAUT_TURNS') || (config.maxTurns != null ? 'config' : 'default') },
|
|
372
|
+
tags: config.tags || [],
|
|
373
|
+
tagMatch: config.tagMatch || 'any',
|
|
374
|
+
addTags: normalizeTags([...(config.addTags || []), ...cliAddTags]),
|
|
375
|
+
screenshots: screenshotsEnabled,
|
|
376
|
+
authenticated: Boolean(config.sessionToken),
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function closestMissionMatch(requested, candidates = []) {
|
|
381
|
+
const target = path.basename(requested).toLowerCase();
|
|
382
|
+
const distance = (a, b) => {
|
|
383
|
+
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
384
|
+
for (let i = 1; i <= a.length; i++) {
|
|
385
|
+
let previous = row[0];
|
|
386
|
+
row[0] = i;
|
|
387
|
+
for (let j = 1; j <= b.length; j++) {
|
|
388
|
+
const saved = row[j];
|
|
389
|
+
row[j] = Math.min(row[j] + 1, row[j - 1] + 1, previous + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
390
|
+
previous = saved;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return row[b.length];
|
|
394
|
+
};
|
|
395
|
+
return candidates
|
|
396
|
+
.map(file => ({ file, distance: distance(target, path.basename(file).toLowerCase()) }))
|
|
397
|
+
.sort((a, b) => a.distance - b.distance || a.file.localeCompare(b.file))[0]?.file || null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function resolveMissionPath(filePath, { cwd = process.cwd(), missionsRoot } = {}) {
|
|
401
|
+
if (path.isAbsolute(filePath)) return path.normalize(filePath);
|
|
402
|
+
|
|
403
|
+
const directPath = path.resolve(cwd, filePath);
|
|
404
|
+
const hasPathSegments = path.dirname(filePath) !== '.';
|
|
405
|
+
if (hasPathSegments || fs.existsSync(directPath)) return directPath;
|
|
406
|
+
|
|
407
|
+
return path.resolve(missionsRoot || path.join(cwd, 'missions'), filePath);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function collectReportScreenshotNames(report) {
|
|
411
|
+
return collectReportScreenshotPaths(report).map(file => path.basename(file));
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function collectReportScreenshotPaths(report) {
|
|
415
|
+
const names = [];
|
|
416
|
+
const seen = new Set();
|
|
417
|
+
for (const mission of Array.isArray(report?.missions) ? report.missions : []) {
|
|
418
|
+
for (const step of Array.isArray(mission?.steps) ? mission.steps : []) {
|
|
419
|
+
if (typeof step?.screenshotPath !== 'string') continue;
|
|
420
|
+
const name = step.screenshotPath.trim().replace(/^\.\//, '').replaceAll('\\', '/');
|
|
421
|
+
if (!name || seen.has(name)) continue;
|
|
422
|
+
seen.add(name);
|
|
423
|
+
names.push(name);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return names;
|
|
427
|
+
}
|
|
428
|
+
|
|
316
429
|
function parseTagArgs(argsList) {
|
|
317
430
|
const nextArgs = [];
|
|
318
431
|
const tagValues = [];
|
|
@@ -603,9 +716,11 @@ const HELP_TEXT = `
|
|
|
603
716
|
|
|
604
717
|
Usage:
|
|
605
718
|
${cliName} Run all missions in the ./missions directory
|
|
606
|
-
${cliName} <file> Run a
|
|
719
|
+
${cliName} <file> Run a .mission.js or .mission.ts file by name or path
|
|
607
720
|
${cliName} login Log in and store session token
|
|
608
|
-
${cliName}
|
|
721
|
+
${cliName} list List discovered missions and tags without running them
|
|
722
|
+
${cliName} config Show effective configuration and value sources
|
|
723
|
+
${cliName} upload [file] Upload the latest report or a selected report file/run ID
|
|
609
724
|
${cliName} serve Serve & open the most recent HTML report (read-only)
|
|
610
725
|
${cliName} view Alias of 'serve'
|
|
611
726
|
|
|
@@ -613,7 +728,7 @@ Options:
|
|
|
613
728
|
--init Scaffold project folders and a welcome mission
|
|
614
729
|
--turns=<n> Override max turns for this run (e.g., --turns=30)
|
|
615
730
|
--debug[=<bool>] Enable verbose debug logs (or set TESTRONAUT_DEBUG=1)
|
|
616
|
-
--provider=<id> Override LLM provider
|
|
731
|
+
--provider=<id> Override LLM provider (openai, gemini, or anthropic)
|
|
617
732
|
-o, --options key=value Set run options, such as mfa=github-test-mfa
|
|
618
733
|
--dev Use the staging API base URL
|
|
619
734
|
--vercel-bypass=<secret> Send Vercel protection bypass header for protected deployments
|
|
@@ -621,6 +736,11 @@ Options:
|
|
|
621
736
|
--no-human-input Disable human-in-the-loop verification prompts for automated runs
|
|
622
737
|
--human-input-timeout=<s> Override human input wait timeout in seconds (default: 60)
|
|
623
738
|
--help Show this help message
|
|
739
|
+
--dry-run Show the resolved mission plan without launching a browser
|
|
740
|
+
--no-screenshots Remove the screenshot tool for this run
|
|
741
|
+
--no-upload-screenshots Upload report JSON without its screenshots
|
|
742
|
+
--json Emit machine-readable command/run output
|
|
743
|
+
--quiet Suppress informational logs
|
|
624
744
|
--retry_limit=<n> Override agent turn retry limits (minimum 1, maximum 10)
|
|
625
745
|
--tag=<tag> Run missions matching a tag; repeat for multiple tags
|
|
626
746
|
--tags=<tag,...> Compact comma-list form (OR/any by default)
|
|
@@ -631,6 +751,7 @@ Options:
|
|
|
631
751
|
Examples:
|
|
632
752
|
${cliName}
|
|
633
753
|
${cliName} login
|
|
754
|
+
${cliName} list
|
|
634
755
|
${cliName} upload
|
|
635
756
|
${cliName} serve
|
|
636
757
|
${cliName} --init
|
|
@@ -689,8 +810,8 @@ if (args.includes('login')) {
|
|
|
689
810
|
}
|
|
690
811
|
|
|
691
812
|
// Handle the upload command
|
|
692
|
-
if (args
|
|
693
|
-
await uploadReport();
|
|
813
|
+
if (args[0] === 'upload') {
|
|
814
|
+
await uploadReport(args[1], { uploadScreenshots });
|
|
694
815
|
process.exit(0);
|
|
695
816
|
}
|
|
696
817
|
|
|
@@ -704,6 +825,37 @@ if (args.includes('serve') || args.includes('view')) {
|
|
|
704
825
|
|
|
705
826
|
const { root: missionsRoot, files: discoveredMissions } = await discoverMissionFiles({ cwd: process.cwd() });
|
|
706
827
|
const tagConfig = await loadConfig(process.cwd());
|
|
828
|
+
const outputDir = resolveReportDir(tagConfig);
|
|
829
|
+
process.env.TESTRONAUT_OUTPUT_DIR = outputDir;
|
|
830
|
+
process.env.TESTRONAUT_RUN_ID = runId;
|
|
831
|
+
|
|
832
|
+
if (args.length === 1 && args[0] === 'config') {
|
|
833
|
+
writeOutput(buildEffectiveConfig(tagConfig));
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
if (args.length === 1 && args[0] === 'list') {
|
|
838
|
+
if (!discoveredMissions.length) {
|
|
839
|
+
writeOutput(jsonOutput ? { missions: [], root: missionsRoot } : `No missions found in ${path.relative(process.cwd(), missionsRoot) || '.'}.`);
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
const listedMissions = [];
|
|
843
|
+
if (!jsonOutput) console.log(`Missions in ${path.relative(process.cwd(), missionsRoot) || '.'}:`);
|
|
844
|
+
for (const file of discoveredMissions) {
|
|
845
|
+
try {
|
|
846
|
+
const mission = await loadMissionModule(path.resolve(missionsRoot, file));
|
|
847
|
+
const tags = normalizeTags(mission.tags);
|
|
848
|
+
listedMissions.push({ file, tags });
|
|
849
|
+
if (!jsonOutput) console.log(` ${file}${tags.length ? ` [${tags.join(', ')}]` : ''}`);
|
|
850
|
+
} catch (error) {
|
|
851
|
+
console.log(` ${file} [could not load: ${error.message}]`);
|
|
852
|
+
process.exitCode = 1;
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
if (jsonOutput) writeOutput({ root: missionsRoot, missions: listedMissions });
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
|
|
707
859
|
const explicitFiles = args.length > 0;
|
|
708
860
|
let requestedTags = [];
|
|
709
861
|
let tagMatch = 'any';
|
|
@@ -719,32 +871,71 @@ try {
|
|
|
719
871
|
process.exit(1);
|
|
720
872
|
}
|
|
721
873
|
|
|
722
|
-
if (!fs.existsSync(missionsRoot)) {
|
|
874
|
+
if (!explicitFiles && !fs.existsSync(missionsRoot)) {
|
|
723
875
|
console.error(`❌ Missions directory not found: ${path.relative(process.cwd(), missionsRoot)}`);
|
|
724
876
|
process.exit(1);
|
|
725
877
|
}
|
|
726
878
|
|
|
879
|
+
if (dryRun) {
|
|
880
|
+
const candidates = explicitFiles ? args : discoveredMissions;
|
|
881
|
+
const missions = [];
|
|
882
|
+
let invalid = false;
|
|
883
|
+
for (const file of candidates) {
|
|
884
|
+
const modulePath = resolveMissionPath(file, { cwd: process.cwd(), missionsRoot });
|
|
885
|
+
if (!fs.existsSync(modulePath)) {
|
|
886
|
+
missions.push({ file, status: 'missing', suggestion: closestMissionMatch(file, discoveredMissions) });
|
|
887
|
+
invalid = true;
|
|
888
|
+
continue;
|
|
889
|
+
}
|
|
890
|
+
try {
|
|
891
|
+
const mission = await loadMissionModule(modulePath);
|
|
892
|
+
const tags = normalizeTags(mission.tags);
|
|
893
|
+
const selected = explicitFiles || matchesTagFilter(tags, requestedTags, tagMatch);
|
|
894
|
+
missions.push({ file: path.relative(process.cwd(), modulePath), tags, selected, valid: typeof mission.executeMission === 'function' });
|
|
895
|
+
if (typeof mission.executeMission !== 'function') invalid = true;
|
|
896
|
+
} catch (error) {
|
|
897
|
+
missions.push({ file, status: 'load-error', error: error.message });
|
|
898
|
+
invalid = true;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
writeOutput({ dryRun: true, outputDir, provider: resolveProviderModel({ cwd: process.cwd() }), screenshots: screenshotsEnabled, missions });
|
|
902
|
+
if (invalid) process.exitCode = 1;
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
|
|
727
906
|
const runFile = async (filePath) => {
|
|
907
|
+
const modulePath = resolveMissionPath(filePath, { cwd: process.cwd(), missionsRoot });
|
|
728
908
|
try {
|
|
729
|
-
|
|
730
|
-
|
|
909
|
+
if (!fs.existsSync(modulePath)) {
|
|
910
|
+
const suggestion = closestMissionMatch(filePath, discoveredMissions);
|
|
911
|
+
console.error(`❌ Mission file not found: ${filePath}`);
|
|
912
|
+
if (suggestion) console.error(` Did you mean "${suggestion}"?`);
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
const missionsModule = await loadMissionModule(modulePath);
|
|
731
916
|
|
|
732
917
|
if (!explicitFiles) {
|
|
733
918
|
const moduleTags = normalizeTags(missionsModule.tags);
|
|
734
|
-
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return
|
|
919
|
+
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return null;
|
|
735
920
|
}
|
|
736
921
|
|
|
737
922
|
if (typeof missionsModule.executeMission === 'function') {
|
|
738
923
|
const result = await missionsModule.executeMission();
|
|
924
|
+
if (result == null) {
|
|
925
|
+
console.error(`❌ Mission did not return a result: ${filePath}`);
|
|
926
|
+
return false;
|
|
927
|
+
}
|
|
739
928
|
allResults.push({
|
|
740
|
-
file:
|
|
929
|
+
file: path.relative(process.cwd(), modulePath) || path.basename(modulePath),
|
|
741
930
|
result
|
|
742
931
|
});
|
|
743
|
-
|
|
932
|
+
const missionResults = Array.isArray(result) ? result : [result];
|
|
933
|
+
return !missionResults.some(item => item?.status === 'failed');
|
|
744
934
|
}
|
|
935
|
+
console.error(`❌ Mission does not export executeMission(): ${filePath}`);
|
|
745
936
|
} catch (err) {
|
|
746
937
|
console.error(`❌ Error running mission: ${filePath}`);
|
|
747
|
-
console.error(err);
|
|
938
|
+
console.error(` ${err?.message || err}`);
|
|
748
939
|
}
|
|
749
940
|
return false;
|
|
750
941
|
};
|
|
@@ -752,20 +943,27 @@ const runFile = async (filePath) => {
|
|
|
752
943
|
if (args.length > 0) {
|
|
753
944
|
// Run specific file(s)
|
|
754
945
|
for (const file of args) {
|
|
755
|
-
await runFile(file);
|
|
946
|
+
if (!await runFile(file)) process.exitCode = 1;
|
|
756
947
|
}
|
|
757
948
|
} else {
|
|
758
949
|
// Run missions discovered from config (or default behavior)
|
|
759
950
|
for (const file of discoveredMissions) {
|
|
760
|
-
await runFile(file);
|
|
951
|
+
if (await runFile(file) === false) process.exitCode = 1;
|
|
761
952
|
}
|
|
762
953
|
}
|
|
763
954
|
|
|
955
|
+
if (!explicitFiles && discoveredMissions.length === 0) {
|
|
956
|
+
console.error(`❌ No mission files found in ${path.relative(process.cwd(), missionsRoot) || '.'}.`);
|
|
957
|
+
process.exit(1);
|
|
958
|
+
}
|
|
959
|
+
|
|
764
960
|
if (!explicitFiles && requestedTags.length && allResults.length === 0) {
|
|
765
961
|
console.error(`❌ No missions matched ${tagMatch === 'all' ? 'all' : 'any'} of: ${requestedTags.join(', ')}`);
|
|
766
962
|
process.exit(1);
|
|
767
963
|
}
|
|
768
964
|
|
|
965
|
+
if (allResults.length === 0) return;
|
|
966
|
+
|
|
769
967
|
const endTime = new Date();
|
|
770
968
|
|
|
771
969
|
const flatMissions = allResults.flatMap(entry => {
|
|
@@ -794,6 +992,7 @@ const { provider: llmProvider, model: llmModel } = resolveProviderModel({ cwd: p
|
|
|
794
992
|
|
|
795
993
|
const report = {
|
|
796
994
|
runId,
|
|
995
|
+
cli: { version: CLI_VERSION },
|
|
797
996
|
startTime: startTime.toISOString(),
|
|
798
997
|
endTime: endTime.toISOString(),
|
|
799
998
|
llm: {
|
|
@@ -811,10 +1010,10 @@ report.tags = normalizeTags(flatMissions.flatMap(m =>
|
|
|
811
1010
|
m.submissionType === 'mission' ? (m.tags ?? []) : []
|
|
812
1011
|
));
|
|
813
1012
|
|
|
814
|
-
const outputDir = './missions/mission_reports';
|
|
815
1013
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
816
|
-
fs.writeFileSync(`${
|
|
817
|
-
generateHtmlReport(report, `${
|
|
1014
|
+
fs.writeFileSync(path.join(outputDir, `${runId}.json`), JSON.stringify(report, null, 2));
|
|
1015
|
+
generateHtmlReport(report, path.join(outputDir, `${runId}.html`));
|
|
1016
|
+
if (jsonOutput) writeOutput(report);
|
|
818
1017
|
|
|
819
1018
|
try {
|
|
820
1019
|
if (!process.env.TN_KEEP_TMP && fs.existsSync(TMP_DIR)) {
|
|
@@ -1010,7 +1209,7 @@ async function handleLogin() {
|
|
|
1010
1209
|
}
|
|
1011
1210
|
|
|
1012
1211
|
// Upload the most recent report
|
|
1013
|
-
async function uploadReport() {
|
|
1212
|
+
async function uploadReport(requestedReport, { uploadScreenshots: shouldUploadScreenshots = true } = {}) {
|
|
1014
1213
|
const configPath = path.resolve(process.cwd(), 'testronaut-config.json');
|
|
1015
1214
|
if (!fs.existsSync(configPath)) {
|
|
1016
1215
|
console.error('❌ Configuration file not found.');
|
|
@@ -1025,16 +1224,29 @@ async function uploadReport() {
|
|
|
1025
1224
|
|
|
1026
1225
|
|
|
1027
1226
|
// 1) Find latest report JSON
|
|
1028
|
-
const reportDir =
|
|
1029
|
-
|
|
1030
|
-
|
|
1227
|
+
const reportDir = resolveReportDir(config);
|
|
1228
|
+
if (!requestedReport && !fs.existsSync(reportDir)) {
|
|
1229
|
+
console.error(`❌ Report directory not found: ${path.relative(process.cwd(), reportDir) || reportDir}`);
|
|
1230
|
+
process.exit(1);
|
|
1231
|
+
}
|
|
1232
|
+
const files = fs.existsSync(reportDir) ? fs.readdirSync(reportDir).filter(f => f.endsWith('.json')) : [];
|
|
1233
|
+
if (!requestedReport && files.length === 0) {
|
|
1031
1234
|
console.error('❌ No report files found.');
|
|
1032
1235
|
process.exit(1);
|
|
1033
1236
|
}
|
|
1034
1237
|
files.sort((a, b) => parseInt(b.split('_')[1]) - parseInt(a.split('_')[1]));
|
|
1035
|
-
const
|
|
1036
|
-
|
|
1238
|
+
const selectedReportPath = requestedReport
|
|
1239
|
+
? resolveReportFile(requestedReport, reportDir)
|
|
1240
|
+
: path.join(reportDir, files[0]);
|
|
1241
|
+
if (!fs.existsSync(selectedReportPath)) {
|
|
1242
|
+
console.error(`❌ Report file not found: ${requestedReport}`);
|
|
1243
|
+
process.exit(1);
|
|
1244
|
+
}
|
|
1245
|
+
const latestReportPath = selectedReportPath;
|
|
1246
|
+
const latestReportFile = path.basename(selectedReportPath);
|
|
1037
1247
|
const reportJson = fs.readFileSync(latestReportPath, 'utf8');
|
|
1248
|
+
const report = JSON.parse(reportJson);
|
|
1249
|
+
const selectedReportDir = path.dirname(selectedReportPath);
|
|
1038
1250
|
|
|
1039
1251
|
// 2) Upload report FIRST and capture its ID
|
|
1040
1252
|
console.log(`🛰️ Uploading report: ${latestReportFile}`);
|
|
@@ -1061,10 +1273,17 @@ async function uploadReport() {
|
|
|
1061
1273
|
process.exit(1);
|
|
1062
1274
|
}
|
|
1063
1275
|
|
|
1276
|
+
if (!shouldUploadScreenshots) {
|
|
1277
|
+
console.log('ℹ️ Screenshot upload disabled. Report JSON upload complete.');
|
|
1278
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1064
1282
|
// 3) Now find the fixed 'screenshots' folder next to JSON
|
|
1065
|
-
const screenshotsDir = path.join(
|
|
1283
|
+
const screenshotsDir = path.join(selectedReportDir, 'screenshots');
|
|
1066
1284
|
if (!fs.existsSync(screenshotsDir) || !fs.statSync(screenshotsDir).isDirectory()) {
|
|
1067
1285
|
console.log('ℹ️ No screenshots directory found. Done.');
|
|
1286
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1068
1287
|
return;
|
|
1069
1288
|
}
|
|
1070
1289
|
|
|
@@ -1076,13 +1295,22 @@ async function uploadReport() {
|
|
|
1076
1295
|
return Date.UTC(+Y, +M - 1, +D, +h, +mnt, +s, +ms);
|
|
1077
1296
|
};
|
|
1078
1297
|
|
|
1079
|
-
|
|
1080
|
-
.
|
|
1081
|
-
|
|
1082
|
-
|
|
1298
|
+
const imageFiles = collectReportScreenshotPaths(report)
|
|
1299
|
+
.map(relativePath => ({
|
|
1300
|
+
relativePath,
|
|
1301
|
+
filePath: path.resolve(selectedReportDir, relativePath),
|
|
1302
|
+
}))
|
|
1303
|
+
.filter(({ filePath }) =>
|
|
1304
|
+
filePath.startsWith(`${path.resolve(screenshotsDir)}${path.sep}`) &&
|
|
1305
|
+
fs.existsSync(filePath) &&
|
|
1306
|
+
fs.statSync(filePath).isFile() &&
|
|
1307
|
+
/\.(png|jpg|jpeg|webp|gif)$/i.test(filePath)
|
|
1308
|
+
)
|
|
1309
|
+
.sort((a, b) => parseScreenshotTimestamp(path.basename(a.filePath)) - parseScreenshotTimestamp(path.basename(b.filePath)));
|
|
1083
1310
|
|
|
1084
1311
|
if (!imageFiles.length) {
|
|
1085
|
-
console.log('ℹ️
|
|
1312
|
+
console.log('ℹ️ This report does not reference any available screenshots. Done.');
|
|
1313
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: 0 });
|
|
1086
1314
|
return;
|
|
1087
1315
|
}
|
|
1088
1316
|
|
|
@@ -1101,11 +1329,10 @@ async function uploadReport() {
|
|
|
1101
1329
|
|
|
1102
1330
|
const failures = [];
|
|
1103
1331
|
for (let i = 0; i < imageFiles.length; i++) {
|
|
1104
|
-
const f = imageFiles[i];
|
|
1105
|
-
const filePath = path.join(screenshotsDir, f);
|
|
1332
|
+
const { filePath, relativePath: f } = imageFiles[i];
|
|
1106
1333
|
const stepIndex = i;
|
|
1107
1334
|
|
|
1108
|
-
process.stdout.write(` ➜ ${f} (step ${stepIndex}) … `);
|
|
1335
|
+
if (!quietOutput) process.stdout.write(` ➜ ${f} (step ${stepIndex}) … `);
|
|
1109
1336
|
try {
|
|
1110
1337
|
const buf = fs.readFileSync(filePath);
|
|
1111
1338
|
const stat = fs.statSync(filePath);
|
|
@@ -1154,9 +1381,9 @@ async function uploadReport() {
|
|
|
1154
1381
|
const finish = await parseJsonSafe(finishRes, 'uploads/finish');
|
|
1155
1382
|
if (!finish.ok) throw new Error(`uploads/finish responded ok=false`);
|
|
1156
1383
|
|
|
1157
|
-
process.stdout.write('ok\n');
|
|
1384
|
+
if (!quietOutput) process.stdout.write('ok\n');
|
|
1158
1385
|
} catch (err) {
|
|
1159
|
-
process.stdout.write('FAIL\n');
|
|
1386
|
+
if (!quietOutput) process.stdout.write('FAIL\n');
|
|
1160
1387
|
failures.push({ file: f, error: err.message || String(err) });
|
|
1161
1388
|
}
|
|
1162
1389
|
}
|
|
@@ -1165,8 +1392,10 @@ async function uploadReport() {
|
|
|
1165
1392
|
console.log('\n⚠️ Some screenshots failed to upload:');
|
|
1166
1393
|
for (const f of failures) console.log(` - ${f.file}: ${f.error}`);
|
|
1167
1394
|
process.exitCode = 1;
|
|
1395
|
+
if (jsonOutput) writeOutput({ ok: false, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: imageFiles.length - failures.length, failures });
|
|
1168
1396
|
} else {
|
|
1169
1397
|
console.log('✅ All screenshots uploaded.');
|
|
1398
|
+
if (jsonOutput) writeOutput({ ok: true, reportId: savedReportId, report: latestReportFile, screenshotsUploaded: imageFiles.length });
|
|
1170
1399
|
}
|
|
1171
1400
|
}
|
|
1172
1401
|
|
|
@@ -1248,11 +1477,12 @@ function findLatestReportPair(reportDir) {
|
|
|
1248
1477
|
}
|
|
1249
1478
|
|
|
1250
1479
|
async function serveLatestReport() {
|
|
1251
|
-
const
|
|
1480
|
+
const config = await loadConfig(process.cwd());
|
|
1481
|
+
const reportDir = resolveReportDir(config);
|
|
1252
1482
|
|
|
1253
1483
|
const latest = findLatestReportPair(reportDir);
|
|
1254
1484
|
if (!latest) {
|
|
1255
|
-
console.error(
|
|
1485
|
+
console.error(`❌ No HTML reports found in ${path.relative(process.cwd(), reportDir) || reportDir}.`);
|
|
1256
1486
|
process.exit(1);
|
|
1257
1487
|
}
|
|
1258
1488
|
|