testronaut 1.2.1 β 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -109
- package/bin/cli.js +86 -2
- package/core/agent.js +55 -39
- package/core/config.js +146 -0
- package/package.json +1 -1
- package/runner/testronaut.js +44 -21
- package/tests/coreTests/agent.test.js +127 -0
- package/tests/coreTests/config.test.js +146 -0
- package/tests/runnerTests/testronaut.test.js +164 -0
package/README.md
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
# π§βπ Testronaut
|
|
2
2
|
|
|
3
|
-
**Testronaut** is an autonomous testing framework powered by
|
|
3
|
+
**Testronaut** is an autonomous testing framework powered by **LLMs and Playwright**.
|
|
4
|
+
It lets you define *mission-based tests* in plain English, then runs them through a real browser to validate UI workflows β all while generating human-readable reports.
|
|
4
5
|
|
|
5
6
|
---
|
|
6
7
|
|
|
7
8
|
## π Join the Mission Control Community
|
|
8
9
|
|
|
9
|
-
Got questions, ideas, or
|
|
10
|
-
Join
|
|
10
|
+
Got questions, ideas, or missions to share?
|
|
11
|
+
Join the Discord to connect with other Testronauts, get support, and help shape the frameworkβs future.
|
|
11
12
|
|
|
12
13
|
[](https://discord.gg/pBfdef92ba)
|
|
13
14
|
|
|
@@ -15,46 +16,52 @@ Join our Discord to connect with other Testronauts, get support, and help shape
|
|
|
15
16
|
|
|
16
17
|
## π Features
|
|
17
18
|
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
-
|
|
21
|
-
-
|
|
22
|
-
-
|
|
23
|
-
-
|
|
19
|
+
- Write tests in plain English β no brittle selectors
|
|
20
|
+
- Runs real browser sessions via **Playwright**
|
|
21
|
+
- Works with **multiple LLM providers** (OpenAI, Google Gemini, and more coming)
|
|
22
|
+
- Modular tool and DOM-reasoning system
|
|
23
|
+
- Dynamic rate-limit and token-tracking logic
|
|
24
|
+
- Generates JSON + HTML reports automatically
|
|
24
25
|
|
|
25
26
|
---
|
|
26
27
|
|
|
27
28
|
## π Documentation
|
|
28
29
|
|
|
29
30
|
Looking for deeper guides, API references, and examples?
|
|
30
|
-
|
|
31
|
+
Check the official docs:
|
|
31
32
|
|
|
32
33
|
[](https://docs.testronaut.app)
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
-
|
|
36
|
-
- Writing advanced missions
|
|
37
|
-
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
35
|
+
Includes:
|
|
36
|
+
- Quickstart and setup
|
|
37
|
+
- Writing advanced missions
|
|
38
|
+
- Configuring providers and models
|
|
39
|
+
- CLI options
|
|
40
|
+
- Mission Control integration
|
|
41
|
+
- Troubleshooting and FAQs
|
|
40
42
|
|
|
41
43
|
---
|
|
42
44
|
|
|
43
45
|
## π¦ Installation
|
|
44
46
|
|
|
45
|
-
```
|
|
47
|
+
```bash
|
|
46
48
|
npm install -g testronaut
|
|
47
49
|
```
|
|
48
|
-
|
|
49
|
-
|
|
50
|
+
|
|
51
|
+
Then initialize your project:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
50
54
|
npx testronaut --init
|
|
51
55
|
```
|
|
52
|
-
|
|
53
|
-
|
|
56
|
+
|
|
57
|
+
Run the sample mission:
|
|
58
|
+
```bash
|
|
54
59
|
npx testronaut welcome.mission.js
|
|
55
60
|
```
|
|
56
|
-
|
|
57
|
-
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## π Project Structure
|
|
58
65
|
|
|
59
66
|
```
|
|
60
67
|
missions/
|
|
@@ -62,142 +69,137 @@ missions/
|
|
|
62
69
|
βββ logout.mission.js
|
|
63
70
|
βββ dashboard.mission.js
|
|
64
71
|
```
|
|
65
|
-
Each file exports a mission string or function and invokes runMissions.
|
|
66
72
|
|
|
67
|
-
|
|
68
|
-
Create a file in your missions/ directory, e.g. missions/login.mission.js:
|
|
73
|
+
Each mission exports a string or function and calls `runMissions`.
|
|
69
74
|
|
|
70
|
-
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## βοΈ Example Mission
|
|
78
|
+
|
|
79
|
+
```js
|
|
71
80
|
import { runMissions } from 'testronaut';
|
|
72
81
|
|
|
73
82
|
export const loginMission = `
|
|
74
83
|
Visit ${process.env.URL}.
|
|
75
|
-
Fill the username field with ${process.env.USERNAME}.
|
|
84
|
+
Fill the username field with ${process.env.USERNAME}.
|
|
76
85
|
Fill the password field with ${process.env.PASSWORD}.
|
|
77
|
-
|
|
78
|
-
Submit the form.
|
|
86
|
+
Click the Login button.
|
|
79
87
|
Wait for the dashboard to appear.
|
|
80
|
-
Take
|
|
81
|
-
Report SUCCESS if the dashboard is loaded, otherwise
|
|
88
|
+
Take a screenshot.
|
|
89
|
+
Report SUCCESS if the dashboard is loaded, otherwise FAILURE.
|
|
82
90
|
`;
|
|
83
91
|
|
|
84
92
|
export async function executeMission() {
|
|
85
|
-
await runMissions({
|
|
86
|
-
mission: loginMission
|
|
87
|
-
}, "Login Mission");
|
|
93
|
+
await runMissions({ mission: loginMission }, "Login Mission");
|
|
88
94
|
}
|
|
89
95
|
```
|
|
90
96
|
|
|
91
|
-
|
|
97
|
+
Create a `.env` file with your credentials **and LLM API key** (depending on your chosen provider):
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# For OpenAI
|
|
101
|
+
OPENAI_API_KEY=sk-...
|
|
102
|
+
|
|
103
|
+
# Or for Gemini
|
|
104
|
+
GEMINI_API_KEY=AIza...
|
|
92
105
|
|
|
93
|
-
```
|
|
94
|
-
OPENAI_API_KEY=sk-proj-############
|
|
95
106
|
URL=https://example.com/login
|
|
96
107
|
USERNAME=example@example.com
|
|
97
108
|
PASSWORD=********
|
|
98
109
|
```
|
|
99
110
|
|
|
100
|
-
|
|
111
|
+
---
|
|
101
112
|
|
|
102
|
-
|
|
103
|
-
import { runMissions } from 'testronaut';
|
|
104
|
-
import { loginMission } from './login.mission.js';
|
|
105
|
-
import { logoutMission } from './logout.mission.js';
|
|
106
|
-
import { navigateToContactFormMission } from './navigateToContactForm.mission.js';
|
|
107
|
-
|
|
108
|
-
export const fillContactFormMission =
|
|
109
|
-
`Input the text "example@example.com" in the email field.
|
|
110
|
-
Input the phone number (555)555-5555 into the phone number field.
|
|
111
|
-
Click the submit button.
|
|
112
|
-
Upon submission success there should be a toast notification indicating the form information was saved successfully.
|
|
113
|
-
Take a screenshot.
|
|
114
|
-
If found, report SUCCESS. Otherwise, report FAILURE.`
|
|
113
|
+
## π§ LLM Provider Support
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
preMission: [loginMission, navigateToContactFormMission],
|
|
119
|
-
mission: fillContactFormMission,
|
|
120
|
-
postMission: logoutMission,
|
|
121
|
-
}, "Fill Contact Form Mission");
|
|
122
|
-
}
|
|
123
|
-
```
|
|
115
|
+
Testronaut is provider-agnostic.
|
|
116
|
+
Choose your preferred LLM at init or via environment variables.
|
|
124
117
|
|
|
125
|
-
|
|
126
|
-
|
|
118
|
+
```bash
|
|
119
|
+
# During init
|
|
120
|
+
npx testronaut --init
|
|
127
121
|
|
|
122
|
+
# Or override anytime
|
|
123
|
+
TESTRONAUT_PROVIDER=gemini TESTRONAUT_MODEL=gemini-2.5-pro npx testronaut
|
|
128
124
|
```
|
|
125
|
+
|
|
126
|
+
Current supported providers:
|
|
127
|
+
|
|
128
|
+
| Provider | Example Models |
|
|
129
|
+
|-----------|----------------|
|
|
130
|
+
| **OpenAI** | gpt-4o, gpt-4.1, o3, gpt-5, etc. |
|
|
131
|
+
| **Google Gemini** | gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-8b |
|
|
132
|
+
|
|
133
|
+
More providers coming soon (Anthropic, Mistral, etc.).
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## π Running Missions
|
|
138
|
+
|
|
139
|
+
Run all missions:
|
|
140
|
+
```bash
|
|
129
141
|
npx testronaut
|
|
130
142
|
```
|
|
131
|
-
Run a specific file:
|
|
132
143
|
|
|
133
|
-
|
|
144
|
+
Run a specific mission:
|
|
145
|
+
```bash
|
|
134
146
|
npx testronaut login.mission.js
|
|
135
147
|
```
|
|
136
148
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
149
|
+
Chain missions together:
|
|
150
|
+
```js
|
|
151
|
+
await runMissions({
|
|
152
|
+
preMission: [loginMission],
|
|
153
|
+
mission: fillContactFormMission,
|
|
154
|
+
postMission: logoutMission,
|
|
155
|
+
}, "Contact Form Flow");
|
|
141
156
|
```
|
|
142
|
-
|
|
157
|
+
---
|
|
143
158
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
β Optional suite syntax
|
|
159
|
+
## π Reports
|
|
160
|
+
|
|
161
|
+
Testronaut generates both JSON and HTML reports automatically under:
|
|
148
162
|
|
|
149
163
|
```
|
|
150
|
-
|
|
164
|
+
missions/mission_reports/
|
|
151
165
|
```
|
|
152
|
-
β Executes all registered test objectives
|
|
153
|
-
|
|
154
|
-
### π Reports
|
|
155
|
-
HTML and JSON reports automatically created in the ```missions/mission_reports/``` directory for each testronaut run
|
|
156
166
|
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
Each includes:
|
|
168
|
+
- Steps executed
|
|
169
|
+
- Token usage
|
|
170
|
+
- Screenshots
|
|
171
|
+
- Pass/Fail summaries
|
|
159
172
|
|
|
160
|
-
|
|
173
|
+
---
|
|
161
174
|
|
|
162
|
-
|
|
175
|
+
## π§ͺ Under the Hood
|
|
163
176
|
|
|
164
|
-
|
|
165
|
-
|
|
177
|
+
- **Playwright** for browser automation
|
|
178
|
+
- **LLMs** for reasoning, DOM parsing, and tool use
|
|
179
|
+
- **Token throttling** + adaptive cooldowns
|
|
180
|
+
- **Extensible architecture** for custom tools and workflows
|
|
166
181
|
|
|
167
|
-
|
|
168
|
-
npm install
|
|
169
|
-
npm link
|
|
170
|
-
```
|
|
171
|
-
Now you can use testronaut in any local project.
|
|
182
|
+
---
|
|
172
183
|
|
|
173
|
-
|
|
174
|
-
MIT
|
|
184
|
+
## π§ Mission Control
|
|
175
185
|
|
|
176
|
-
|
|
186
|
+
[Mission Control](https://mission.testronaut.app) lets you:
|
|
187
|
+
- View all reports in one dashboard
|
|
188
|
+
- Track mission history and success rates
|
|
189
|
+
- Compare results across environments
|
|
190
|
+
- Access screenshots and step details anytime
|
|
177
191
|
|
|
178
|
-
|
|
179
|
-
Want to see your test results evolve, spot trends, and keep all your reports in one place?
|
|
180
|
-
Sign up for **Mission Control** β your personal dashboard for Testronaut.
|
|
192
|
+
---
|
|
181
193
|
|
|
182
|
-
|
|
183
|
-
- π Track your mission reports historically
|
|
184
|
-
- π Drill into past runs with full step-by-step details
|
|
185
|
-
- π οΈ Compare results between different builds or environments
|
|
186
|
-
- βοΈ Access reports from anywhere, without digging through local files
|
|
194
|
+
## π License
|
|
187
195
|
|
|
188
|
-
|
|
189
|
-
1. [](https://mission.testronaut.app)
|
|
190
|
-
2. Link your account to Testronaut CLI
|
|
191
|
-
3. Watch your mission history grow!
|
|
196
|
+
MIT
|
|
192
197
|
|
|
193
198
|
---
|
|
194
199
|
|
|
195
200
|
## β Support the Mission
|
|
196
|
-
π€ Built with β€οΈ by [Shane Fast](https://github.com/scfast)
|
|
197
201
|
|
|
198
|
-
|
|
199
|
-
Your support helps cover hosting, development time, and the occasional coffee needed to debug at 2am.
|
|
202
|
+
π€ Built with β€οΈ by [Shane Fast](https://github.com/scfast)
|
|
200
203
|
|
|
204
|
+
If Testronaut saves you time, consider fueling the mission:
|
|
201
205
|
[](https://buymeacoffee.com/testronaut)
|
|
202
|
-
|
|
203
|
-
---
|
package/bin/cli.js
CHANGED
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* cli.js
|
|
5
|
+
* -------
|
|
6
|
+
* Purpose:
|
|
7
|
+
* Single entry point for the Testronaut CLI:
|
|
8
|
+
* - Parses flags (e.g., --model, --turns, --init, help).
|
|
9
|
+
* - Runs mission files (single or all), aggregates results, and writes HTML/JSON reports.
|
|
10
|
+
* - Provides subcommands: login, upload (report + screenshots), serve/view (static file server).
|
|
11
|
+
*
|
|
12
|
+
* Key flags:
|
|
13
|
+
* --model <id> / --model=<id> β sets TESTRONAUT_MODEL env (wins over config file)
|
|
14
|
+
* --turns <n> / --turns=<n> β sets TESTRONAUT_TURNS env (wins over config file)
|
|
15
|
+
* --init β scaffolds project + optional Playwright browsers
|
|
16
|
+
* --help β prints help
|
|
17
|
+
*
|
|
18
|
+
* Notable helpers (defined below):
|
|
19
|
+
* parseJsonSafe(res, label) β tolerant JSON parse with good error messages
|
|
20
|
+
* pkgManagerForCwd(cwd) β detects npm/pnpm/yarn/bun
|
|
21
|
+
* hasAny(modNames) β βdo we have at least one of these deps?β (ESM-safe)
|
|
22
|
+
* installDev(pm, pkg) β dev-install a package with the detected package manager
|
|
23
|
+
* ensurePlaywrightInstalled() β install @playwright/test and browsers (skippable in CI)
|
|
24
|
+
* guessMimeType(p) β static server content types
|
|
25
|
+
* safeJoin(root, relUrlPath) β path traversal protection for static server
|
|
26
|
+
* findLatestReportPair(reportDir) β find latest run_<ts>.html (+ matching .json if present)
|
|
27
|
+
* serveLatestReport() β read-only file server for most recent HTML report
|
|
28
|
+
*
|
|
29
|
+
* Test strategy:
|
|
30
|
+
* To test pure helpers without running the whole CLI, we export a tiny test bundle:
|
|
31
|
+
* export const __test__ = { guessMimeType, safeJoin, findLatestReportPair, pkgManagerForCwd }
|
|
32
|
+
* See tests in tests/cliTests/cli.helpers.test.js
|
|
33
|
+
*/
|
|
34
|
+
|
|
2
35
|
import fs from 'fs';
|
|
3
36
|
import path from 'path';
|
|
4
37
|
import { fileURLToPath } from 'url';
|
|
@@ -46,6 +79,29 @@ if (modelFlagIndex >= 0) {
|
|
|
46
79
|
args.splice(modelFlagIndex, modelOverride ? 2 : 1);
|
|
47
80
|
}
|
|
48
81
|
|
|
82
|
+
// Look for --turns=<n> or --turns <n>
|
|
83
|
+
let turnsOverride;
|
|
84
|
+
const turnsFlagIndex = args.findIndex(a => a === '--turns' || a.startsWith('--turns='));
|
|
85
|
+
if (turnsFlagIndex >= 0) {
|
|
86
|
+
if (args[turnsFlagIndex].includes('=')) {
|
|
87
|
+
turnsOverride = args[turnsFlagIndex].split('=')[1];
|
|
88
|
+
} else if (args[turnsFlagIndex + 1]) {
|
|
89
|
+
turnsOverride = args[turnsFlagIndex + 1];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (turnsOverride) {
|
|
93
|
+
const n = Number(turnsOverride.trim());
|
|
94
|
+
if (Number.isFinite(n) && n > 0) {
|
|
95
|
+
process.env.TESTRONAUT_TURNS = String(n);
|
|
96
|
+
console.log(`π― Turn override: ${process.env.TESTRONAUT_TURNS}`);
|
|
97
|
+
} else {
|
|
98
|
+
console.warn(`β οΈ Invalid --turns value "${turnsOverride}". Ignoring.`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Remove flag & value so they arenβt treated as filenames
|
|
103
|
+
args.splice(turnsFlagIndex, turnsOverride ? 2 : 1);
|
|
104
|
+
}
|
|
49
105
|
|
|
50
106
|
const missionsDir = path.resolve(process.cwd(), 'missions');
|
|
51
107
|
|
|
@@ -66,6 +122,7 @@ Usage:
|
|
|
66
122
|
|
|
67
123
|
Options:
|
|
68
124
|
--init Scaffold project folders and a welcome mission
|
|
125
|
+
--turns=<n> Override max turns for this run (e.g., --turns=30)
|
|
69
126
|
--help Show this help message
|
|
70
127
|
|
|
71
128
|
Examples:
|
|
@@ -228,7 +285,13 @@ fs.mkdirSync(outputDir, { recursive: true });
|
|
|
228
285
|
fs.writeFileSync(`${outputDir}/${runId}.json`, JSON.stringify(report, null, 2));
|
|
229
286
|
generateHtmlReport(report, `${outputDir}/${runId}.html`);
|
|
230
287
|
|
|
231
|
-
|
|
288
|
+
/**
|
|
289
|
+
* Parse a fetch Response safely and return JSON or throw a rich error.
|
|
290
|
+
* @param {Response} res
|
|
291
|
+
* @param {string} urlLabel - human-friendly label for error messages
|
|
292
|
+
* @returns {Promise<any>}
|
|
293
|
+
* @throws when HTTP status is not ok or body is empty/invalid
|
|
294
|
+
*/
|
|
232
295
|
async function parseJsonSafe(res, urlLabel) {
|
|
233
296
|
const text = await res.text(); // read once
|
|
234
297
|
let data = null;
|
|
@@ -248,6 +311,11 @@ async function parseJsonSafe(res, urlLabel) {
|
|
|
248
311
|
return data;
|
|
249
312
|
}
|
|
250
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Detect the local package manager based on lockfiles.
|
|
316
|
+
* @param {string} [cwd=process.cwd()]
|
|
317
|
+
* @returns {'pnpm'|'yarn'|'bun'|'npm'}
|
|
318
|
+
*/
|
|
251
319
|
function pkgManagerForCwd(cwd = process.cwd()) {
|
|
252
320
|
const has = (f) => fs.existsSync(path.join(cwd, f));
|
|
253
321
|
if (has('pnpm-lock.yaml')) return 'pnpm';
|
|
@@ -559,7 +627,11 @@ async function uploadReport() {
|
|
|
559
627
|
}
|
|
560
628
|
}
|
|
561
629
|
|
|
562
|
-
|
|
630
|
+
/**
|
|
631
|
+
* Guess a reasonable MIME type for static serving.
|
|
632
|
+
* @param {string} p - file path
|
|
633
|
+
* @returns {string}
|
|
634
|
+
*/
|
|
563
635
|
function guessMimeType(p) {
|
|
564
636
|
const ext = path.extname(p).toLowerCase();
|
|
565
637
|
if (ext === '.html' || ext === '.htm') return 'text/html; charset=utf-8';
|
|
@@ -574,6 +646,12 @@ function guessMimeType(p) {
|
|
|
574
646
|
return 'application/octet-stream';
|
|
575
647
|
}
|
|
576
648
|
|
|
649
|
+
/**
|
|
650
|
+
* Path-join a request path to a root directory with traversal protection.
|
|
651
|
+
* @param {string} root - filesystem directory
|
|
652
|
+
* @param {string} relUrlPath - URL path from incoming request
|
|
653
|
+
* @returns {string|null} resolved path or null on attempted escape
|
|
654
|
+
*/
|
|
577
655
|
function safeJoin(root, relUrlPath) {
|
|
578
656
|
// Prevent path traversal and keep within root
|
|
579
657
|
const decoded = decodeURIComponent(relUrlPath);
|
|
@@ -593,6 +671,12 @@ async function openInBrowser(targetUrl) {
|
|
|
593
671
|
return exec(`xdg-open ${quoted}`).catch(() => {}); // best-effort on Linux
|
|
594
672
|
}
|
|
595
673
|
|
|
674
|
+
/**
|
|
675
|
+
* Find the latest run HTML (and its matching JSON, if present).
|
|
676
|
+
* Expects files like "run_<timestamp>.html" in the report directory.
|
|
677
|
+
* @param {string} reportDir
|
|
678
|
+
* @returns {{htmlFile:string, jsonFile:string|null}|null}
|
|
679
|
+
*/
|
|
596
680
|
function findLatestReportPair(reportDir) {
|
|
597
681
|
if (!fs.existsSync(reportDir)) return null;
|
|
598
682
|
const files = fs.readdirSync(reportDir).filter(f => f.endsWith('.html'));
|
package/core/agent.js
CHANGED
|
@@ -1,18 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* agent.js
|
|
3
|
+
* --------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Orchestrate a browser-backed autonomous agent over a sequence of goals.
|
|
6
|
+
*
|
|
7
|
+
* Responsibilities:
|
|
8
|
+
* - Start/stop the browser lifecycle.
|
|
9
|
+
* - For each goal, construct initial messages and invoke turnLoop().
|
|
10
|
+
* - Collect per-goal traces into a normalized missionResults array.
|
|
11
|
+
*
|
|
12
|
+
* Message contract (initial):
|
|
13
|
+
* messages = [
|
|
14
|
+
* { role: 'system', content: 'instructions...' },
|
|
15
|
+
* { role: 'user', content: <goal text> }
|
|
16
|
+
* ]
|
|
17
|
+
*
|
|
18
|
+
* Related tests:
|
|
19
|
+
* tests/agentTests/agent.test.js
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - cli/testronaut.js (runMissions)
|
|
23
|
+
*/
|
|
24
|
+
|
|
1
25
|
import 'dotenv/config';
|
|
2
26
|
import { ChromeBrowser } from '../tools/chromeBrowser.js';
|
|
3
27
|
import { turnLoop } from './turnLoop.js';
|
|
4
28
|
|
|
5
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Execute goals with a browser agent.
|
|
31
|
+
*
|
|
32
|
+
* @param {Array<{goal:any, submissionType?:string, submissionName?:string, label?:string}>} goals
|
|
33
|
+
* @param {string} missionName
|
|
34
|
+
* @param {number} [maxTurns=20] - upper bound for turnLoop per goal
|
|
35
|
+
* @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'failed', steps:any[], startTime:number, endTime:number}>>}
|
|
36
|
+
*/
|
|
37
|
+
export async function runAgent(goals, missionName, maxTurns = 20) {
|
|
6
38
|
const browser = new ChromeBrowser();
|
|
7
39
|
await browser.start();
|
|
8
|
-
const startTime = Date.now();
|
|
9
40
|
let result;
|
|
10
41
|
|
|
11
42
|
try {
|
|
12
43
|
const missionResults = [];
|
|
44
|
+
|
|
13
45
|
for (const goal of goals) {
|
|
14
|
-
// console.log(`\nπ Running Agent with goal:\n${goal}\n`);
|
|
15
46
|
const steps = [];
|
|
47
|
+
|
|
48
|
+
// Ensure user message is a string (functions/objects β toString fallback)
|
|
49
|
+
const userContent =
|
|
50
|
+
typeof goal.goal === 'string'
|
|
51
|
+
? goal.goal
|
|
52
|
+
: goal?.goal?.toString?.() ?? String(goal.goal);
|
|
53
|
+
|
|
16
54
|
const messages = [
|
|
17
55
|
{
|
|
18
56
|
role: 'system',
|
|
@@ -29,61 +67,39 @@ export async function runAgent(goals, missionName, maxTurns = 20, ) {
|
|
|
29
67
|
After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
|
|
30
68
|
`.trim(),
|
|
31
69
|
},
|
|
32
|
-
{ role: 'user', content:
|
|
70
|
+
{ role: 'user', content: userContent },
|
|
33
71
|
];
|
|
34
72
|
|
|
73
|
+
// Run the turn loop for this goal
|
|
35
74
|
result = await turnLoop(
|
|
36
|
-
browser,
|
|
37
|
-
messages,
|
|
38
|
-
maxTurns,
|
|
39
|
-
0,
|
|
40
|
-
0,
|
|
41
|
-
null,
|
|
42
|
-
{
|
|
43
|
-
steps, // π pass in
|
|
44
|
-
missionName, // π pass in (for tagging)
|
|
45
|
-
}
|
|
75
|
+
browser,
|
|
76
|
+
messages,
|
|
77
|
+
maxTurns,
|
|
78
|
+
0, // currentTurn
|
|
79
|
+
0, // retryCount
|
|
80
|
+
null, // currentStep
|
|
81
|
+
{ steps, missionName }
|
|
46
82
|
);
|
|
47
|
-
|
|
83
|
+
|
|
84
|
+
// Snapshot steps (avoid retaining references to mutable arrays)
|
|
48
85
|
missionResults.push({
|
|
49
86
|
missionName,
|
|
50
87
|
submissionType: goal.submissionType || goal.label,
|
|
51
88
|
submissionName: goal.submissionName || null,
|
|
52
89
|
status: result?.success ? 'passed' : 'failed',
|
|
53
|
-
steps: JSON.parse(JSON.stringify(steps)), //
|
|
54
|
-
startTime: Date.now() - 1, //
|
|
90
|
+
steps: JSON.parse(JSON.stringify(steps)), // deep-clone
|
|
91
|
+
startTime: Date.now() - 1, // TODO: wire actual timings if needed
|
|
55
92
|
endTime: Date.now(),
|
|
56
93
|
});
|
|
57
94
|
|
|
58
|
-
if (!result
|
|
95
|
+
if (!result?.success) {
|
|
59
96
|
console.log('π Agent stopped due to failed goal.\n');
|
|
60
97
|
return missionResults;
|
|
61
98
|
}
|
|
62
|
-
|
|
63
|
-
// if (!result.success) {
|
|
64
|
-
// console.log('π Agent stopped due to failed goal.\n');
|
|
65
|
-
// const endTime = Date.now();
|
|
66
|
-
// return {
|
|
67
|
-
// missionName: missionName,
|
|
68
|
-
// status: 'failed',
|
|
69
|
-
// steps: result.steps,
|
|
70
|
-
// startTime: startTime,
|
|
71
|
-
// endTime: endTime,
|
|
72
|
-
// };
|
|
73
|
-
// }
|
|
74
99
|
}
|
|
75
100
|
|
|
76
101
|
console.log('β
All goals completed successfully.\n');
|
|
77
|
-
// const endTime = Date.now();
|
|
78
102
|
return missionResults;
|
|
79
|
-
// return {
|
|
80
|
-
// missionName: missionName,
|
|
81
|
-
// status: 'passed',
|
|
82
|
-
// steps: result.steps,
|
|
83
|
-
// startTime: startTime,
|
|
84
|
-
// endTime: endTime,
|
|
85
|
-
// };
|
|
86
|
-
|
|
87
103
|
} finally {
|
|
88
104
|
await browser.close();
|
|
89
105
|
}
|
package/core/config.js
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config.js
|
|
3
|
+
* ----------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Centralized config loading and turn-budget guardrails for Testronaut.
|
|
6
|
+
*
|
|
7
|
+
* Responsibilities:
|
|
8
|
+
* - Load `testronaut-config.json` (best effort).
|
|
9
|
+
* - Read specific knobs with safe fallbacks (e.g., maxTurns).
|
|
10
|
+
* - Define baseline limits (soft/hard/min/idle/errors/time).
|
|
11
|
+
* - Enforce user requests via clamp + warn (default) or fail-fast (strict).
|
|
12
|
+
*
|
|
13
|
+
* Related tests:
|
|
14
|
+
* tests/configTests/config.test.js
|
|
15
|
+
*
|
|
16
|
+
* Used by:
|
|
17
|
+
* - cli/testronaut.js (to derive effectiveMax turns per run)
|
|
18
|
+
* - core/agent.js / core/turnLoop.js (runtime stop conditions)
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// core/config.js
|
|
22
|
+
import { readFile } from 'node:fs/promises';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Load testronaut-config.json from the given cwd.
|
|
27
|
+
* Returns {} when missing or invalid JSON.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} [cwd=process.cwd()]
|
|
30
|
+
* @returns {Promise<object>}
|
|
31
|
+
*/
|
|
32
|
+
export async function loadConfig(cwd = process.cwd()) {
|
|
33
|
+
try {
|
|
34
|
+
const raw = await readFile(path.join(cwd, 'testronaut-config.json'), 'utf8');
|
|
35
|
+
return JSON.parse(raw);
|
|
36
|
+
} catch {
|
|
37
|
+
// No config or invalid JSON β fall back to defaults at call sites
|
|
38
|
+
return {};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Resolve the requested maxTurns from config with a safe fallback.
|
|
44
|
+
*
|
|
45
|
+
* @param {object} cfg
|
|
46
|
+
* @param {number} [fallback=20]
|
|
47
|
+
* @returns {number}
|
|
48
|
+
*/
|
|
49
|
+
export function getMaxTurns(cfg, fallback = 20) {
|
|
50
|
+
const n = Number(cfg?.maxTurns);
|
|
51
|
+
if (!Number.isFinite(n) || n <= 0) return fallback;
|
|
52
|
+
return n;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Baseline limits for mission runs. Adjust here for global defaults.
|
|
57
|
+
*
|
|
58
|
+
* @returns {{
|
|
59
|
+
* softMaxTurns:number,
|
|
60
|
+
* hardMaxTurns:number,
|
|
61
|
+
* hardMinTurns:number,
|
|
62
|
+
* maxIdleTurns:number,
|
|
63
|
+
* maxErrors:number,
|
|
64
|
+
* maxSeconds:number
|
|
65
|
+
* }}
|
|
66
|
+
*/
|
|
67
|
+
export function resolveTurnLimits() {
|
|
68
|
+
const softMaxTurns = 50;
|
|
69
|
+
const hardMaxTurns = 200;
|
|
70
|
+
const hardMinTurns = 5;
|
|
71
|
+
const maxIdleTurns = 6;
|
|
72
|
+
const maxErrors = 5;
|
|
73
|
+
const maxSeconds = 600; //10 minutes
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
softMaxTurns,
|
|
77
|
+
hardMaxTurns,
|
|
78
|
+
hardMinTurns,
|
|
79
|
+
maxIdleTurns,
|
|
80
|
+
maxErrors,
|
|
81
|
+
maxSeconds,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Compute an effective turn budget and normalized limits from config.
|
|
87
|
+
* - Default: clamp + warn (lenient).
|
|
88
|
+
* - Strict mode (cfg.strictLimits or env STRICT_LIMITS): throw on violations.
|
|
89
|
+
*
|
|
90
|
+
* @param {object} cfg
|
|
91
|
+
* @param {ReturnType<typeof resolveTurnLimits>} [baseLimits=resolveTurnLimits()]
|
|
92
|
+
* @returns {{
|
|
93
|
+
* effectiveMax:number,
|
|
94
|
+
* limits: ReturnType<typeof resolveTurnLimits>,
|
|
95
|
+
* notes:string[],
|
|
96
|
+
* strict:boolean
|
|
97
|
+
* }}
|
|
98
|
+
*/
|
|
99
|
+
export function enforceTurnBudget(cfg, baseLimits = resolveTurnLimits()) {
|
|
100
|
+
const strict = !!(cfg?.strictLimits || process.env.STRICT_LIMITS);
|
|
101
|
+
|
|
102
|
+
const limits = { ...baseLimits };
|
|
103
|
+
const notes = [];
|
|
104
|
+
|
|
105
|
+
// Normalize soft/hard ordering
|
|
106
|
+
if (limits.softMaxTurns > limits.hardMaxTurns) {
|
|
107
|
+
if (strict) throw new Error(`[limits] softMaxTurns (${limits.softMaxTurns}) > hardMaxTurns (${limits.hardMaxTurns}).`);
|
|
108
|
+
notes.push(`Adjusted softMaxTurns down to hardMaxTurns (${limits.hardMaxTurns}).`);
|
|
109
|
+
limits.softMaxTurns = limits.hardMaxTurns;
|
|
110
|
+
}
|
|
111
|
+
if (limits.hardMinTurns > limits.softMaxTurns) {
|
|
112
|
+
if (strict) throw new Error(`[limits] hardMinTurns (${limits.hardMinTurns}) > softMaxTurns (${limits.softMaxTurns}).`);
|
|
113
|
+
notes.push(`Adjusted hardMinTurns down to softMaxTurns (${limits.softMaxTurns}).`);
|
|
114
|
+
limits.hardMinTurns = limits.softMaxTurns;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Resolve requested maxTurns (env override wins over config)
|
|
118
|
+
const envTurns = Number(process.env.TESTRONAUT_TURNS);
|
|
119
|
+
const hasEnv = Number.isFinite(envTurns) && envTurns > 0;
|
|
120
|
+
const cfgTurns = Number(cfg?.maxTurns);
|
|
121
|
+
const hasCfg = Number.isFinite(cfgTurns) && cfgTurns > 0;
|
|
122
|
+
let requested = hasEnv ? envTurns : (hasCfg ? cfgTurns : 20);
|
|
123
|
+
let effectiveMax = requested;
|
|
124
|
+
|
|
125
|
+
if (hasEnv) {
|
|
126
|
+
notes.push(`βΉοΈ Using CLI turn override (TESTRONAUT_TURNS=${envTurns}).`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (effectiveMax > limits.hardMaxTurns) {
|
|
130
|
+
const msg = `maxTurns (${effectiveMax}) exceeds hardMaxTurns (${limits.hardMaxTurns}).`;
|
|
131
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
132
|
+
notes.push(`β οΈ ${msg} Clamping to ${limits.hardMaxTurns}.`);
|
|
133
|
+
effectiveMax = limits.hardMaxTurns;
|
|
134
|
+
} else if (effectiveMax > limits.softMaxTurns) {
|
|
135
|
+
const msg = `maxTurns (${effectiveMax}) exceeds softMaxTurns (${limits.softMaxTurns}).`;
|
|
136
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
137
|
+
notes.push(`β οΈ ${msg} Proceeding, but consider lowering to control cost.`);
|
|
138
|
+
} else if (effectiveMax < limits.hardMinTurns) {
|
|
139
|
+
const msg = `maxTurns (${effectiveMax}) is below hardMinTurns (${limits.hardMinTurns}).`;
|
|
140
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
141
|
+
notes.push(`β οΈ ${msg} Raising to ${limits.hardMinTurns}.`);
|
|
142
|
+
effectiveMax = limits.hardMinTurns;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return { effectiveMax, limits, notes, strict };
|
|
146
|
+
}
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -1,27 +1,49 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
1
|
+
/**
|
|
2
|
+
* testronaut.js
|
|
3
|
+
* -------------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* High-level mission runner: normalize pre/main/post submissions,
|
|
6
|
+
* compute a safe turn budget from config, and invoke the agent.
|
|
7
|
+
*
|
|
8
|
+
* Responsibilities:
|
|
9
|
+
* - Read config once and derive turn limits via enforceTurnBudget().
|
|
10
|
+
* - Build a normalized list of submissions with human-friendly names.
|
|
11
|
+
* - Redact sensitive text when logging mission strings.
|
|
12
|
+
* - Call runAgent(goals, missionName, maxTurns) and post-process status.
|
|
13
|
+
*
|
|
14
|
+
* Message contract (goal β initial messages inside agent):
|
|
15
|
+
* - system: operational guidance + success/failure contract
|
|
16
|
+
* - user: the mission text (or coerced string)
|
|
17
|
+
*
|
|
18
|
+
* Related tests:
|
|
19
|
+
* tests/missionTests/testronaut.test.js
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - CLI entrypoint(s)
|
|
23
|
+
*/
|
|
20
24
|
|
|
21
25
|
|
|
26
|
+
import { runAgent } from '../core/agent.js';
|
|
27
|
+
import { redactPasswordInText } from '../core/redaction.js';
|
|
28
|
+
import { loadConfig, enforceTurnBudget } from '../core/config.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Run a mission flow.
|
|
32
|
+
*
|
|
33
|
+
* @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
|
|
34
|
+
* @param {string} missionName
|
|
35
|
+
*/
|
|
22
36
|
export async function runMissions({ preMission, mission, postMission }, missionName) {
|
|
23
|
-
|
|
37
|
+
// 1) Config and turn budget (with guardrails)
|
|
38
|
+
const cfg = await loadConfig();
|
|
39
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
40
|
+
const maxTurns = effectiveMax;
|
|
41
|
+
if (notes.length) {
|
|
42
|
+
console.warn(notes.join('\n'));
|
|
43
|
+
}
|
|
24
44
|
|
|
45
|
+
// 2) Normalize submissions
|
|
46
|
+
const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
|
|
25
47
|
const pre = normalizeToArray(preMission);
|
|
26
48
|
const main = normalizeToArray(mission);
|
|
27
49
|
const post = normalizeToArray(postMission);
|
|
@@ -100,7 +122,8 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
100
122
|
})
|
|
101
123
|
);
|
|
102
124
|
|
|
103
|
-
|
|
125
|
+
// 3) Execute
|
|
126
|
+
const success = await runAgent(goals, missionName, maxTurns);
|
|
104
127
|
if (!success) {
|
|
105
128
|
console.log(`β Aborting after failed goal.`);
|
|
106
129
|
return;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// tests/coreTests/agent.test.js
|
|
2
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// 1) Provide a constructable mock without using `vi` inside the factory.
|
|
5
|
+
vi.mock('../../tools/chromeBrowser.js', () => {
|
|
6
|
+
function ChromeBrowser() {}
|
|
7
|
+
ChromeBrowser.prototype.start = async function () {};
|
|
8
|
+
ChromeBrowser.prototype.close = async function () {};
|
|
9
|
+
return { ChromeBrowser };
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
// Keep turnLoop simple; using vi is fine here (no factory-hoist issue)
|
|
13
|
+
vi.mock('../../core/turnLoop.js', () => ({
|
|
14
|
+
turnLoop: vi.fn(),
|
|
15
|
+
}));
|
|
16
|
+
|
|
17
|
+
import { ChromeBrowser } from '../../tools/chromeBrowser.js';
|
|
18
|
+
import { turnLoop } from '../../core/turnLoop.js';
|
|
19
|
+
import { runAgent } from '../../core/agent.js';
|
|
20
|
+
|
|
21
|
+
let startSpy, closeSpy;
|
|
22
|
+
|
|
23
|
+
describe('core/agent.runAgent', () => {
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
// 2) Spy on prototype methods AFTER import time
|
|
26
|
+
startSpy = vi.spyOn(ChromeBrowser.prototype, 'start').mockResolvedValue();
|
|
27
|
+
closeSpy = vi.spyOn(ChromeBrowser.prototype, 'close').mockResolvedValue();
|
|
28
|
+
vi.clearAllMocks();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
startSpy.mockRestore();
|
|
33
|
+
closeSpy.mockRestore();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('runs all goals and returns missionResults (all passed)', async () => {
|
|
37
|
+
turnLoop
|
|
38
|
+
.mockResolvedValueOnce({ success: true })
|
|
39
|
+
.mockResolvedValueOnce({ success: true });
|
|
40
|
+
|
|
41
|
+
const goals = [
|
|
42
|
+
{ goal: 'Do A', submissionType: 'mission', submissionName: 'A' },
|
|
43
|
+
{ goal: 'Do B', submissionType: 'mission', submissionName: 'B' },
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
const res = await runAgent(goals, 'My Mission', 10);
|
|
47
|
+
|
|
48
|
+
// Browser lifecycle
|
|
49
|
+
expect(startSpy).toHaveBeenCalledTimes(1);
|
|
50
|
+
expect(closeSpy).toHaveBeenCalledTimes(1);
|
|
51
|
+
|
|
52
|
+
// turnLoop called once per goal and receives maxTurns
|
|
53
|
+
expect(turnLoop).toHaveBeenCalledTimes(2);
|
|
54
|
+
expect(turnLoop.mock.calls[0][2]).toBe(10);
|
|
55
|
+
expect(turnLoop.mock.calls[1][2]).toBe(10);
|
|
56
|
+
|
|
57
|
+
expect(res).toHaveLength(2);
|
|
58
|
+
expect(res[0].status).toBe('passed');
|
|
59
|
+
expect(res[1].status).toBe('passed');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('stops early on first failure and returns partial results', async () => {
|
|
63
|
+
turnLoop.mockResolvedValueOnce({ success: false });
|
|
64
|
+
|
|
65
|
+
const res = await runAgent(
|
|
66
|
+
[
|
|
67
|
+
{ goal: 'A', submissionType: 'mission', submissionName: 'A' },
|
|
68
|
+
{ goal: 'B', submissionType: 'mission', submissionName: 'B' },
|
|
69
|
+
],
|
|
70
|
+
'Fail Mission',
|
|
71
|
+
7
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
expect(turnLoop).toHaveBeenCalledTimes(1);
|
|
75
|
+
expect(res).toHaveLength(1);
|
|
76
|
+
expect(res[0].status).toBe('failed');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('coerces non-string goal to string for the user message', async () => {
|
|
80
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
81
|
+
|
|
82
|
+
await runAgent([{ goal: { toString: () => 'coerced text' } }], 'Coerce', 5);
|
|
83
|
+
|
|
84
|
+
const messages = turnLoop.mock.calls[0][1];
|
|
85
|
+
expect(messages[1]).toEqual({ role: 'user', content: 'coerced text' });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('deep-clones steps so external mutations do not leak', async () => {
|
|
89
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
90
|
+
ctx.steps.push({ result: 'SUCCESS: step1' });
|
|
91
|
+
return { success: true };
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const res1 = await runAgent([{ goal: 'Check clone' }], 'Clone Test', 3);
|
|
95
|
+
res1[0].steps[0].result = 'TAMPERED';
|
|
96
|
+
|
|
97
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
98
|
+
ctx.steps.push({ result: 'SUCCESS: step2' });
|
|
99
|
+
return { success: true };
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const res2 = await runAgent([{ goal: 'Check clone' }], 'Clone Test 2', 3);
|
|
103
|
+
expect(res1[0].steps[0].result).toBe('TAMPERED');
|
|
104
|
+
expect(res2[0].steps[0].result).toBe('SUCCESS: step2');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('includes missionName and submission metadata in results', async () => {
|
|
108
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
109
|
+
|
|
110
|
+
const res = await runAgent(
|
|
111
|
+
[
|
|
112
|
+
{
|
|
113
|
+
goal: 'Meta',
|
|
114
|
+
submissionType: 'mission',
|
|
115
|
+
submissionName: 'MetaName',
|
|
116
|
+
label: 'legacy-label',
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
'MetaMission',
|
|
120
|
+
4
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
expect(res[0].missionName).toBe('MetaMission');
|
|
124
|
+
expect(res[0].submissionType).toBe('mission');
|
|
125
|
+
expect(res[0].submissionName).toBe('MetaName');
|
|
126
|
+
});
|
|
127
|
+
});
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
vi.mock('node:fs/promises', () => ({
|
|
4
|
+
readFile: vi.fn(),
|
|
5
|
+
}));
|
|
6
|
+
|
|
7
|
+
import { readFile } from 'node:fs/promises';
|
|
8
|
+
import {
|
|
9
|
+
loadConfig,
|
|
10
|
+
getMaxTurns,
|
|
11
|
+
resolveTurnLimits,
|
|
12
|
+
enforceTurnBudget,
|
|
13
|
+
} from '../../core/config.js';
|
|
14
|
+
|
|
15
|
+
describe('core/config', () => {
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
vi.resetAllMocks();
|
|
18
|
+
delete process.env.STRICT_LIMITS;
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe('loadConfig', () => {
|
|
22
|
+
it('parses JSON when present', async () => {
|
|
23
|
+
readFile.mockResolvedValueOnce(JSON.stringify({ maxTurns: 33 }));
|
|
24
|
+
const cfg = await loadConfig('/any');
|
|
25
|
+
expect(cfg).toEqual({ maxTurns: 33 });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('returns {} on ENOENT', async () => {
|
|
29
|
+
readFile.mockRejectedValueOnce(new Error('ENOENT'));
|
|
30
|
+
const cfg = await loadConfig('/missing');
|
|
31
|
+
expect(cfg).toEqual({});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('returns {} on invalid JSON', async () => {
|
|
35
|
+
readFile.mockResolvedValueOnce('{ nope ');
|
|
36
|
+
const cfg = await loadConfig('/bad');
|
|
37
|
+
expect(cfg).toEqual({});
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
describe('getMaxTurns', () => {
|
|
42
|
+
it('uses fallback when invalid or missing', () => {
|
|
43
|
+
expect(getMaxTurns({}, 20)).toBe(20);
|
|
44
|
+
expect(getMaxTurns({ maxTurns: 'x' }, 20)).toBe(20);
|
|
45
|
+
expect(getMaxTurns({ maxTurns: 0 }, 20)).toBe(20);
|
|
46
|
+
expect(getMaxTurns({ maxTurns: -5 }, 20)).toBe(20);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('returns numeric when valid', () => {
|
|
50
|
+
expect(getMaxTurns({ maxTurns: 7 }, 20)).toBe(7);
|
|
51
|
+
expect(getMaxTurns({ maxTurns: '15' }, 20)).toBe(15);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
describe('resolveTurnLimits', () => {
|
|
56
|
+
it('returns expected defaults', () => {
|
|
57
|
+
const l = resolveTurnLimits();
|
|
58
|
+
expect(l.softMaxTurns).toBe(50);
|
|
59
|
+
expect(l.hardMaxTurns).toBe(200);
|
|
60
|
+
expect(l.hardMinTurns).toBe(5);
|
|
61
|
+
expect(l.maxIdleTurns).toBe(6);
|
|
62
|
+
expect(l.maxErrors).toBe(5);
|
|
63
|
+
expect(l.maxSeconds).toBe(600);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('enforceTurnBudget (lenient)', () => {
|
|
68
|
+
it('clamps > hardMaxTurns', () => {
|
|
69
|
+
const cfg = { maxTurns: 1000 };
|
|
70
|
+
const { effectiveMax, limits, notes, strict } = enforceTurnBudget(cfg);
|
|
71
|
+
expect(strict).toBe(false);
|
|
72
|
+
expect(effectiveMax).toBe(limits.hardMaxTurns);
|
|
73
|
+
expect(notes.join('\n')).toMatch(/Clamping to/i);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('warns when > softMaxTurns but <= hardMaxTurns', () => {
|
|
77
|
+
const cfg = { maxTurns: 120 };
|
|
78
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
79
|
+
expect(effectiveMax).toBe(120);
|
|
80
|
+
expect(notes.join('\n')).toMatch(/exceeds softMaxTurns/i);
|
|
81
|
+
expect(effectiveMax).toBeLessThanOrEqual(limits.hardMaxTurns);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('raises to hardMinTurns when below minimum', () => {
|
|
85
|
+
const cfg = { maxTurns: 1 };
|
|
86
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
87
|
+
expect(effectiveMax).toBe(limits.hardMinTurns);
|
|
88
|
+
expect(notes.join('\n')).toMatch(/Raising to/i);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('normalizes soft>hard ordering with a note', () => {
|
|
92
|
+
const weird = {
|
|
93
|
+
softMaxTurns: 300,
|
|
94
|
+
hardMaxTurns: 200,
|
|
95
|
+
hardMinTurns: 5,
|
|
96
|
+
maxIdleTurns: 6,
|
|
97
|
+
maxErrors: 5,
|
|
98
|
+
maxSeconds: 600,
|
|
99
|
+
};
|
|
100
|
+
const { limits, notes } = enforceTurnBudget({ maxTurns: 20 }, weird);
|
|
101
|
+
expect(limits.softMaxTurns).toBe(limits.hardMaxTurns);
|
|
102
|
+
expect(notes.join('\n')).toMatch(/Adjusted softMaxTurns down to hardMaxTurns/);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe('enforceTurnBudget (strict)', () => {
|
|
107
|
+
it('throws when > hardMaxTurns', () => {
|
|
108
|
+
const cfg = { maxTurns: 999, strictLimits: true };
|
|
109
|
+
expect(() => enforceTurnBudget(cfg)).toThrow(/exceeds hardMaxTurns/i);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it('throws when below hardMinTurns', () => {
|
|
113
|
+
process.env.STRICT_LIMITS = '1';
|
|
114
|
+
const cfg = { maxTurns: 2 };
|
|
115
|
+
expect(() => enforceTurnBudget(cfg)).toThrow(/below hardMinTurns/i);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it('throws when softMaxTurns > hardMaxTurns', () => {
|
|
119
|
+
const bad = {
|
|
120
|
+
softMaxTurns: 300,
|
|
121
|
+
hardMaxTurns: 200,
|
|
122
|
+
hardMinTurns: 5,
|
|
123
|
+
maxIdleTurns: 6,
|
|
124
|
+
maxErrors: 5,
|
|
125
|
+
maxSeconds: 600,
|
|
126
|
+
};
|
|
127
|
+
expect(() => enforceTurnBudget({ strictLimits: true, maxTurns: 20 }, bad))
|
|
128
|
+
.toThrow(/softMaxTurns .* > hardMaxTurns/);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
describe('enforceTurnBudget env override', () => {
|
|
134
|
+
const old = process.env.TESTRONAUT_TURNS;
|
|
135
|
+
|
|
136
|
+
beforeEach(() => { delete process.env.TESTRONAUT_TURNS; });
|
|
137
|
+
afterEach(() => { if (old === undefined) delete process.env.TESTRONAUT_TURNS; else process.env.TESTRONAUT_TURNS = old; });
|
|
138
|
+
|
|
139
|
+
it('uses TESTRONAUT_TURNS over cfg.maxTurns', () => {
|
|
140
|
+
process.env.TESTRONAUT_TURNS = '30';
|
|
141
|
+
const { effectiveMax, notes } = enforceTurnBudget({ maxTurns: 5 });
|
|
142
|
+
expect(effectiveMax).toBe(30);
|
|
143
|
+
expect(notes.join('\n')).toMatch(/CLI turn override/i);
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
});
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
// Mocks
|
|
4
|
+
vi.mock('../../core/agent.js', () => ({ runAgent: vi.fn() }));
|
|
5
|
+
vi.mock('../../core/redaction.js', () => ({
|
|
6
|
+
// simple deterministic redaction stub
|
|
7
|
+
redactPasswordInText: (s) => String(s).replace(/password:\s*\S+/gi, 'password: ********'),
|
|
8
|
+
}));
|
|
9
|
+
vi.mock('../../core/config.js', () => ({
|
|
10
|
+
loadConfig: vi.fn(),
|
|
11
|
+
enforceTurnBudget: vi.fn(),
|
|
12
|
+
}));
|
|
13
|
+
|
|
14
|
+
import { runAgent } from '../../core/agent.js';
|
|
15
|
+
import { loadConfig, enforceTurnBudget } from '../../core/config.js';
|
|
16
|
+
|
|
17
|
+
// Adjust the import path if your file lives elsewhere
|
|
18
|
+
import { runMissions } from '../../runner/testronaut.js';
|
|
19
|
+
|
|
20
|
+
describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
vi.clearAllMocks();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('passes effectiveMax to runAgent and logs any notes', async () => {
|
|
26
|
+
loadConfig.mockResolvedValue({ maxTurns: 999 });
|
|
27
|
+
enforceTurnBudget.mockReturnValue({
|
|
28
|
+
effectiveMax: 200,
|
|
29
|
+
limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
|
|
30
|
+
notes: ['β οΈ maxTurns (999) exceeds hardMaxTurns (200). Clamping to 200.'],
|
|
31
|
+
strict: false,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
35
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
36
|
+
|
|
37
|
+
runAgent.mockResolvedValue([
|
|
38
|
+
{ steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
await runMissions({ mission: 'Do xyz' }, 'Budgeted Run');
|
|
42
|
+
|
|
43
|
+
expect(loadConfig).toHaveBeenCalledTimes(1);
|
|
44
|
+
expect(enforceTurnBudget).toHaveBeenCalledWith({ maxTurns: 999 });
|
|
45
|
+
expect(warn).toHaveBeenCalledWith(
|
|
46
|
+
expect.stringContaining('Clamping to 200')
|
|
47
|
+
);
|
|
48
|
+
expect(runAgent).toHaveBeenCalledWith(
|
|
49
|
+
expect.any(Array), 'Budgeted Run', 200
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
warn.mockRestore();
|
|
53
|
+
log.mockRestore();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('does not warn when no notes are returned', async () => {
|
|
57
|
+
loadConfig.mockResolvedValue({ maxTurns: 20 });
|
|
58
|
+
enforceTurnBudget.mockReturnValue({
|
|
59
|
+
effectiveMax: 20,
|
|
60
|
+
limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
|
|
61
|
+
notes: [],
|
|
62
|
+
strict: false,
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
66
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
67
|
+
|
|
68
|
+
runAgent.mockResolvedValue([
|
|
69
|
+
{ steps: [{ result: 'SUCCESS: ok' }], status: 'passed' },
|
|
70
|
+
]);
|
|
71
|
+
|
|
72
|
+
await runMissions({ mission: 'No warnings' }, 'Clean');
|
|
73
|
+
|
|
74
|
+
expect(warn).not.toHaveBeenCalled();
|
|
75
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20);
|
|
76
|
+
|
|
77
|
+
warn.mockRestore();
|
|
78
|
+
log.mockRestore();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('marks first mission failed when last step result includes "failure"', async () => {
|
|
82
|
+
loadConfig.mockResolvedValue({});
|
|
83
|
+
enforceTurnBudget.mockReturnValue({
|
|
84
|
+
effectiveMax: 20,
|
|
85
|
+
limits: {},
|
|
86
|
+
notes: [],
|
|
87
|
+
strict: false,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
91
|
+
runAgent.mockResolvedValue([
|
|
92
|
+
{ steps: [{ result: 'FAILURE: could not do it' }], status: 'passed' },
|
|
93
|
+
]);
|
|
94
|
+
|
|
95
|
+
const res = await runMissions({ mission: 'Will fail' }, 'Status');
|
|
96
|
+
expect(res[0].status).toBe('failed');
|
|
97
|
+
|
|
98
|
+
log.mockRestore();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('normalizes single strings and functions into arrays and names missions', async () => {
|
|
102
|
+
loadConfig.mockResolvedValue({});
|
|
103
|
+
enforceTurnBudget.mockReturnValue({
|
|
104
|
+
effectiveMax: 15,
|
|
105
|
+
limits: {},
|
|
106
|
+
notes: [],
|
|
107
|
+
strict: false,
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
111
|
+
runAgent.mockResolvedValue([
|
|
112
|
+
{ steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
function preTask() {}
|
|
116
|
+
function postTask() {}
|
|
117
|
+
|
|
118
|
+
await runMissions(
|
|
119
|
+
{
|
|
120
|
+
preMission: preTask,
|
|
121
|
+
mission: 'Main mission text\nwith details',
|
|
122
|
+
postMission: postTask,
|
|
123
|
+
},
|
|
124
|
+
'My Mission'
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
// Check that runAgent got 3 goals (pre/main/post)
|
|
128
|
+
const call = (runAgent.mock.calls[0] || []);
|
|
129
|
+
const goals = call[0];
|
|
130
|
+
expect(Array.isArray(goals)).toBe(true);
|
|
131
|
+
expect(goals).toHaveLength(3);
|
|
132
|
+
|
|
133
|
+
// Submission types and names are set
|
|
134
|
+
expect(goals[0].submissionType).toBe('premission');
|
|
135
|
+
expect(goals[1].submissionType).toBe('mission');
|
|
136
|
+
expect(goals[2].submissionType).toBe('postmission');
|
|
137
|
+
|
|
138
|
+
// Mission name is the passed-in missionName for main
|
|
139
|
+
expect(goals[1].submissionName).toMatch(/^My Mission/);
|
|
140
|
+
|
|
141
|
+
// Effective max turns passed through
|
|
142
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15);
|
|
143
|
+
|
|
144
|
+
log.mockRestore();
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it('aborts early when runAgent returns falsy (defensive)', async () => {
|
|
148
|
+
loadConfig.mockResolvedValue({});
|
|
149
|
+
enforceTurnBudget.mockReturnValue({
|
|
150
|
+
effectiveMax: 10,
|
|
151
|
+
limits: {},
|
|
152
|
+
notes: [],
|
|
153
|
+
strict: false,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
157
|
+
runAgent.mockResolvedValue(null); // simulate unexpected falsy
|
|
158
|
+
|
|
159
|
+
const res = await runMissions({ mission: 'Edge case' }, 'Abort');
|
|
160
|
+
expect(res).toBeUndefined();
|
|
161
|
+
|
|
162
|
+
log.mockRestore();
|
|
163
|
+
});
|
|
164
|
+
});
|