testronaut 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +86 -2
- package/core/agent.js +55 -39
- package/core/config.js +146 -0
- package/package.json +1 -1
- package/runner/testronaut.js +44 -21
- package/tests/coreTests/agent.test.js +127 -0
- package/tests/coreTests/config.test.js +146 -0
- package/tests/runnerTests/testronaut.test.js +164 -0
package/bin/cli.js
CHANGED
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* cli.js
|
|
5
|
+
* -------
|
|
6
|
+
* Purpose:
|
|
7
|
+
* Single entry point for the Testronaut CLI:
|
|
8
|
+
* - Parses flags (e.g., --model, --turns, --init, help).
|
|
9
|
+
* - Runs mission files (single or all), aggregates results, and writes HTML/JSON reports.
|
|
10
|
+
* - Provides subcommands: login, upload (report + screenshots), serve/view (static file server).
|
|
11
|
+
*
|
|
12
|
+
* Key flags:
|
|
13
|
+
* --model <id> / --model=<id> → sets TESTRONAUT_MODEL env (wins over config file)
|
|
14
|
+
* --turns <n> / --turns=<n> → sets TESTRONAUT_TURNS env (wins over config file)
|
|
15
|
+
* --init → scaffolds project + optional Playwright browsers
|
|
16
|
+
* --help → prints help
|
|
17
|
+
*
|
|
18
|
+
* Notable helpers (defined below):
|
|
19
|
+
* parseJsonSafe(res, label) → tolerant JSON parse with good error messages
|
|
20
|
+
* pkgManagerForCwd(cwd) → detects npm/pnpm/yarn/bun
|
|
21
|
+
* hasAny(modNames) → “do we have at least one of these deps?” (ESM-safe)
|
|
22
|
+
* installDev(pm, pkg) → dev-install a package with the detected package manager
|
|
23
|
+
* ensurePlaywrightInstalled() → install @playwright/test and browsers (skippable in CI)
|
|
24
|
+
* guessMimeType(p) → static server content types
|
|
25
|
+
* safeJoin(root, relUrlPath) → path traversal protection for static server
|
|
26
|
+
* findLatestReportPair(reportDir) → find latest run_<ts>.html (+ matching .json if present)
|
|
27
|
+
* serveLatestReport() → read-only file server for most recent HTML report
|
|
28
|
+
*
|
|
29
|
+
* Test strategy:
|
|
30
|
+
* To test pure helpers without running the whole CLI, we export a tiny test bundle:
|
|
31
|
+
* export const __test__ = { guessMimeType, safeJoin, findLatestReportPair, pkgManagerForCwd }
|
|
32
|
+
* See tests in tests/cliTests/cli.helpers.test.js
|
|
33
|
+
*/
|
|
34
|
+
|
|
2
35
|
import fs from 'fs';
|
|
3
36
|
import path from 'path';
|
|
4
37
|
import { fileURLToPath } from 'url';
|
|
@@ -46,6 +79,29 @@ if (modelFlagIndex >= 0) {
|
|
|
46
79
|
args.splice(modelFlagIndex, modelOverride ? 2 : 1);
|
|
47
80
|
}
|
|
48
81
|
|
|
82
|
+
// Look for --turns=<n> or --turns <n>
|
|
83
|
+
let turnsOverride;
|
|
84
|
+
const turnsFlagIndex = args.findIndex(a => a === '--turns' || a.startsWith('--turns='));
|
|
85
|
+
if (turnsFlagIndex >= 0) {
|
|
86
|
+
if (args[turnsFlagIndex].includes('=')) {
|
|
87
|
+
turnsOverride = args[turnsFlagIndex].split('=')[1];
|
|
88
|
+
} else if (args[turnsFlagIndex + 1]) {
|
|
89
|
+
turnsOverride = args[turnsFlagIndex + 1];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (turnsOverride) {
|
|
93
|
+
const n = Number(turnsOverride.trim());
|
|
94
|
+
if (Number.isFinite(n) && n > 0) {
|
|
95
|
+
process.env.TESTRONAUT_TURNS = String(n);
|
|
96
|
+
console.log(`🎯 Turn override: ${process.env.TESTRONAUT_TURNS}`);
|
|
97
|
+
} else {
|
|
98
|
+
console.warn(`⚠️ Invalid --turns value "${turnsOverride}". Ignoring.`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Remove flag & value so they aren’t treated as filenames
|
|
103
|
+
args.splice(turnsFlagIndex, turnsOverride ? 2 : 1);
|
|
104
|
+
}
|
|
49
105
|
|
|
50
106
|
const missionsDir = path.resolve(process.cwd(), 'missions');
|
|
51
107
|
|
|
@@ -66,6 +122,7 @@ Usage:
|
|
|
66
122
|
|
|
67
123
|
Options:
|
|
68
124
|
--init Scaffold project folders and a welcome mission
|
|
125
|
+
--turns=<n> Override max turns for this run (e.g., --turns=30)
|
|
69
126
|
--help Show this help message
|
|
70
127
|
|
|
71
128
|
Examples:
|
|
@@ -228,7 +285,13 @@ fs.mkdirSync(outputDir, { recursive: true });
|
|
|
228
285
|
fs.writeFileSync(`${outputDir}/${runId}.json`, JSON.stringify(report, null, 2));
|
|
229
286
|
generateHtmlReport(report, `${outputDir}/${runId}.html`);
|
|
230
287
|
|
|
231
|
-
|
|
288
|
+
/**
|
|
289
|
+
* Parse a fetch Response safely and return JSON or throw a rich error.
|
|
290
|
+
* @param {Response} res
|
|
291
|
+
* @param {string} urlLabel - human-friendly label for error messages
|
|
292
|
+
* @returns {Promise<any>}
|
|
293
|
+
* @throws when HTTP status is not ok or body is empty/invalid
|
|
294
|
+
*/
|
|
232
295
|
async function parseJsonSafe(res, urlLabel) {
|
|
233
296
|
const text = await res.text(); // read once
|
|
234
297
|
let data = null;
|
|
@@ -248,6 +311,11 @@ async function parseJsonSafe(res, urlLabel) {
|
|
|
248
311
|
return data;
|
|
249
312
|
}
|
|
250
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Detect the local package manager based on lockfiles.
|
|
316
|
+
* @param {string} [cwd=process.cwd()]
|
|
317
|
+
* @returns {'pnpm'|'yarn'|'bun'|'npm'}
|
|
318
|
+
*/
|
|
251
319
|
function pkgManagerForCwd(cwd = process.cwd()) {
|
|
252
320
|
const has = (f) => fs.existsSync(path.join(cwd, f));
|
|
253
321
|
if (has('pnpm-lock.yaml')) return 'pnpm';
|
|
@@ -559,7 +627,11 @@ async function uploadReport() {
|
|
|
559
627
|
}
|
|
560
628
|
}
|
|
561
629
|
|
|
562
|
-
|
|
630
|
+
/**
|
|
631
|
+
* Guess a reasonable MIME type for static serving.
|
|
632
|
+
* @param {string} p - file path
|
|
633
|
+
* @returns {string}
|
|
634
|
+
*/
|
|
563
635
|
function guessMimeType(p) {
|
|
564
636
|
const ext = path.extname(p).toLowerCase();
|
|
565
637
|
if (ext === '.html' || ext === '.htm') return 'text/html; charset=utf-8';
|
|
@@ -574,6 +646,12 @@ function guessMimeType(p) {
|
|
|
574
646
|
return 'application/octet-stream';
|
|
575
647
|
}
|
|
576
648
|
|
|
649
|
+
/**
|
|
650
|
+
* Path-join a request path to a root directory with traversal protection.
|
|
651
|
+
* @param {string} root - filesystem directory
|
|
652
|
+
* @param {string} relUrlPath - URL path from incoming request
|
|
653
|
+
* @returns {string|null} resolved path or null on attempted escape
|
|
654
|
+
*/
|
|
577
655
|
function safeJoin(root, relUrlPath) {
|
|
578
656
|
// Prevent path traversal and keep within root
|
|
579
657
|
const decoded = decodeURIComponent(relUrlPath);
|
|
@@ -593,6 +671,12 @@ async function openInBrowser(targetUrl) {
|
|
|
593
671
|
return exec(`xdg-open ${quoted}`).catch(() => {}); // best-effort on Linux
|
|
594
672
|
}
|
|
595
673
|
|
|
674
|
+
/**
|
|
675
|
+
* Find the latest run HTML (and its matching JSON, if present).
|
|
676
|
+
* Expects files like "run_<timestamp>.html" in the report directory.
|
|
677
|
+
* @param {string} reportDir
|
|
678
|
+
* @returns {{htmlFile:string, jsonFile:string|null}|null}
|
|
679
|
+
*/
|
|
596
680
|
function findLatestReportPair(reportDir) {
|
|
597
681
|
if (!fs.existsSync(reportDir)) return null;
|
|
598
682
|
const files = fs.readdirSync(reportDir).filter(f => f.endsWith('.html'));
|
package/core/agent.js
CHANGED
|
@@ -1,18 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* agent.js
|
|
3
|
+
* --------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Orchestrate a browser-backed autonomous agent over a sequence of goals.
|
|
6
|
+
*
|
|
7
|
+
* Responsibilities:
|
|
8
|
+
* - Start/stop the browser lifecycle.
|
|
9
|
+
* - For each goal, construct initial messages and invoke turnLoop().
|
|
10
|
+
* - Collect per-goal traces into a normalized missionResults array.
|
|
11
|
+
*
|
|
12
|
+
* Message contract (initial):
|
|
13
|
+
* messages = [
|
|
14
|
+
* { role: 'system', content: 'instructions...' },
|
|
15
|
+
* { role: 'user', content: <goal text> }
|
|
16
|
+
* ]
|
|
17
|
+
*
|
|
18
|
+
* Related tests:
|
|
19
|
+
* tests/agentTests/agent.test.js
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - cli/testronaut.js (runMissions)
|
|
23
|
+
*/
|
|
24
|
+
|
|
1
25
|
import 'dotenv/config';
|
|
2
26
|
import { ChromeBrowser } from '../tools/chromeBrowser.js';
|
|
3
27
|
import { turnLoop } from './turnLoop.js';
|
|
4
28
|
|
|
5
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Execute goals with a browser agent.
|
|
31
|
+
*
|
|
32
|
+
* @param {Array<{goal:any, submissionType?:string, submissionName?:string, label?:string}>} goals
|
|
33
|
+
* @param {string} missionName
|
|
34
|
+
* @param {number} [maxTurns=20] - upper bound for turnLoop per goal
|
|
35
|
+
* @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'failed', steps:any[], startTime:number, endTime:number}>>}
|
|
36
|
+
*/
|
|
37
|
+
export async function runAgent(goals, missionName, maxTurns = 20) {
|
|
6
38
|
const browser = new ChromeBrowser();
|
|
7
39
|
await browser.start();
|
|
8
|
-
const startTime = Date.now();
|
|
9
40
|
let result;
|
|
10
41
|
|
|
11
42
|
try {
|
|
12
43
|
const missionResults = [];
|
|
44
|
+
|
|
13
45
|
for (const goal of goals) {
|
|
14
|
-
// console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
|
|
15
46
|
const steps = [];
|
|
47
|
+
|
|
48
|
+
// Ensure user message is a string (functions/objects → toString fallback)
|
|
49
|
+
const userContent =
|
|
50
|
+
typeof goal.goal === 'string'
|
|
51
|
+
? goal.goal
|
|
52
|
+
: goal?.goal?.toString?.() ?? String(goal.goal);
|
|
53
|
+
|
|
16
54
|
const messages = [
|
|
17
55
|
{
|
|
18
56
|
role: 'system',
|
|
@@ -29,61 +67,39 @@ export async function runAgent(goals, missionName, maxTurns = 20, ) {
|
|
|
29
67
|
After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
|
|
30
68
|
`.trim(),
|
|
31
69
|
},
|
|
32
|
-
{ role: 'user', content:
|
|
70
|
+
{ role: 'user', content: userContent },
|
|
33
71
|
];
|
|
34
72
|
|
|
73
|
+
// Run the turn loop for this goal
|
|
35
74
|
result = await turnLoop(
|
|
36
|
-
browser,
|
|
37
|
-
messages,
|
|
38
|
-
maxTurns,
|
|
39
|
-
0,
|
|
40
|
-
0,
|
|
41
|
-
null,
|
|
42
|
-
{
|
|
43
|
-
steps, // 👈 pass in
|
|
44
|
-
missionName, // 👈 pass in (for tagging)
|
|
45
|
-
}
|
|
75
|
+
browser,
|
|
76
|
+
messages,
|
|
77
|
+
maxTurns,
|
|
78
|
+
0, // currentTurn
|
|
79
|
+
0, // retryCount
|
|
80
|
+
null, // currentStep
|
|
81
|
+
{ steps, missionName }
|
|
46
82
|
);
|
|
47
|
-
|
|
83
|
+
|
|
84
|
+
// Snapshot steps (avoid retaining references to mutable arrays)
|
|
48
85
|
missionResults.push({
|
|
49
86
|
missionName,
|
|
50
87
|
submissionType: goal.submissionType || goal.label,
|
|
51
88
|
submissionName: goal.submissionName || null,
|
|
52
89
|
status: result?.success ? 'passed' : 'failed',
|
|
53
|
-
steps: JSON.parse(JSON.stringify(steps)), //
|
|
54
|
-
startTime: Date.now() - 1, //
|
|
90
|
+
steps: JSON.parse(JSON.stringify(steps)), // deep-clone
|
|
91
|
+
startTime: Date.now() - 1, // TODO: wire actual timings if needed
|
|
55
92
|
endTime: Date.now(),
|
|
56
93
|
});
|
|
57
94
|
|
|
58
|
-
if (!result
|
|
95
|
+
if (!result?.success) {
|
|
59
96
|
console.log('🛑 Agent stopped due to failed goal.\n');
|
|
60
97
|
return missionResults;
|
|
61
98
|
}
|
|
62
|
-
|
|
63
|
-
// if (!result.success) {
|
|
64
|
-
// console.log('🛑 Agent stopped due to failed goal.\n');
|
|
65
|
-
// const endTime = Date.now();
|
|
66
|
-
// return {
|
|
67
|
-
// missionName: missionName,
|
|
68
|
-
// status: 'failed',
|
|
69
|
-
// steps: result.steps,
|
|
70
|
-
// startTime: startTime,
|
|
71
|
-
// endTime: endTime,
|
|
72
|
-
// };
|
|
73
|
-
// }
|
|
74
99
|
}
|
|
75
100
|
|
|
76
101
|
console.log('✅ All goals completed successfully.\n');
|
|
77
|
-
// const endTime = Date.now();
|
|
78
102
|
return missionResults;
|
|
79
|
-
// return {
|
|
80
|
-
// missionName: missionName,
|
|
81
|
-
// status: 'passed',
|
|
82
|
-
// steps: result.steps,
|
|
83
|
-
// startTime: startTime,
|
|
84
|
-
// endTime: endTime,
|
|
85
|
-
// };
|
|
86
|
-
|
|
87
103
|
} finally {
|
|
88
104
|
await browser.close();
|
|
89
105
|
}
|
package/core/config.js
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* config.js
|
|
3
|
+
* ----------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Centralized config loading and turn-budget guardrails for Testronaut.
|
|
6
|
+
*
|
|
7
|
+
* Responsibilities:
|
|
8
|
+
* - Load `testronaut-config.json` (best effort).
|
|
9
|
+
* - Read specific knobs with safe fallbacks (e.g., maxTurns).
|
|
10
|
+
* - Define baseline limits (soft/hard/min/idle/errors/time).
|
|
11
|
+
* - Enforce user requests via clamp + warn (default) or fail-fast (strict).
|
|
12
|
+
*
|
|
13
|
+
* Related tests:
|
|
14
|
+
* tests/configTests/config.test.js
|
|
15
|
+
*
|
|
16
|
+
* Used by:
|
|
17
|
+
* - cli/testronaut.js (to derive effectiveMax turns per run)
|
|
18
|
+
* - core/agent.js / core/turnLoop.js (runtime stop conditions)
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// core/config.js
|
|
22
|
+
import { readFile } from 'node:fs/promises';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Load testronaut-config.json from the given cwd.
|
|
27
|
+
* Returns {} when missing or invalid JSON.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} [cwd=process.cwd()]
|
|
30
|
+
* @returns {Promise<object>}
|
|
31
|
+
*/
|
|
32
|
+
export async function loadConfig(cwd = process.cwd()) {
|
|
33
|
+
try {
|
|
34
|
+
const raw = await readFile(path.join(cwd, 'testronaut-config.json'), 'utf8');
|
|
35
|
+
return JSON.parse(raw);
|
|
36
|
+
} catch {
|
|
37
|
+
// No config or invalid JSON — fall back to defaults at call sites
|
|
38
|
+
return {};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Resolve the requested maxTurns from config with a safe fallback.
|
|
44
|
+
*
|
|
45
|
+
* @param {object} cfg
|
|
46
|
+
* @param {number} [fallback=20]
|
|
47
|
+
* @returns {number}
|
|
48
|
+
*/
|
|
49
|
+
export function getMaxTurns(cfg, fallback = 20) {
|
|
50
|
+
const n = Number(cfg?.maxTurns);
|
|
51
|
+
if (!Number.isFinite(n) || n <= 0) return fallback;
|
|
52
|
+
return n;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Baseline limits for mission runs. Adjust here for global defaults.
|
|
57
|
+
*
|
|
58
|
+
* @returns {{
|
|
59
|
+
* softMaxTurns:number,
|
|
60
|
+
* hardMaxTurns:number,
|
|
61
|
+
* hardMinTurns:number,
|
|
62
|
+
* maxIdleTurns:number,
|
|
63
|
+
* maxErrors:number,
|
|
64
|
+
* maxSeconds:number
|
|
65
|
+
* }}
|
|
66
|
+
*/
|
|
67
|
+
export function resolveTurnLimits() {
|
|
68
|
+
const softMaxTurns = 50;
|
|
69
|
+
const hardMaxTurns = 200;
|
|
70
|
+
const hardMinTurns = 5;
|
|
71
|
+
const maxIdleTurns = 6;
|
|
72
|
+
const maxErrors = 5;
|
|
73
|
+
const maxSeconds = 600; //10 minutes
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
softMaxTurns,
|
|
77
|
+
hardMaxTurns,
|
|
78
|
+
hardMinTurns,
|
|
79
|
+
maxIdleTurns,
|
|
80
|
+
maxErrors,
|
|
81
|
+
maxSeconds,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Compute an effective turn budget and normalized limits from config.
|
|
87
|
+
* - Default: clamp + warn (lenient).
|
|
88
|
+
* - Strict mode (cfg.strictLimits or env STRICT_LIMITS): throw on violations.
|
|
89
|
+
*
|
|
90
|
+
* @param {object} cfg
|
|
91
|
+
* @param {ReturnType<typeof resolveTurnLimits>} [baseLimits=resolveTurnLimits()]
|
|
92
|
+
* @returns {{
|
|
93
|
+
* effectiveMax:number,
|
|
94
|
+
* limits: ReturnType<typeof resolveTurnLimits>,
|
|
95
|
+
* notes:string[],
|
|
96
|
+
* strict:boolean
|
|
97
|
+
* }}
|
|
98
|
+
*/
|
|
99
|
+
export function enforceTurnBudget(cfg, baseLimits = resolveTurnLimits()) {
|
|
100
|
+
const strict = !!(cfg?.strictLimits || process.env.STRICT_LIMITS);
|
|
101
|
+
|
|
102
|
+
const limits = { ...baseLimits };
|
|
103
|
+
const notes = [];
|
|
104
|
+
|
|
105
|
+
// Normalize soft/hard ordering
|
|
106
|
+
if (limits.softMaxTurns > limits.hardMaxTurns) {
|
|
107
|
+
if (strict) throw new Error(`[limits] softMaxTurns (${limits.softMaxTurns}) > hardMaxTurns (${limits.hardMaxTurns}).`);
|
|
108
|
+
notes.push(`Adjusted softMaxTurns down to hardMaxTurns (${limits.hardMaxTurns}).`);
|
|
109
|
+
limits.softMaxTurns = limits.hardMaxTurns;
|
|
110
|
+
}
|
|
111
|
+
if (limits.hardMinTurns > limits.softMaxTurns) {
|
|
112
|
+
if (strict) throw new Error(`[limits] hardMinTurns (${limits.hardMinTurns}) > softMaxTurns (${limits.softMaxTurns}).`);
|
|
113
|
+
notes.push(`Adjusted hardMinTurns down to softMaxTurns (${limits.softMaxTurns}).`);
|
|
114
|
+
limits.hardMinTurns = limits.softMaxTurns;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Resolve requested maxTurns (env override wins over config)
|
|
118
|
+
const envTurns = Number(process.env.TESTRONAUT_TURNS);
|
|
119
|
+
const hasEnv = Number.isFinite(envTurns) && envTurns > 0;
|
|
120
|
+
const cfgTurns = Number(cfg?.maxTurns);
|
|
121
|
+
const hasCfg = Number.isFinite(cfgTurns) && cfgTurns > 0;
|
|
122
|
+
let requested = hasEnv ? envTurns : (hasCfg ? cfgTurns : 20);
|
|
123
|
+
let effectiveMax = requested;
|
|
124
|
+
|
|
125
|
+
if (hasEnv) {
|
|
126
|
+
notes.push(`ℹ️ Using CLI turn override (TESTRONAUT_TURNS=${envTurns}).`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (effectiveMax > limits.hardMaxTurns) {
|
|
130
|
+
const msg = `maxTurns (${effectiveMax}) exceeds hardMaxTurns (${limits.hardMaxTurns}).`;
|
|
131
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
132
|
+
notes.push(`⚠️ ${msg} Clamping to ${limits.hardMaxTurns}.`);
|
|
133
|
+
effectiveMax = limits.hardMaxTurns;
|
|
134
|
+
} else if (effectiveMax > limits.softMaxTurns) {
|
|
135
|
+
const msg = `maxTurns (${effectiveMax}) exceeds softMaxTurns (${limits.softMaxTurns}).`;
|
|
136
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
137
|
+
notes.push(`⚠️ ${msg} Proceeding, but consider lowering to control cost.`);
|
|
138
|
+
} else if (effectiveMax < limits.hardMinTurns) {
|
|
139
|
+
const msg = `maxTurns (${effectiveMax}) is below hardMinTurns (${limits.hardMinTurns}).`;
|
|
140
|
+
if (strict) throw new Error(`[limits] ${msg}`);
|
|
141
|
+
notes.push(`⚠️ ${msg} Raising to ${limits.hardMinTurns}.`);
|
|
142
|
+
effectiveMax = limits.hardMinTurns;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return { effectiveMax, limits, notes, strict };
|
|
146
|
+
}
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -1,27 +1,49 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
1
|
+
/**
|
|
2
|
+
* testronaut.js
|
|
3
|
+
* -------------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* High-level mission runner: normalize pre/main/post submissions,
|
|
6
|
+
* compute a safe turn budget from config, and invoke the agent.
|
|
7
|
+
*
|
|
8
|
+
* Responsibilities:
|
|
9
|
+
* - Read config once and derive turn limits via enforceTurnBudget().
|
|
10
|
+
* - Build a normalized list of submissions with human-friendly names.
|
|
11
|
+
* - Redact sensitive text when logging mission strings.
|
|
12
|
+
* - Call runAgent(goals, missionName, maxTurns) and post-process status.
|
|
13
|
+
*
|
|
14
|
+
* Message contract (goal → initial messages inside agent):
|
|
15
|
+
* - system: operational guidance + success/failure contract
|
|
16
|
+
* - user: the mission text (or coerced string)
|
|
17
|
+
*
|
|
18
|
+
* Related tests:
|
|
19
|
+
* tests/missionTests/testronaut.test.js
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - CLI entrypoint(s)
|
|
23
|
+
*/
|
|
20
24
|
|
|
21
25
|
|
|
26
|
+
import { runAgent } from '../core/agent.js';
|
|
27
|
+
import { redactPasswordInText } from '../core/redaction.js';
|
|
28
|
+
import { loadConfig, enforceTurnBudget } from '../core/config.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Run a mission flow.
|
|
32
|
+
*
|
|
33
|
+
* @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
|
|
34
|
+
* @param {string} missionName
|
|
35
|
+
*/
|
|
22
36
|
export async function runMissions({ preMission, mission, postMission }, missionName) {
|
|
23
|
-
|
|
37
|
+
// 1) Config and turn budget (with guardrails)
|
|
38
|
+
const cfg = await loadConfig();
|
|
39
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
40
|
+
const maxTurns = effectiveMax;
|
|
41
|
+
if (notes.length) {
|
|
42
|
+
console.warn(notes.join('\n'));
|
|
43
|
+
}
|
|
24
44
|
|
|
45
|
+
// 2) Normalize submissions
|
|
46
|
+
const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
|
|
25
47
|
const pre = normalizeToArray(preMission);
|
|
26
48
|
const main = normalizeToArray(mission);
|
|
27
49
|
const post = normalizeToArray(postMission);
|
|
@@ -100,7 +122,8 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
100
122
|
})
|
|
101
123
|
);
|
|
102
124
|
|
|
103
|
-
|
|
125
|
+
// 3) Execute
|
|
126
|
+
const success = await runAgent(goals, missionName, maxTurns);
|
|
104
127
|
if (!success) {
|
|
105
128
|
console.log(`❌ Aborting after failed goal.`);
|
|
106
129
|
return;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// tests/coreTests/agent.test.js
|
|
2
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// 1) Provide a constructable mock without using `vi` inside the factory.
|
|
5
|
+
vi.mock('../../tools/chromeBrowser.js', () => {
|
|
6
|
+
function ChromeBrowser() {}
|
|
7
|
+
ChromeBrowser.prototype.start = async function () {};
|
|
8
|
+
ChromeBrowser.prototype.close = async function () {};
|
|
9
|
+
return { ChromeBrowser };
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
// Keep turnLoop simple; using vi is fine here (no factory-hoist issue)
|
|
13
|
+
vi.mock('../../core/turnLoop.js', () => ({
|
|
14
|
+
turnLoop: vi.fn(),
|
|
15
|
+
}));
|
|
16
|
+
|
|
17
|
+
import { ChromeBrowser } from '../../tools/chromeBrowser.js';
|
|
18
|
+
import { turnLoop } from '../../core/turnLoop.js';
|
|
19
|
+
import { runAgent } from '../../core/agent.js';
|
|
20
|
+
|
|
21
|
+
let startSpy, closeSpy;
|
|
22
|
+
|
|
23
|
+
describe('core/agent.runAgent', () => {
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
// 2) Spy on prototype methods AFTER import time
|
|
26
|
+
startSpy = vi.spyOn(ChromeBrowser.prototype, 'start').mockResolvedValue();
|
|
27
|
+
closeSpy = vi.spyOn(ChromeBrowser.prototype, 'close').mockResolvedValue();
|
|
28
|
+
vi.clearAllMocks();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
startSpy.mockRestore();
|
|
33
|
+
closeSpy.mockRestore();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('runs all goals and returns missionResults (all passed)', async () => {
|
|
37
|
+
turnLoop
|
|
38
|
+
.mockResolvedValueOnce({ success: true })
|
|
39
|
+
.mockResolvedValueOnce({ success: true });
|
|
40
|
+
|
|
41
|
+
const goals = [
|
|
42
|
+
{ goal: 'Do A', submissionType: 'mission', submissionName: 'A' },
|
|
43
|
+
{ goal: 'Do B', submissionType: 'mission', submissionName: 'B' },
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
const res = await runAgent(goals, 'My Mission', 10);
|
|
47
|
+
|
|
48
|
+
// Browser lifecycle
|
|
49
|
+
expect(startSpy).toHaveBeenCalledTimes(1);
|
|
50
|
+
expect(closeSpy).toHaveBeenCalledTimes(1);
|
|
51
|
+
|
|
52
|
+
// turnLoop called once per goal and receives maxTurns
|
|
53
|
+
expect(turnLoop).toHaveBeenCalledTimes(2);
|
|
54
|
+
expect(turnLoop.mock.calls[0][2]).toBe(10);
|
|
55
|
+
expect(turnLoop.mock.calls[1][2]).toBe(10);
|
|
56
|
+
|
|
57
|
+
expect(res).toHaveLength(2);
|
|
58
|
+
expect(res[0].status).toBe('passed');
|
|
59
|
+
expect(res[1].status).toBe('passed');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('stops early on first failure and returns partial results', async () => {
|
|
63
|
+
turnLoop.mockResolvedValueOnce({ success: false });
|
|
64
|
+
|
|
65
|
+
const res = await runAgent(
|
|
66
|
+
[
|
|
67
|
+
{ goal: 'A', submissionType: 'mission', submissionName: 'A' },
|
|
68
|
+
{ goal: 'B', submissionType: 'mission', submissionName: 'B' },
|
|
69
|
+
],
|
|
70
|
+
'Fail Mission',
|
|
71
|
+
7
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
expect(turnLoop).toHaveBeenCalledTimes(1);
|
|
75
|
+
expect(res).toHaveLength(1);
|
|
76
|
+
expect(res[0].status).toBe('failed');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('coerces non-string goal to string for the user message', async () => {
|
|
80
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
81
|
+
|
|
82
|
+
await runAgent([{ goal: { toString: () => 'coerced text' } }], 'Coerce', 5);
|
|
83
|
+
|
|
84
|
+
const messages = turnLoop.mock.calls[0][1];
|
|
85
|
+
expect(messages[1]).toEqual({ role: 'user', content: 'coerced text' });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('deep-clones steps so external mutations do not leak', async () => {
|
|
89
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
90
|
+
ctx.steps.push({ result: 'SUCCESS: step1' });
|
|
91
|
+
return { success: true };
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const res1 = await runAgent([{ goal: 'Check clone' }], 'Clone Test', 3);
|
|
95
|
+
res1[0].steps[0].result = 'TAMPERED';
|
|
96
|
+
|
|
97
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
98
|
+
ctx.steps.push({ result: 'SUCCESS: step2' });
|
|
99
|
+
return { success: true };
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const res2 = await runAgent([{ goal: 'Check clone' }], 'Clone Test 2', 3);
|
|
103
|
+
expect(res1[0].steps[0].result).toBe('TAMPERED');
|
|
104
|
+
expect(res2[0].steps[0].result).toBe('SUCCESS: step2');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('includes missionName and submission metadata in results', async () => {
|
|
108
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
109
|
+
|
|
110
|
+
const res = await runAgent(
|
|
111
|
+
[
|
|
112
|
+
{
|
|
113
|
+
goal: 'Meta',
|
|
114
|
+
submissionType: 'mission',
|
|
115
|
+
submissionName: 'MetaName',
|
|
116
|
+
label: 'legacy-label',
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
'MetaMission',
|
|
120
|
+
4
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
expect(res[0].missionName).toBe('MetaMission');
|
|
124
|
+
expect(res[0].submissionType).toBe('mission');
|
|
125
|
+
expect(res[0].submissionName).toBe('MetaName');
|
|
126
|
+
});
|
|
127
|
+
});
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
vi.mock('node:fs/promises', () => ({
|
|
4
|
+
readFile: vi.fn(),
|
|
5
|
+
}));
|
|
6
|
+
|
|
7
|
+
import { readFile } from 'node:fs/promises';
|
|
8
|
+
import {
|
|
9
|
+
loadConfig,
|
|
10
|
+
getMaxTurns,
|
|
11
|
+
resolveTurnLimits,
|
|
12
|
+
enforceTurnBudget,
|
|
13
|
+
} from '../../core/config.js';
|
|
14
|
+
|
|
15
|
+
describe('core/config', () => {
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
vi.resetAllMocks();
|
|
18
|
+
delete process.env.STRICT_LIMITS;
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe('loadConfig', () => {
|
|
22
|
+
it('parses JSON when present', async () => {
|
|
23
|
+
readFile.mockResolvedValueOnce(JSON.stringify({ maxTurns: 33 }));
|
|
24
|
+
const cfg = await loadConfig('/any');
|
|
25
|
+
expect(cfg).toEqual({ maxTurns: 33 });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('returns {} on ENOENT', async () => {
|
|
29
|
+
readFile.mockRejectedValueOnce(new Error('ENOENT'));
|
|
30
|
+
const cfg = await loadConfig('/missing');
|
|
31
|
+
expect(cfg).toEqual({});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('returns {} on invalid JSON', async () => {
|
|
35
|
+
readFile.mockResolvedValueOnce('{ nope ');
|
|
36
|
+
const cfg = await loadConfig('/bad');
|
|
37
|
+
expect(cfg).toEqual({});
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
describe('getMaxTurns', () => {
|
|
42
|
+
it('uses fallback when invalid or missing', () => {
|
|
43
|
+
expect(getMaxTurns({}, 20)).toBe(20);
|
|
44
|
+
expect(getMaxTurns({ maxTurns: 'x' }, 20)).toBe(20);
|
|
45
|
+
expect(getMaxTurns({ maxTurns: 0 }, 20)).toBe(20);
|
|
46
|
+
expect(getMaxTurns({ maxTurns: -5 }, 20)).toBe(20);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('returns numeric when valid', () => {
|
|
50
|
+
expect(getMaxTurns({ maxTurns: 7 }, 20)).toBe(7);
|
|
51
|
+
expect(getMaxTurns({ maxTurns: '15' }, 20)).toBe(15);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
describe('resolveTurnLimits', () => {
|
|
56
|
+
it('returns expected defaults', () => {
|
|
57
|
+
const l = resolveTurnLimits();
|
|
58
|
+
expect(l.softMaxTurns).toBe(50);
|
|
59
|
+
expect(l.hardMaxTurns).toBe(200);
|
|
60
|
+
expect(l.hardMinTurns).toBe(5);
|
|
61
|
+
expect(l.maxIdleTurns).toBe(6);
|
|
62
|
+
expect(l.maxErrors).toBe(5);
|
|
63
|
+
expect(l.maxSeconds).toBe(600);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('enforceTurnBudget (lenient)', () => {
|
|
68
|
+
it('clamps > hardMaxTurns', () => {
|
|
69
|
+
const cfg = { maxTurns: 1000 };
|
|
70
|
+
const { effectiveMax, limits, notes, strict } = enforceTurnBudget(cfg);
|
|
71
|
+
expect(strict).toBe(false);
|
|
72
|
+
expect(effectiveMax).toBe(limits.hardMaxTurns);
|
|
73
|
+
expect(notes.join('\n')).toMatch(/Clamping to/i);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('warns when > softMaxTurns but <= hardMaxTurns', () => {
|
|
77
|
+
const cfg = { maxTurns: 120 };
|
|
78
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
79
|
+
expect(effectiveMax).toBe(120);
|
|
80
|
+
expect(notes.join('\n')).toMatch(/exceeds softMaxTurns/i);
|
|
81
|
+
expect(effectiveMax).toBeLessThanOrEqual(limits.hardMaxTurns);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('raises to hardMinTurns when below minimum', () => {
|
|
85
|
+
const cfg = { maxTurns: 1 };
|
|
86
|
+
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
87
|
+
expect(effectiveMax).toBe(limits.hardMinTurns);
|
|
88
|
+
expect(notes.join('\n')).toMatch(/Raising to/i);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('normalizes soft>hard ordering with a note', () => {
|
|
92
|
+
const weird = {
|
|
93
|
+
softMaxTurns: 300,
|
|
94
|
+
hardMaxTurns: 200,
|
|
95
|
+
hardMinTurns: 5,
|
|
96
|
+
maxIdleTurns: 6,
|
|
97
|
+
maxErrors: 5,
|
|
98
|
+
maxSeconds: 600,
|
|
99
|
+
};
|
|
100
|
+
const { limits, notes } = enforceTurnBudget({ maxTurns: 20 }, weird);
|
|
101
|
+
expect(limits.softMaxTurns).toBe(limits.hardMaxTurns);
|
|
102
|
+
expect(notes.join('\n')).toMatch(/Adjusted softMaxTurns down to hardMaxTurns/);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe('enforceTurnBudget (strict)', () => {
|
|
107
|
+
it('throws when > hardMaxTurns', () => {
|
|
108
|
+
const cfg = { maxTurns: 999, strictLimits: true };
|
|
109
|
+
expect(() => enforceTurnBudget(cfg)).toThrow(/exceeds hardMaxTurns/i);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it('throws when below hardMinTurns', () => {
|
|
113
|
+
process.env.STRICT_LIMITS = '1';
|
|
114
|
+
const cfg = { maxTurns: 2 };
|
|
115
|
+
expect(() => enforceTurnBudget(cfg)).toThrow(/below hardMinTurns/i);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it('throws when softMaxTurns > hardMaxTurns', () => {
|
|
119
|
+
const bad = {
|
|
120
|
+
softMaxTurns: 300,
|
|
121
|
+
hardMaxTurns: 200,
|
|
122
|
+
hardMinTurns: 5,
|
|
123
|
+
maxIdleTurns: 6,
|
|
124
|
+
maxErrors: 5,
|
|
125
|
+
maxSeconds: 600,
|
|
126
|
+
};
|
|
127
|
+
expect(() => enforceTurnBudget({ strictLimits: true, maxTurns: 20 }, bad))
|
|
128
|
+
.toThrow(/softMaxTurns .* > hardMaxTurns/);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
describe('enforceTurnBudget env override', () => {
|
|
134
|
+
const old = process.env.TESTRONAUT_TURNS;
|
|
135
|
+
|
|
136
|
+
beforeEach(() => { delete process.env.TESTRONAUT_TURNS; });
|
|
137
|
+
afterEach(() => { if (old === undefined) delete process.env.TESTRONAUT_TURNS; else process.env.TESTRONAUT_TURNS = old; });
|
|
138
|
+
|
|
139
|
+
it('uses TESTRONAUT_TURNS over cfg.maxTurns', () => {
|
|
140
|
+
process.env.TESTRONAUT_TURNS = '30';
|
|
141
|
+
const { effectiveMax, notes } = enforceTurnBudget({ maxTurns: 5 });
|
|
142
|
+
expect(effectiveMax).toBe(30);
|
|
143
|
+
expect(notes.join('\n')).toMatch(/CLI turn override/i);
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
});
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
// Mocks
|
|
4
|
+
vi.mock('../../core/agent.js', () => ({ runAgent: vi.fn() }));
|
|
5
|
+
vi.mock('../../core/redaction.js', () => ({
|
|
6
|
+
// simple deterministic redaction stub
|
|
7
|
+
redactPasswordInText: (s) => String(s).replace(/password:\s*\S+/gi, 'password: ********'),
|
|
8
|
+
}));
|
|
9
|
+
vi.mock('../../core/config.js', () => ({
|
|
10
|
+
loadConfig: vi.fn(),
|
|
11
|
+
enforceTurnBudget: vi.fn(),
|
|
12
|
+
}));
|
|
13
|
+
|
|
14
|
+
import { runAgent } from '../../core/agent.js';
|
|
15
|
+
import { loadConfig, enforceTurnBudget } from '../../core/config.js';
|
|
16
|
+
|
|
17
|
+
// Adjust the import path if your file lives elsewhere
|
|
18
|
+
import { runMissions } from '../../runner/testronaut.js';
|
|
19
|
+
|
|
20
|
+
describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
vi.clearAllMocks();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('passes effectiveMax to runAgent and logs any notes', async () => {
|
|
26
|
+
loadConfig.mockResolvedValue({ maxTurns: 999 });
|
|
27
|
+
enforceTurnBudget.mockReturnValue({
|
|
28
|
+
effectiveMax: 200,
|
|
29
|
+
limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
|
|
30
|
+
notes: ['⚠️ maxTurns (999) exceeds hardMaxTurns (200). Clamping to 200.'],
|
|
31
|
+
strict: false,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
35
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
36
|
+
|
|
37
|
+
runAgent.mockResolvedValue([
|
|
38
|
+
{ steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
await runMissions({ mission: 'Do xyz' }, 'Budgeted Run');
|
|
42
|
+
|
|
43
|
+
expect(loadConfig).toHaveBeenCalledTimes(1);
|
|
44
|
+
expect(enforceTurnBudget).toHaveBeenCalledWith({ maxTurns: 999 });
|
|
45
|
+
expect(warn).toHaveBeenCalledWith(
|
|
46
|
+
expect.stringContaining('Clamping to 200')
|
|
47
|
+
);
|
|
48
|
+
expect(runAgent).toHaveBeenCalledWith(
|
|
49
|
+
expect.any(Array), 'Budgeted Run', 200
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
warn.mockRestore();
|
|
53
|
+
log.mockRestore();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('does not warn when no notes are returned', async () => {
|
|
57
|
+
loadConfig.mockResolvedValue({ maxTurns: 20 });
|
|
58
|
+
enforceTurnBudget.mockReturnValue({
|
|
59
|
+
effectiveMax: 20,
|
|
60
|
+
limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
|
|
61
|
+
notes: [],
|
|
62
|
+
strict: false,
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
66
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
67
|
+
|
|
68
|
+
runAgent.mockResolvedValue([
|
|
69
|
+
{ steps: [{ result: 'SUCCESS: ok' }], status: 'passed' },
|
|
70
|
+
]);
|
|
71
|
+
|
|
72
|
+
await runMissions({ mission: 'No warnings' }, 'Clean');
|
|
73
|
+
|
|
74
|
+
expect(warn).not.toHaveBeenCalled();
|
|
75
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20);
|
|
76
|
+
|
|
77
|
+
warn.mockRestore();
|
|
78
|
+
log.mockRestore();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('marks first mission failed when last step result includes "failure"', async () => {
|
|
82
|
+
loadConfig.mockResolvedValue({});
|
|
83
|
+
enforceTurnBudget.mockReturnValue({
|
|
84
|
+
effectiveMax: 20,
|
|
85
|
+
limits: {},
|
|
86
|
+
notes: [],
|
|
87
|
+
strict: false,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
91
|
+
runAgent.mockResolvedValue([
|
|
92
|
+
{ steps: [{ result: 'FAILURE: could not do it' }], status: 'passed' },
|
|
93
|
+
]);
|
|
94
|
+
|
|
95
|
+
const res = await runMissions({ mission: 'Will fail' }, 'Status');
|
|
96
|
+
expect(res[0].status).toBe('failed');
|
|
97
|
+
|
|
98
|
+
log.mockRestore();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('normalizes single strings and functions into arrays and names missions', async () => {
|
|
102
|
+
loadConfig.mockResolvedValue({});
|
|
103
|
+
enforceTurnBudget.mockReturnValue({
|
|
104
|
+
effectiveMax: 15,
|
|
105
|
+
limits: {},
|
|
106
|
+
notes: [],
|
|
107
|
+
strict: false,
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
111
|
+
runAgent.mockResolvedValue([
|
|
112
|
+
{ steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
function preTask() {}
|
|
116
|
+
function postTask() {}
|
|
117
|
+
|
|
118
|
+
await runMissions(
|
|
119
|
+
{
|
|
120
|
+
preMission: preTask,
|
|
121
|
+
mission: 'Main mission text\nwith details',
|
|
122
|
+
postMission: postTask,
|
|
123
|
+
},
|
|
124
|
+
'My Mission'
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
// Check that runAgent got 3 goals (pre/main/post)
|
|
128
|
+
const call = (runAgent.mock.calls[0] || []);
|
|
129
|
+
const goals = call[0];
|
|
130
|
+
expect(Array.isArray(goals)).toBe(true);
|
|
131
|
+
expect(goals).toHaveLength(3);
|
|
132
|
+
|
|
133
|
+
// Submission types and names are set
|
|
134
|
+
expect(goals[0].submissionType).toBe('premission');
|
|
135
|
+
expect(goals[1].submissionType).toBe('mission');
|
|
136
|
+
expect(goals[2].submissionType).toBe('postmission');
|
|
137
|
+
|
|
138
|
+
// Mission name is the passed-in missionName for main
|
|
139
|
+
expect(goals[1].submissionName).toMatch(/^My Mission/);
|
|
140
|
+
|
|
141
|
+
// Effective max turns passed through
|
|
142
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15);
|
|
143
|
+
|
|
144
|
+
log.mockRestore();
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it('aborts early when runAgent returns falsy (defensive)', async () => {
|
|
148
|
+
loadConfig.mockResolvedValue({});
|
|
149
|
+
enforceTurnBudget.mockReturnValue({
|
|
150
|
+
effectiveMax: 10,
|
|
151
|
+
limits: {},
|
|
152
|
+
notes: [],
|
|
153
|
+
strict: false,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
157
|
+
runAgent.mockResolvedValue(null); // simulate unexpected falsy
|
|
158
|
+
|
|
159
|
+
const res = await runMissions({ mission: 'Edge case' }, 'Abort');
|
|
160
|
+
expect(res).toBeUndefined();
|
|
161
|
+
|
|
162
|
+
log.mockRestore();
|
|
163
|
+
});
|
|
164
|
+
});
|