testronaut 1.2.2 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/cli.js CHANGED
@@ -1,4 +1,37 @@
1
1
  #!/usr/bin/env node
2
+
3
+ /**
4
+ * cli.js
5
+ * -------
6
+ * Purpose:
7
+ * Single entry point for the Testronaut CLI:
8
+ * - Parses flags (e.g., --model, --turns, --init, help).
9
+ * - Runs mission files (single or all), aggregates results, and writes HTML/JSON reports.
10
+ * - Provides subcommands: login, upload (report + screenshots), serve/view (static file server).
11
+ *
12
+ * Key flags:
13
+ * --model <id> / --model=<id> → sets TESTRONAUT_MODEL env (wins over config file)
14
+ * --turns <n> / --turns=<n> → sets TESTRONAUT_TURNS env (wins over config file)
15
+ * --init → scaffolds project + optional Playwright browsers
16
+ * --help → prints help
17
+ *
18
+ * Notable helpers (defined below):
19
+ * parseJsonSafe(res, label) → tolerant JSON parse with good error messages
20
+ * pkgManagerForCwd(cwd) → detects npm/pnpm/yarn/bun
21
+ * hasAny(modNames) → “do we have at least one of these deps?” (ESM-safe)
22
+ * installDev(pm, pkg) → dev-install a package with the detected package manager
23
+ * ensurePlaywrightInstalled() → install @playwright/test and browsers (skippable in CI)
24
+ * guessMimeType(p) → static server content types
25
+ * safeJoin(root, relUrlPath) → path traversal protection for static server
26
+ * findLatestReportPair(reportDir) → find latest run_<ts>.html (+ matching .json if present)
27
+ * serveLatestReport() → read-only file server for most recent HTML report
28
+ *
29
+ * Test strategy:
30
+ * To test pure helpers without running the whole CLI, we export a tiny test bundle:
31
+ * export const __test__ = { guessMimeType, safeJoin, findLatestReportPair, pkgManagerForCwd }
32
+ * See tests in tests/cliTests/cli.helpers.test.js
33
+ */
34
+
2
35
  import fs from 'fs';
3
36
  import path from 'path';
4
37
  import { fileURLToPath } from 'url';
@@ -46,6 +79,29 @@ if (modelFlagIndex >= 0) {
46
79
  args.splice(modelFlagIndex, modelOverride ? 2 : 1);
47
80
  }
48
81
 
82
+ // Look for --turns=<n> or --turns <n>
83
+ let turnsOverride;
84
+ const turnsFlagIndex = args.findIndex(a => a === '--turns' || a.startsWith('--turns='));
85
+ if (turnsFlagIndex >= 0) {
86
+ if (args[turnsFlagIndex].includes('=')) {
87
+ turnsOverride = args[turnsFlagIndex].split('=')[1];
88
+ } else if (args[turnsFlagIndex + 1]) {
89
+ turnsOverride = args[turnsFlagIndex + 1];
90
+ }
91
+
92
+ if (turnsOverride) {
93
+ const n = Number(turnsOverride.trim());
94
+ if (Number.isFinite(n) && n > 0) {
95
+ process.env.TESTRONAUT_TURNS = String(n);
96
+ console.log(`🎯 Turn override: ${process.env.TESTRONAUT_TURNS}`);
97
+ } else {
98
+ console.warn(`⚠️ Invalid --turns value "${turnsOverride}". Ignoring.`);
99
+ }
100
+ }
101
+
102
+ // Remove flag & value so they aren’t treated as filenames
103
+ args.splice(turnsFlagIndex, turnsOverride ? 2 : 1);
104
+ }
49
105
 
50
106
  const missionsDir = path.resolve(process.cwd(), 'missions');
51
107
 
@@ -66,6 +122,7 @@ Usage:
66
122
 
67
123
  Options:
68
124
  --init Scaffold project folders and a welcome mission
125
+ --turns=<n> Override max turns for this run (e.g., --turns=30)
69
126
  --help Show this help message
70
127
 
71
128
  Examples:
@@ -228,7 +285,13 @@ fs.mkdirSync(outputDir, { recursive: true });
228
285
  fs.writeFileSync(`${outputDir}/${runId}.json`, JSON.stringify(report, null, 2));
229
286
  generateHtmlReport(report, `${outputDir}/${runId}.html`);
230
287
 
231
- // helpers near top of cli.js
288
+ /**
289
+ * Parse a fetch Response safely and return JSON or throw a rich error.
290
+ * @param {Response} res
291
+ * @param {string} urlLabel - human-friendly label for error messages
292
+ * @returns {Promise<any>}
293
+ * @throws when HTTP status is not ok or body is empty/invalid
294
+ */
232
295
  async function parseJsonSafe(res, urlLabel) {
233
296
  const text = await res.text(); // read once
234
297
  let data = null;
@@ -248,6 +311,11 @@ async function parseJsonSafe(res, urlLabel) {
248
311
  return data;
249
312
  }
250
313
 
314
+ /**
315
+ * Detect the local package manager based on lockfiles.
316
+ * @param {string} [cwd=process.cwd()]
317
+ * @returns {'pnpm'|'yarn'|'bun'|'npm'}
318
+ */
251
319
  function pkgManagerForCwd(cwd = process.cwd()) {
252
320
  const has = (f) => fs.existsSync(path.join(cwd, f));
253
321
  if (has('pnpm-lock.yaml')) return 'pnpm';
@@ -559,7 +627,11 @@ async function uploadReport() {
559
627
  }
560
628
  }
561
629
 
562
-
630
+ /**
631
+ * Guess a reasonable MIME type for static serving.
632
+ * @param {string} p - file path
633
+ * @returns {string}
634
+ */
563
635
  function guessMimeType(p) {
564
636
  const ext = path.extname(p).toLowerCase();
565
637
  if (ext === '.html' || ext === '.htm') return 'text/html; charset=utf-8';
@@ -574,6 +646,12 @@ function guessMimeType(p) {
574
646
  return 'application/octet-stream';
575
647
  }
576
648
 
649
+ /**
650
+ * Path-join a request path to a root directory with traversal protection.
651
+ * @param {string} root - filesystem directory
652
+ * @param {string} relUrlPath - URL path from incoming request
653
+ * @returns {string|null} resolved path or null on attempted escape
654
+ */
577
655
  function safeJoin(root, relUrlPath) {
578
656
  // Prevent path traversal and keep within root
579
657
  const decoded = decodeURIComponent(relUrlPath);
@@ -593,6 +671,12 @@ async function openInBrowser(targetUrl) {
593
671
  return exec(`xdg-open ${quoted}`).catch(() => {}); // best-effort on Linux
594
672
  }
595
673
 
674
+ /**
675
+ * Find the latest run HTML (and its matching JSON, if present).
676
+ * Expects files like "run_<timestamp>.html" in the report directory.
677
+ * @param {string} reportDir
678
+ * @returns {{htmlFile:string, jsonFile:string|null}|null}
679
+ */
596
680
  function findLatestReportPair(reportDir) {
597
681
  if (!fs.existsSync(reportDir)) return null;
598
682
  const files = fs.readdirSync(reportDir).filter(f => f.endsWith('.html'));
package/core/agent.js CHANGED
@@ -1,18 +1,56 @@
1
+ /**
2
+ * agent.js
3
+ * --------
4
+ * Purpose:
5
+ * Orchestrate a browser-backed autonomous agent over a sequence of goals.
6
+ *
7
+ * Responsibilities:
8
+ * - Start/stop the browser lifecycle.
9
+ * - For each goal, construct initial messages and invoke turnLoop().
10
+ * - Collect per-goal traces into a normalized missionResults array.
11
+ *
12
+ * Message contract (initial):
13
+ * messages = [
14
+ * { role: 'system', content: 'instructions...' },
15
+ * { role: 'user', content: <goal text> }
16
+ * ]
17
+ *
18
+ * Related tests:
19
+ * tests/agentTests/agent.test.js
20
+ *
21
+ * Used by:
22
+ * - cli/testronaut.js (runMissions)
23
+ */
24
+
1
25
  import 'dotenv/config';
2
26
  import { ChromeBrowser } from '../tools/chromeBrowser.js';
3
27
  import { turnLoop } from './turnLoop.js';
4
28
 
5
- export async function runAgent(goals, missionName, maxTurns = 20, ) {
29
+ /**
30
+ * Execute goals with a browser agent.
31
+ *
32
+ * @param {Array<{goal:any, submissionType?:string, submissionName?:string, label?:string}>} goals
33
+ * @param {string} missionName
34
+ * @param {number} [maxTurns=20] - upper bound for turnLoop per goal
35
+ * @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'failed', steps:any[], startTime:number, endTime:number}>>}
36
+ */
37
+ export async function runAgent(goals, missionName, maxTurns = 20) {
6
38
  const browser = new ChromeBrowser();
7
39
  await browser.start();
8
- const startTime = Date.now();
9
40
  let result;
10
41
 
11
42
  try {
12
43
  const missionResults = [];
44
+
13
45
  for (const goal of goals) {
14
- // console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
15
46
  const steps = [];
47
+
48
+ // Ensure user message is a string (functions/objects → toString fallback)
49
+ const userContent =
50
+ typeof goal.goal === 'string'
51
+ ? goal.goal
52
+ : goal?.goal?.toString?.() ?? String(goal.goal);
53
+
16
54
  const messages = [
17
55
  {
18
56
  role: 'system',
@@ -29,61 +67,39 @@ export async function runAgent(goals, missionName, maxTurns = 20, ) {
29
67
  After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
30
68
  `.trim(),
31
69
  },
32
- { role: 'user', content: goal.goal },
70
+ { role: 'user', content: userContent },
33
71
  ];
34
72
 
73
+ // Run the turn loop for this goal
35
74
  result = await turnLoop(
36
- browser,
37
- messages,
38
- maxTurns,
39
- 0, /* currentTurn */
40
- 0, /* retryCount */
41
- null, /* currentStep */
42
- {
43
- steps, // 👈 pass in
44
- missionName, // 👈 pass in (for tagging)
45
- }
75
+ browser,
76
+ messages,
77
+ maxTurns,
78
+ 0, // currentTurn
79
+ 0, // retryCount
80
+ null, // currentStep
81
+ { steps, missionName }
46
82
  );
47
-
83
+
84
+ // Snapshot steps (avoid retaining references to mutable arrays)
48
85
  missionResults.push({
49
86
  missionName,
50
87
  submissionType: goal.submissionType || goal.label,
51
88
  submissionName: goal.submissionName || null,
52
89
  status: result?.success ? 'passed' : 'failed',
53
- steps: JSON.parse(JSON.stringify(steps)), // 👈 deep-clone to avoid refs
54
- startTime: Date.now() - 1, // set real values if you have them
90
+ steps: JSON.parse(JSON.stringify(steps)), // deep-clone
91
+ startTime: Date.now() - 1, // TODO: wire actual timings if needed
55
92
  endTime: Date.now(),
56
93
  });
57
94
 
58
- if (!result.success) {
95
+ if (!result?.success) {
59
96
  console.log('🛑 Agent stopped due to failed goal.\n');
60
97
  return missionResults;
61
98
  }
62
-
63
- // if (!result.success) {
64
- // console.log('🛑 Agent stopped due to failed goal.\n');
65
- // const endTime = Date.now();
66
- // return {
67
- // missionName: missionName,
68
- // status: 'failed',
69
- // steps: result.steps,
70
- // startTime: startTime,
71
- // endTime: endTime,
72
- // };
73
- // }
74
99
  }
75
100
 
76
101
  console.log('✅ All goals completed successfully.\n');
77
- // const endTime = Date.now();
78
102
  return missionResults;
79
- // return {
80
- // missionName: missionName,
81
- // status: 'passed',
82
- // steps: result.steps,
83
- // startTime: startTime,
84
- // endTime: endTime,
85
- // };
86
-
87
103
  } finally {
88
104
  await browser.close();
89
105
  }
package/core/config.js ADDED
@@ -0,0 +1,146 @@
1
+ /**
2
+ * config.js
3
+ * ----------
4
+ * Purpose:
5
+ * Centralized config loading and turn-budget guardrails for Testronaut.
6
+ *
7
+ * Responsibilities:
8
+ * - Load `testronaut-config.json` (best effort).
9
+ * - Read specific knobs with safe fallbacks (e.g., maxTurns).
10
+ * - Define baseline limits (soft/hard/min/idle/errors/time).
11
+ * - Enforce user requests via clamp + warn (default) or fail-fast (strict).
12
+ *
13
+ * Related tests:
14
+ * tests/configTests/config.test.js
15
+ *
16
+ * Used by:
17
+ * - cli/testronaut.js (to derive effectiveMax turns per run)
18
+ * - core/agent.js / core/turnLoop.js (runtime stop conditions)
19
+ */
20
+
21
+ // core/config.js
22
+ import { readFile } from 'node:fs/promises';
23
+ import path from 'node:path';
24
+
25
+ /**
26
+ * Load testronaut-config.json from the given cwd.
27
+ * Returns {} when missing or invalid JSON.
28
+ *
29
+ * @param {string} [cwd=process.cwd()]
30
+ * @returns {Promise<object>}
31
+ */
32
+ export async function loadConfig(cwd = process.cwd()) {
33
+ try {
34
+ const raw = await readFile(path.join(cwd, 'testronaut-config.json'), 'utf8');
35
+ return JSON.parse(raw);
36
+ } catch {
37
+ // No config or invalid JSON — fall back to defaults at call sites
38
+ return {};
39
+ }
40
+ }
41
+
42
+ /**
43
+ * Resolve the requested maxTurns from config with a safe fallback.
44
+ *
45
+ * @param {object} cfg
46
+ * @param {number} [fallback=20]
47
+ * @returns {number}
48
+ */
49
+ export function getMaxTurns(cfg, fallback = 20) {
50
+ const n = Number(cfg?.maxTurns);
51
+ if (!Number.isFinite(n) || n <= 0) return fallback;
52
+ return n;
53
+ }
54
+
55
+ /**
56
+ * Baseline limits for mission runs. Adjust here for global defaults.
57
+ *
58
+ * @returns {{
59
+ * softMaxTurns:number,
60
+ * hardMaxTurns:number,
61
+ * hardMinTurns:number,
62
+ * maxIdleTurns:number,
63
+ * maxErrors:number,
64
+ * maxSeconds:number
65
+ * }}
66
+ */
67
+ export function resolveTurnLimits() {
68
+ const softMaxTurns = 50;
69
+ const hardMaxTurns = 200;
70
+ const hardMinTurns = 5;
71
+ const maxIdleTurns = 6;
72
+ const maxErrors = 5;
73
+ const maxSeconds = 600; //10 minutes
74
+
75
+ return {
76
+ softMaxTurns,
77
+ hardMaxTurns,
78
+ hardMinTurns,
79
+ maxIdleTurns,
80
+ maxErrors,
81
+ maxSeconds,
82
+ };
83
+ }
84
+
85
+ /**
86
+ * Compute an effective turn budget and normalized limits from config.
87
+ * - Default: clamp + warn (lenient).
88
+ * - Strict mode (cfg.strictLimits or env STRICT_LIMITS): throw on violations.
89
+ *
90
+ * @param {object} cfg
91
+ * @param {ReturnType<typeof resolveTurnLimits>} [baseLimits=resolveTurnLimits()]
92
+ * @returns {{
93
+ * effectiveMax:number,
94
+ * limits: ReturnType<typeof resolveTurnLimits>,
95
+ * notes:string[],
96
+ * strict:boolean
97
+ * }}
98
+ */
99
+ export function enforceTurnBudget(cfg, baseLimits = resolveTurnLimits()) {
100
+ const strict = !!(cfg?.strictLimits || process.env.STRICT_LIMITS);
101
+
102
+ const limits = { ...baseLimits };
103
+ const notes = [];
104
+
105
+ // Normalize soft/hard ordering
106
+ if (limits.softMaxTurns > limits.hardMaxTurns) {
107
+ if (strict) throw new Error(`[limits] softMaxTurns (${limits.softMaxTurns}) > hardMaxTurns (${limits.hardMaxTurns}).`);
108
+ notes.push(`Adjusted softMaxTurns down to hardMaxTurns (${limits.hardMaxTurns}).`);
109
+ limits.softMaxTurns = limits.hardMaxTurns;
110
+ }
111
+ if (limits.hardMinTurns > limits.softMaxTurns) {
112
+ if (strict) throw new Error(`[limits] hardMinTurns (${limits.hardMinTurns}) > softMaxTurns (${limits.softMaxTurns}).`);
113
+ notes.push(`Adjusted hardMinTurns down to softMaxTurns (${limits.softMaxTurns}).`);
114
+ limits.hardMinTurns = limits.softMaxTurns;
115
+ }
116
+
117
+ // Resolve requested maxTurns (env override wins over config)
118
+ const envTurns = Number(process.env.TESTRONAUT_TURNS);
119
+ const hasEnv = Number.isFinite(envTurns) && envTurns > 0;
120
+ const cfgTurns = Number(cfg?.maxTurns);
121
+ const hasCfg = Number.isFinite(cfgTurns) && cfgTurns > 0;
122
+ let requested = hasEnv ? envTurns : (hasCfg ? cfgTurns : 20);
123
+ let effectiveMax = requested;
124
+
125
+ if (hasEnv) {
126
+ notes.push(`ℹ️ Using CLI turn override (TESTRONAUT_TURNS=${envTurns}).`);
127
+ }
128
+
129
+ if (effectiveMax > limits.hardMaxTurns) {
130
+ const msg = `maxTurns (${effectiveMax}) exceeds hardMaxTurns (${limits.hardMaxTurns}).`;
131
+ if (strict) throw new Error(`[limits] ${msg}`);
132
+ notes.push(`⚠️ ${msg} Clamping to ${limits.hardMaxTurns}.`);
133
+ effectiveMax = limits.hardMaxTurns;
134
+ } else if (effectiveMax > limits.softMaxTurns) {
135
+ const msg = `maxTurns (${effectiveMax}) exceeds softMaxTurns (${limits.softMaxTurns}).`;
136
+ if (strict) throw new Error(`[limits] ${msg}`);
137
+ notes.push(`⚠️ ${msg} Proceeding, but consider lowering to control cost.`);
138
+ } else if (effectiveMax < limits.hardMinTurns) {
139
+ const msg = `maxTurns (${effectiveMax}) is below hardMinTurns (${limits.hardMinTurns}).`;
140
+ if (strict) throw new Error(`[limits] ${msg}`);
141
+ notes.push(`⚠️ ${msg} Raising to ${limits.hardMinTurns}.`);
142
+ effectiveMax = limits.hardMinTurns;
143
+ }
144
+
145
+ return { effectiveMax, limits, notes, strict };
146
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testronaut",
3
- "version": "1.2.2",
3
+ "version": "1.2.3",
4
4
  "main": "./runner/testronaut.js",
5
5
  "bin": {
6
6
  "testronaut": "./bin/cli.js"
@@ -1,27 +1,49 @@
1
- import { runAgent } from '../core/agent.js';
2
- import { redactPasswordInText } from '../core/redaction.js';
3
- // import { runSuite } from './suiteRunner.js';
4
-
5
- // const objectives = [];
6
-
7
- // export function mission(title, fn) {
8
- // console.log(`\n🚀 Mission: ${title}`);
9
- // fn();
10
- // }
11
-
12
- // export function objective(desc, workflow) {
13
- // objectives.push({ name: desc, workflow });
14
- // }
15
-
16
- // export async function launch(preMissionSetup) {
17
- // await runSuite(objectives, preMissionSetup);
18
- // }
19
-
1
+ /**
2
+ * testronaut.js
3
+ * -------------
4
+ * Purpose:
5
+ * High-level mission runner: normalize pre/main/post submissions,
6
+ * compute a safe turn budget from config, and invoke the agent.
7
+ *
8
+ * Responsibilities:
9
+ * - Read config once and derive turn limits via enforceTurnBudget().
10
+ * - Build a normalized list of submissions with human-friendly names.
11
+ * - Redact sensitive text when logging mission strings.
12
+ * - Call runAgent(goals, missionName, maxTurns) and post-process status.
13
+ *
14
+ * Message contract (goal → initial messages inside agent):
15
+ * - system: operational guidance + success/failure contract
16
+ * - user: the mission text (or coerced string)
17
+ *
18
+ * Related tests:
19
+ * tests/missionTests/testronaut.test.js
20
+ *
21
+ * Used by:
22
+ * - CLI entrypoint(s)
23
+ */
20
24
 
21
25
 
26
+ import { runAgent } from '../core/agent.js';
27
+ import { redactPasswordInText } from '../core/redaction.js';
28
+ import { loadConfig, enforceTurnBudget } from '../core/config.js';
29
+
30
+ /**
31
+ * Run a mission flow.
32
+ *
33
+ * @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
34
+ * @param {string} missionName
35
+ */
22
36
  export async function runMissions({ preMission, mission, postMission }, missionName) {
23
- const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
37
+ // 1) Config and turn budget (with guardrails)
38
+ const cfg = await loadConfig();
39
+ const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
40
+ const maxTurns = effectiveMax;
41
+ if (notes.length) {
42
+ console.warn(notes.join('\n'));
43
+ }
24
44
 
45
+ // 2) Normalize submissions
46
+ const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
25
47
  const pre = normalizeToArray(preMission);
26
48
  const main = normalizeToArray(mission);
27
49
  const post = normalizeToArray(postMission);
@@ -100,7 +122,8 @@ export async function runMissions({ preMission, mission, postMission }, missionN
100
122
  })
101
123
  );
102
124
 
103
- const success = await runAgent(goals, missionName);
125
+ // 3) Execute
126
+ const success = await runAgent(goals, missionName, maxTurns);
104
127
  if (!success) {
105
128
  console.log(`❌ Aborting after failed goal.`);
106
129
  return;
@@ -0,0 +1,127 @@
1
+ // tests/coreTests/agent.test.js
2
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
3
+
4
+ // 1) Provide a constructable mock without using `vi` inside the factory.
5
+ vi.mock('../../tools/chromeBrowser.js', () => {
6
+ function ChromeBrowser() {}
7
+ ChromeBrowser.prototype.start = async function () {};
8
+ ChromeBrowser.prototype.close = async function () {};
9
+ return { ChromeBrowser };
10
+ });
11
+
12
+ // Keep turnLoop simple; using vi is fine here (no factory-hoist issue)
13
+ vi.mock('../../core/turnLoop.js', () => ({
14
+ turnLoop: vi.fn(),
15
+ }));
16
+
17
+ import { ChromeBrowser } from '../../tools/chromeBrowser.js';
18
+ import { turnLoop } from '../../core/turnLoop.js';
19
+ import { runAgent } from '../../core/agent.js';
20
+
21
+ let startSpy, closeSpy;
22
+
23
+ describe('core/agent.runAgent', () => {
24
+ beforeEach(() => {
25
+ // 2) Spy on prototype methods AFTER import time
26
+ startSpy = vi.spyOn(ChromeBrowser.prototype, 'start').mockResolvedValue();
27
+ closeSpy = vi.spyOn(ChromeBrowser.prototype, 'close').mockResolvedValue();
28
+ vi.clearAllMocks();
29
+ });
30
+
31
+ afterEach(() => {
32
+ startSpy.mockRestore();
33
+ closeSpy.mockRestore();
34
+ });
35
+
36
+ it('runs all goals and returns missionResults (all passed)', async () => {
37
+ turnLoop
38
+ .mockResolvedValueOnce({ success: true })
39
+ .mockResolvedValueOnce({ success: true });
40
+
41
+ const goals = [
42
+ { goal: 'Do A', submissionType: 'mission', submissionName: 'A' },
43
+ { goal: 'Do B', submissionType: 'mission', submissionName: 'B' },
44
+ ];
45
+
46
+ const res = await runAgent(goals, 'My Mission', 10);
47
+
48
+ // Browser lifecycle
49
+ expect(startSpy).toHaveBeenCalledTimes(1);
50
+ expect(closeSpy).toHaveBeenCalledTimes(1);
51
+
52
+ // turnLoop called once per goal and receives maxTurns
53
+ expect(turnLoop).toHaveBeenCalledTimes(2);
54
+ expect(turnLoop.mock.calls[0][2]).toBe(10);
55
+ expect(turnLoop.mock.calls[1][2]).toBe(10);
56
+
57
+ expect(res).toHaveLength(2);
58
+ expect(res[0].status).toBe('passed');
59
+ expect(res[1].status).toBe('passed');
60
+ });
61
+
62
+ it('stops early on first failure and returns partial results', async () => {
63
+ turnLoop.mockResolvedValueOnce({ success: false });
64
+
65
+ const res = await runAgent(
66
+ [
67
+ { goal: 'A', submissionType: 'mission', submissionName: 'A' },
68
+ { goal: 'B', submissionType: 'mission', submissionName: 'B' },
69
+ ],
70
+ 'Fail Mission',
71
+ 7
72
+ );
73
+
74
+ expect(turnLoop).toHaveBeenCalledTimes(1);
75
+ expect(res).toHaveLength(1);
76
+ expect(res[0].status).toBe('failed');
77
+ });
78
+
79
+ it('coerces non-string goal to string for the user message', async () => {
80
+ turnLoop.mockResolvedValueOnce({ success: true });
81
+
82
+ await runAgent([{ goal: { toString: () => 'coerced text' } }], 'Coerce', 5);
83
+
84
+ const messages = turnLoop.mock.calls[0][1];
85
+ expect(messages[1]).toEqual({ role: 'user', content: 'coerced text' });
86
+ });
87
+
88
+ it('deep-clones steps so external mutations do not leak', async () => {
89
+ turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
90
+ ctx.steps.push({ result: 'SUCCESS: step1' });
91
+ return { success: true };
92
+ });
93
+
94
+ const res1 = await runAgent([{ goal: 'Check clone' }], 'Clone Test', 3);
95
+ res1[0].steps[0].result = 'TAMPERED';
96
+
97
+ turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
98
+ ctx.steps.push({ result: 'SUCCESS: step2' });
99
+ return { success: true };
100
+ });
101
+
102
+ const res2 = await runAgent([{ goal: 'Check clone' }], 'Clone Test 2', 3);
103
+ expect(res1[0].steps[0].result).toBe('TAMPERED');
104
+ expect(res2[0].steps[0].result).toBe('SUCCESS: step2');
105
+ });
106
+
107
+ it('includes missionName and submission metadata in results', async () => {
108
+ turnLoop.mockResolvedValueOnce({ success: true });
109
+
110
+ const res = await runAgent(
111
+ [
112
+ {
113
+ goal: 'Meta',
114
+ submissionType: 'mission',
115
+ submissionName: 'MetaName',
116
+ label: 'legacy-label',
117
+ },
118
+ ],
119
+ 'MetaMission',
120
+ 4
121
+ );
122
+
123
+ expect(res[0].missionName).toBe('MetaMission');
124
+ expect(res[0].submissionType).toBe('mission');
125
+ expect(res[0].submissionName).toBe('MetaName');
126
+ });
127
+ });
@@ -0,0 +1,146 @@
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
2
+
3
+ vi.mock('node:fs/promises', () => ({
4
+ readFile: vi.fn(),
5
+ }));
6
+
7
+ import { readFile } from 'node:fs/promises';
8
+ import {
9
+ loadConfig,
10
+ getMaxTurns,
11
+ resolveTurnLimits,
12
+ enforceTurnBudget,
13
+ } from '../../core/config.js';
14
+
15
+ describe('core/config', () => {
16
+ beforeEach(() => {
17
+ vi.resetAllMocks();
18
+ delete process.env.STRICT_LIMITS;
19
+ });
20
+
21
+ describe('loadConfig', () => {
22
+ it('parses JSON when present', async () => {
23
+ readFile.mockResolvedValueOnce(JSON.stringify({ maxTurns: 33 }));
24
+ const cfg = await loadConfig('/any');
25
+ expect(cfg).toEqual({ maxTurns: 33 });
26
+ });
27
+
28
+ it('returns {} on ENOENT', async () => {
29
+ readFile.mockRejectedValueOnce(new Error('ENOENT'));
30
+ const cfg = await loadConfig('/missing');
31
+ expect(cfg).toEqual({});
32
+ });
33
+
34
+ it('returns {} on invalid JSON', async () => {
35
+ readFile.mockResolvedValueOnce('{ nope ');
36
+ const cfg = await loadConfig('/bad');
37
+ expect(cfg).toEqual({});
38
+ });
39
+ });
40
+
41
+ describe('getMaxTurns', () => {
42
+ it('uses fallback when invalid or missing', () => {
43
+ expect(getMaxTurns({}, 20)).toBe(20);
44
+ expect(getMaxTurns({ maxTurns: 'x' }, 20)).toBe(20);
45
+ expect(getMaxTurns({ maxTurns: 0 }, 20)).toBe(20);
46
+ expect(getMaxTurns({ maxTurns: -5 }, 20)).toBe(20);
47
+ });
48
+
49
+ it('returns numeric when valid', () => {
50
+ expect(getMaxTurns({ maxTurns: 7 }, 20)).toBe(7);
51
+ expect(getMaxTurns({ maxTurns: '15' }, 20)).toBe(15);
52
+ });
53
+ });
54
+
55
+ describe('resolveTurnLimits', () => {
56
+ it('returns expected defaults', () => {
57
+ const l = resolveTurnLimits();
58
+ expect(l.softMaxTurns).toBe(50);
59
+ expect(l.hardMaxTurns).toBe(200);
60
+ expect(l.hardMinTurns).toBe(5);
61
+ expect(l.maxIdleTurns).toBe(6);
62
+ expect(l.maxErrors).toBe(5);
63
+ expect(l.maxSeconds).toBe(600);
64
+ });
65
+ });
66
+
67
+ describe('enforceTurnBudget (lenient)', () => {
68
+ it('clamps > hardMaxTurns', () => {
69
+ const cfg = { maxTurns: 1000 };
70
+ const { effectiveMax, limits, notes, strict } = enforceTurnBudget(cfg);
71
+ expect(strict).toBe(false);
72
+ expect(effectiveMax).toBe(limits.hardMaxTurns);
73
+ expect(notes.join('\n')).toMatch(/Clamping to/i);
74
+ });
75
+
76
+ it('warns when > softMaxTurns but <= hardMaxTurns', () => {
77
+ const cfg = { maxTurns: 120 };
78
+ const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
79
+ expect(effectiveMax).toBe(120);
80
+ expect(notes.join('\n')).toMatch(/exceeds softMaxTurns/i);
81
+ expect(effectiveMax).toBeLessThanOrEqual(limits.hardMaxTurns);
82
+ });
83
+
84
+ it('raises to hardMinTurns when below minimum', () => {
85
+ const cfg = { maxTurns: 1 };
86
+ const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
87
+ expect(effectiveMax).toBe(limits.hardMinTurns);
88
+ expect(notes.join('\n')).toMatch(/Raising to/i);
89
+ });
90
+
91
+ it('normalizes soft>hard ordering with a note', () => {
92
+ const weird = {
93
+ softMaxTurns: 300,
94
+ hardMaxTurns: 200,
95
+ hardMinTurns: 5,
96
+ maxIdleTurns: 6,
97
+ maxErrors: 5,
98
+ maxSeconds: 600,
99
+ };
100
+ const { limits, notes } = enforceTurnBudget({ maxTurns: 20 }, weird);
101
+ expect(limits.softMaxTurns).toBe(limits.hardMaxTurns);
102
+ expect(notes.join('\n')).toMatch(/Adjusted softMaxTurns down to hardMaxTurns/);
103
+ });
104
+ });
105
+
106
+ describe('enforceTurnBudget (strict)', () => {
107
+ it('throws when > hardMaxTurns', () => {
108
+ const cfg = { maxTurns: 999, strictLimits: true };
109
+ expect(() => enforceTurnBudget(cfg)).toThrow(/exceeds hardMaxTurns/i);
110
+ });
111
+
112
+ it('throws when below hardMinTurns', () => {
113
+ process.env.STRICT_LIMITS = '1';
114
+ const cfg = { maxTurns: 2 };
115
+ expect(() => enforceTurnBudget(cfg)).toThrow(/below hardMinTurns/i);
116
+ });
117
+
118
+ it('throws when softMaxTurns > hardMaxTurns', () => {
119
+ const bad = {
120
+ softMaxTurns: 300,
121
+ hardMaxTurns: 200,
122
+ hardMinTurns: 5,
123
+ maxIdleTurns: 6,
124
+ maxErrors: 5,
125
+ maxSeconds: 600,
126
+ };
127
+ expect(() => enforceTurnBudget({ strictLimits: true, maxTurns: 20 }, bad))
128
+ .toThrow(/softMaxTurns .* > hardMaxTurns/);
129
+ });
130
+ });
131
+
132
+
133
+ describe('enforceTurnBudget env override', () => {
134
+ const old = process.env.TESTRONAUT_TURNS;
135
+
136
+ beforeEach(() => { delete process.env.TESTRONAUT_TURNS; });
137
+ afterEach(() => { if (old === undefined) delete process.env.TESTRONAUT_TURNS; else process.env.TESTRONAUT_TURNS = old; });
138
+
139
+ it('uses TESTRONAUT_TURNS over cfg.maxTurns', () => {
140
+ process.env.TESTRONAUT_TURNS = '30';
141
+ const { effectiveMax, notes } = enforceTurnBudget({ maxTurns: 5 });
142
+ expect(effectiveMax).toBe(30);
143
+ expect(notes.join('\n')).toMatch(/CLI turn override/i);
144
+ });
145
+ });
146
+ });
@@ -0,0 +1,164 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+
3
+ // Mocks
4
+ vi.mock('../../core/agent.js', () => ({ runAgent: vi.fn() }));
5
+ vi.mock('../../core/redaction.js', () => ({
6
+ // simple deterministic redaction stub
7
+ redactPasswordInText: (s) => String(s).replace(/password:\s*\S+/gi, 'password: ********'),
8
+ }));
9
+ vi.mock('../../core/config.js', () => ({
10
+ loadConfig: vi.fn(),
11
+ enforceTurnBudget: vi.fn(),
12
+ }));
13
+
14
+ import { runAgent } from '../../core/agent.js';
15
+ import { loadConfig, enforceTurnBudget } from '../../core/config.js';
16
+
17
+ // Adjust the import path if your file lives elsewhere
18
+ import { runMissions } from '../../runner/testronaut.js';
19
+
20
+ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
21
+ beforeEach(() => {
22
+ vi.clearAllMocks();
23
+ });
24
+
25
+ it('passes effectiveMax to runAgent and logs any notes', async () => {
26
+ loadConfig.mockResolvedValue({ maxTurns: 999 });
27
+ enforceTurnBudget.mockReturnValue({
28
+ effectiveMax: 200,
29
+ limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
30
+ notes: ['⚠️ maxTurns (999) exceeds hardMaxTurns (200). Clamping to 200.'],
31
+ strict: false,
32
+ });
33
+
34
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
35
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});
36
+
37
+ runAgent.mockResolvedValue([
38
+ { steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
39
+ ]);
40
+
41
+ await runMissions({ mission: 'Do xyz' }, 'Budgeted Run');
42
+
43
+ expect(loadConfig).toHaveBeenCalledTimes(1);
44
+ expect(enforceTurnBudget).toHaveBeenCalledWith({ maxTurns: 999 });
45
+ expect(warn).toHaveBeenCalledWith(
46
+ expect.stringContaining('Clamping to 200')
47
+ );
48
+ expect(runAgent).toHaveBeenCalledWith(
49
+ expect.any(Array), 'Budgeted Run', 200
50
+ );
51
+
52
+ warn.mockRestore();
53
+ log.mockRestore();
54
+ });
55
+
56
+ it('does not warn when no notes are returned', async () => {
57
+ loadConfig.mockResolvedValue({ maxTurns: 20 });
58
+ enforceTurnBudget.mockReturnValue({
59
+ effectiveMax: 20,
60
+ limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
61
+ notes: [],
62
+ strict: false,
63
+ });
64
+
65
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
66
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});
67
+
68
+ runAgent.mockResolvedValue([
69
+ { steps: [{ result: 'SUCCESS: ok' }], status: 'passed' },
70
+ ]);
71
+
72
+ await runMissions({ mission: 'No warnings' }, 'Clean');
73
+
74
+ expect(warn).not.toHaveBeenCalled();
75
+ expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20);
76
+
77
+ warn.mockRestore();
78
+ log.mockRestore();
79
+ });
80
+
81
+ it('marks first mission failed when last step result includes "failure"', async () => {
82
+ loadConfig.mockResolvedValue({});
83
+ enforceTurnBudget.mockReturnValue({
84
+ effectiveMax: 20,
85
+ limits: {},
86
+ notes: [],
87
+ strict: false,
88
+ });
89
+
90
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});
91
+ runAgent.mockResolvedValue([
92
+ { steps: [{ result: 'FAILURE: could not do it' }], status: 'passed' },
93
+ ]);
94
+
95
+ const res = await runMissions({ mission: 'Will fail' }, 'Status');
96
+ expect(res[0].status).toBe('failed');
97
+
98
+ log.mockRestore();
99
+ });
100
+
101
+ it('normalizes single strings and functions into arrays and names missions', async () => {
102
+ loadConfig.mockResolvedValue({});
103
+ enforceTurnBudget.mockReturnValue({
104
+ effectiveMax: 15,
105
+ limits: {},
106
+ notes: [],
107
+ strict: false,
108
+ });
109
+
110
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});
111
+ runAgent.mockResolvedValue([
112
+ { steps: [{ result: 'SUCCESS: done' }], status: 'passed' },
113
+ ]);
114
+
115
+ function preTask() {}
116
+ function postTask() {}
117
+
118
+ await runMissions(
119
+ {
120
+ preMission: preTask,
121
+ mission: 'Main mission text\nwith details',
122
+ postMission: postTask,
123
+ },
124
+ 'My Mission'
125
+ );
126
+
127
+ // Check that runAgent got 3 goals (pre/main/post)
128
+ const call = (runAgent.mock.calls[0] || []);
129
+ const goals = call[0];
130
+ expect(Array.isArray(goals)).toBe(true);
131
+ expect(goals).toHaveLength(3);
132
+
133
+ // Submission types and names are set
134
+ expect(goals[0].submissionType).toBe('premission');
135
+ expect(goals[1].submissionType).toBe('mission');
136
+ expect(goals[2].submissionType).toBe('postmission');
137
+
138
+ // Mission name is the passed-in missionName for main
139
+ expect(goals[1].submissionName).toMatch(/^My Mission/);
140
+
141
+ // Effective max turns passed through
142
+ expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15);
143
+
144
+ log.mockRestore();
145
+ });
146
+
147
+ it('aborts early when runAgent returns falsy (defensive)', async () => {
148
+ loadConfig.mockResolvedValue({});
149
+ enforceTurnBudget.mockReturnValue({
150
+ effectiveMax: 10,
151
+ limits: {},
152
+ notes: [],
153
+ strict: false,
154
+ });
155
+
156
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});
157
+ runAgent.mockResolvedValue(null); // simulate unexpected falsy
158
+
159
+ const res = await runMissions({ mission: 'Edge case' }, 'Abort');
160
+ expect(res).toBeUndefined();
161
+
162
+ log.mockRestore();
163
+ });
164
+ });