testronaut 2.0.3 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +20 -6
- package/bin/terminalUi.js +4 -3
- package/core/agent.js +22 -5
- package/core/turnLoop.js +12 -1
- package/package.json +1 -1
- package/runner/testronaut.js +3 -1
- package/tests/binTests/cli.helpers.test.js +3 -3
- package/tests/binTests/terminalUi.test.js +2 -1
- package/tests/coreTests/agent.test.js +18 -0
- package/tests/runnerTests/testronaut.test.js +12 -0
- package/tests/toolsTests/contextControl.test.js +11 -0
- package/tests/toolsTests/generateHtmlReport.test.js +11 -0
- package/tests/toolsTests/toolSchema.test.js +11 -0
- package/tools/chromeBrowser.js +8 -0
- package/tools/contextControl.js +18 -0
- package/tools/generateHtmlReport.js +21 -4
- package/tools/toolSchema.js +26 -0
package/bin/cli.js
CHANGED
|
@@ -107,6 +107,8 @@ const screenshotsEnabled = !args.includes('--no-screenshots');
|
|
|
107
107
|
args = args.filter(arg => arg !== '--no-screenshots');
|
|
108
108
|
const uploadScreenshots = !args.includes('--no-upload-screenshots');
|
|
109
109
|
args = args.filter(arg => arg !== '--no-upload-screenshots');
|
|
110
|
+
const strictRecoveryFlag = args.includes('--strict-recovery');
|
|
111
|
+
args = args.filter(arg => arg !== '--strict-recovery');
|
|
110
112
|
process.env.TESTRONAUT_SCREENSHOTS = screenshotsEnabled ? '1' : '0';
|
|
111
113
|
|
|
112
114
|
let sessionMode = 'isolated';
|
|
@@ -430,6 +432,11 @@ function buildEffectiveConfig(config = {}, cwd = process.cwd()) {
|
|
|
430
432
|
tagMatch: config.tagMatch || 'any',
|
|
431
433
|
addTags: normalizeTags([...(config.addTags || []), ...cliAddTags]),
|
|
432
434
|
screenshots: screenshotsEnabled,
|
|
435
|
+
recovery: {
|
|
436
|
+
permitted: Array.isArray(config?.recovery?.permitted) ? config.recovery.permitted : null,
|
|
437
|
+
failBuild: strictRecoveryFlag || config?.recovery?.failBuild === true,
|
|
438
|
+
source: strictRecoveryFlag ? 'cli' : (config?.recovery?.failBuild != null ? 'config' : 'default'),
|
|
439
|
+
},
|
|
433
440
|
authenticated: Boolean(config.sessionToken),
|
|
434
441
|
optimization,
|
|
435
442
|
guardrails: {
|
|
@@ -487,9 +494,13 @@ function collectReportScreenshotPaths(report) {
|
|
|
487
494
|
const names = [];
|
|
488
495
|
const seen = new Set();
|
|
489
496
|
for (const mission of Array.isArray(report?.missions) ? report.missions : []) {
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
497
|
+
const references = [
|
|
498
|
+
...(Array.isArray(mission?.steps) ? mission.steps.map(step => step?.screenshotPath) : []),
|
|
499
|
+
...(Array.isArray(mission?.recoveries) ? mission.recoveries.map(recovery => recovery?.screenshot) : []),
|
|
500
|
+
];
|
|
501
|
+
for (const reference of references) {
|
|
502
|
+
if (typeof reference !== 'string') continue;
|
|
503
|
+
const name = reference.trim().replace(/^\.\//, '').replaceAll('\\', '/');
|
|
493
504
|
if (!name || seen.has(name)) continue;
|
|
494
505
|
seen.add(name);
|
|
495
506
|
names.push(name);
|
|
@@ -888,6 +899,7 @@ Options:
|
|
|
888
899
|
--dry-run Show the resolved mission plan without launching a browser
|
|
889
900
|
--no-screenshots Remove the screenshot tool for this run
|
|
890
901
|
--no-upload-screenshots Upload report JSON without its screenshots
|
|
902
|
+
--strict-recovery Exit non-zero when any mission passes after recovery
|
|
891
903
|
--json Emit machine-readable command/run output
|
|
892
904
|
--quiet Suppress informational logs
|
|
893
905
|
--base=<git-ref> Base ref for Jev suite selection (default: HEAD~1)
|
|
@@ -980,6 +992,7 @@ const {
|
|
|
980
992
|
} = await discoverMissionFiles({ cwd: process.cwd() });
|
|
981
993
|
for (const warning of missionDiscoveryWarnings) console.warn(`⚠️ ${warning}`);
|
|
982
994
|
const tagConfig = await loadConfig(process.cwd());
|
|
995
|
+
const strictRecovery = strictRecoveryFlag || tagConfig?.recovery?.failBuild === true;
|
|
983
996
|
const optimizationPolicy = resolveOptimizationPolicy({ config: tagConfig });
|
|
984
997
|
if (!optimizationPolicy.valid) {
|
|
985
998
|
console.error('❌ Invalid optimization objective or mode in testronaut-config.json.');
|
|
@@ -1186,7 +1199,7 @@ const runFile = async (filePath) => {
|
|
|
1186
1199
|
durationMs: Math.max(0, missionFileEndTime - missionFileStartTime),
|
|
1187
1200
|
});
|
|
1188
1201
|
const missionResults = Array.isArray(result) ? result : [result];
|
|
1189
|
-
return !missionResults.some(item => item?.status === 'failed');
|
|
1202
|
+
return !missionResults.some(item => item?.status === 'failed' || (strictRecovery && item?.status === 'recovered'));
|
|
1190
1203
|
}
|
|
1191
1204
|
console.error(`❌ Mission does not export executeMission(): ${filePath}`);
|
|
1192
1205
|
} catch (err) {
|
|
@@ -1256,7 +1269,7 @@ const { provider: llmProvider, model: llmModel } = resolveProviderModel({ cwd: p
|
|
|
1256
1269
|
|
|
1257
1270
|
|
|
1258
1271
|
const report = {
|
|
1259
|
-
schemaVersion:
|
|
1272
|
+
schemaVersion: 3,
|
|
1260
1273
|
runId,
|
|
1261
1274
|
cli: { version: CLI_VERSION },
|
|
1262
1275
|
startTime: startTime.toISOString(),
|
|
@@ -1271,6 +1284,7 @@ const report = {
|
|
|
1271
1284
|
summary: {
|
|
1272
1285
|
totalMissions: flatMissions.length,
|
|
1273
1286
|
passed: flatMissions.filter(m => m.status === 'passed').length,
|
|
1287
|
+
recovered: flatMissions.filter(m => m.status === 'recovered').length,
|
|
1274
1288
|
failed: flatMissions.filter(m => m.status === 'failed').length,
|
|
1275
1289
|
},
|
|
1276
1290
|
missions: flatMissions
|
|
@@ -1319,7 +1333,7 @@ fs.writeFileSync(path.join(outputDir, `${runId}.json`), JSON.stringify(report, n
|
|
|
1319
1333
|
const htmlReportPath = path.join(outputDir, `${runId}.html`);
|
|
1320
1334
|
generateHtmlReport(report, htmlReportPath);
|
|
1321
1335
|
if (jsonOutput) writeOutput(report);
|
|
1322
|
-
else terminalUi.summary({ ...report.summary, durationMs: report.durationMs, reportPath: path.relative(process.cwd(), htmlReportPath) });
|
|
1336
|
+
else terminalUi.summary({ ...report.summary, strictRecovery, durationMs: report.durationMs, reportPath: path.relative(process.cwd(), htmlReportPath) });
|
|
1323
1337
|
|
|
1324
1338
|
try {
|
|
1325
1339
|
if (!process.env.TN_KEEP_TMP && fs.existsSync(TMP_DIR)) {
|
package/bin/terminalUi.js
CHANGED
|
@@ -6,6 +6,7 @@ const ANSI = {
|
|
|
6
6
|
cyan: '\u001b[38;5;81m',
|
|
7
7
|
teal: '\u001b[38;5;43m',
|
|
8
8
|
red: '\u001b[38;5;203m',
|
|
9
|
+
amber: '\u001b[38;5;214m',
|
|
9
10
|
cream: '\u001b[38;5;230m',
|
|
10
11
|
};
|
|
11
12
|
|
|
@@ -44,13 +45,13 @@ export function createTerminalUi({
|
|
|
44
45
|
const tagText = tags.length ? paint(ANSI.dim, ` ${tags.map(tag => `#${tag}`).join(' ')}`) : '';
|
|
45
46
|
write(` ${marker} ${file}${tagText}`);
|
|
46
47
|
},
|
|
47
|
-
summary({ totalMissions = 0, passed = 0, failed = 0, durationMs, reportPath }) {
|
|
48
|
-
const status = failed === 0
|
|
48
|
+
summary({ totalMissions = 0, passed = 0, recovered = 0, failed = 0, strictRecovery = false, durationMs, reportPath }) {
|
|
49
|
+
const status = failed === 0 && !(strictRecovery && recovered > 0)
|
|
49
50
|
? paint(`${ANSI.bold}${ANSI.teal}`, 'MISSION COMPLETE')
|
|
50
51
|
: paint(`${ANSI.bold}${ANSI.red}`, 'MISSION NEEDS REVIEW');
|
|
51
52
|
write('');
|
|
52
53
|
write(`${label('COMPLETE')}${status}`);
|
|
53
|
-
write(` ${paint(ANSI.teal, `${passed} passed`)} ${paint(failed ? ANSI.red : ANSI.dim, `${failed} failed`)} ${paint(ANSI.dim, `${totalMissions} total`)}`);
|
|
54
|
+
write(` ${paint(ANSI.teal, `${passed} passed`)} ${paint(recovered ? ANSI.amber : ANSI.dim, `${recovered} recovered`)} ${paint(failed ? ANSI.red : ANSI.dim, `${failed} failed`)} ${paint(ANSI.dim, `${totalMissions} total`)}`);
|
|
54
55
|
const duration = formatDuration(durationMs);
|
|
55
56
|
if (duration) write(paint(ANSI.dim, ` Duration: ${duration}`));
|
|
56
57
|
if (reportPath) write(paint(ANSI.dim, ` Report: ${reportPath}`));
|
package/core/agent.js
CHANGED
|
@@ -37,8 +37,8 @@ import { hasCompletedLaunchProtocol, markLaunchProtocolCompleted } from './launc
|
|
|
37
37
|
* @param {string} missionName
|
|
38
38
|
* @param {number} [maxTurns=20] - upper bound for turnLoop per goal
|
|
39
39
|
* @param {number} [retryLimit]
|
|
40
|
-
* @param {{ domListLimit?: number|typeof Infinity, automaticDomLimit?: number, storageStatePath?: string, debug?: boolean, resourceGuard?: { enabled:boolean, hrefIncludes:string[], dataTypes:string[] }, humanInput?: { enabled:boolean, timeoutSeconds:number } }} [opts]
|
|
41
|
-
* @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'failed', steps:any[], startTime:number, endTime:number}>>}
|
|
40
|
+
* @param {{ domListLimit?: number|typeof Infinity, automaticDomLimit?: number, storageStatePath?: string, debug?: boolean, resourceGuard?: { enabled:boolean, hrefIncludes:string[], dataTypes:string[] }, humanInput?: { enabled:boolean, timeoutSeconds:number }, recovery?: { permitted?: string[] } }} [opts]
|
|
41
|
+
* @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'recovered'|'failed', steps:any[], startTime:number, endTime:number}>>}
|
|
42
42
|
*/
|
|
43
43
|
export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, opts = {}) {
|
|
44
44
|
const browser = new ChromeBrowser({
|
|
@@ -61,6 +61,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
|
|
|
61
61
|
for (const goal of goals) {
|
|
62
62
|
const goalStartTime = Date.now();
|
|
63
63
|
const telemetryStart = groundControl.telemetry.length;
|
|
64
|
+
const recoveryStart = groundControl.recoveries.length;
|
|
64
65
|
const steps = [];
|
|
65
66
|
const stepsArchive = [];
|
|
66
67
|
// Unique-ish JSONL file for this mission’s steps
|
|
@@ -220,6 +221,15 @@ Email invitation and magic links:
|
|
|
220
221
|
- open_email_link enforces the operator-configured emailLinks.allowedHosts policy. If the host is not configured or navigation is blocked, report a graceful FAILURE; do not use the ordinary navigate tool as a workaround.
|
|
221
222
|
`.trim();
|
|
222
223
|
|
|
224
|
+
const permittedRecoveries = Array.isArray(opts.recovery?.permitted)
|
|
225
|
+
? opts.recovery.permitted.map(value => String(value).trim().toLowerCase()).filter(Boolean)
|
|
226
|
+
: null;
|
|
227
|
+
systemContent += `\n\nRecovery reporting:\n- Call record_recovery before or immediately after taking an action a normal user would not know to take in order to continue.\n- Ordinary mission steps and retries of selectors are not recoveries.\n- ${permittedRecoveries === null
|
|
228
|
+
? 'This mission has not restricted recovery actions.'
|
|
229
|
+
: permittedRecoveries.length
|
|
230
|
+
? `Only these recovery actions are permitted: ${permittedRecoveries.join(', ')}.`
|
|
231
|
+
: 'No recovery actions are permitted; recording one will fail the mission.'}`;
|
|
232
|
+
|
|
223
233
|
if (opts.humanInput?.enabled !== false) {
|
|
224
234
|
systemContent +=
|
|
225
235
|
'\n' +
|
|
@@ -294,25 +304,32 @@ Email invitation and magic links:
|
|
|
294
304
|
const goalEndTime = Date.now();
|
|
295
305
|
|
|
296
306
|
// Snapshot steps (avoid retaining references to mutable arrays)
|
|
307
|
+
const recoveries = JSON.parse(JSON.stringify((groundControl.recoveries || []).slice(recoveryStart)));
|
|
308
|
+
const disallowedRecoveries = permittedRecoveries === null
|
|
309
|
+
? []
|
|
310
|
+
: recoveries.filter(recovery => !permittedRecoveries.includes(recovery.action));
|
|
311
|
+
const succeeded = Boolean(result?.success) && disallowedRecoveries.length === 0;
|
|
297
312
|
missionResults.push({
|
|
298
313
|
missionName,
|
|
299
314
|
submissionType: goal.submissionType || goal.label,
|
|
300
315
|
submissionName: goal.submissionName || null,
|
|
301
|
-
status:
|
|
316
|
+
status: succeeded ? (recoveries.length ? 'recovered' : 'passed') : 'failed',
|
|
302
317
|
steps: JSON.parse(JSON.stringify(compact)), // last 20 (by memory design)
|
|
303
318
|
stepFile, // full history is in this JSONL (one step per line)
|
|
304
319
|
startTime: goalStartTime,
|
|
305
320
|
endTime: goalEndTime,
|
|
306
321
|
durationMs: Math.max(0, goalEndTime - goalStartTime),
|
|
307
322
|
telemetry: JSON.parse(JSON.stringify(groundControl.telemetry.slice(telemetryStart))),
|
|
323
|
+
recoveries,
|
|
324
|
+
...(disallowedRecoveries.length ? { recoveryPolicyViolation: disallowedRecoveries.map(item => item.action) } : {}),
|
|
308
325
|
...(protocolResult ? { launchProtocol: protocolResult } : {}),
|
|
309
326
|
});
|
|
310
327
|
|
|
311
|
-
if (protocol &&
|
|
328
|
+
if (protocol && succeeded) {
|
|
312
329
|
markLaunchProtocolCompleted(protocol.id);
|
|
313
330
|
}
|
|
314
331
|
|
|
315
|
-
if (!
|
|
332
|
+
if (!succeeded) {
|
|
316
333
|
console.log('🛑 Agent stopped due to failed goal.\n');
|
|
317
334
|
return missionResults;
|
|
318
335
|
}
|
package/core/turnLoop.js
CHANGED
|
@@ -59,6 +59,7 @@ import {
|
|
|
59
59
|
createEmptyGroundControl,
|
|
60
60
|
applyGroundControlUpdate,
|
|
61
61
|
recordGroundTelemetry,
|
|
62
|
+
recordRecovery,
|
|
62
63
|
} from '../tools/contextControl.js';
|
|
63
64
|
|
|
64
65
|
// ─────────────────────────────────────────────
|
|
@@ -159,6 +160,7 @@ const TURN_RETRY_BASE_DELAY_MS = 500;
|
|
|
159
160
|
const FIRE_AND_FORGET_TOOLS = new Set([
|
|
160
161
|
'screenshot',
|
|
161
162
|
'click',
|
|
163
|
+
'reload',
|
|
162
164
|
'click_text',
|
|
163
165
|
// 'expand_menu',
|
|
164
166
|
'fill',
|
|
@@ -170,6 +172,7 @@ const FIRE_AND_FORGET_TOOLS = new Set([
|
|
|
170
172
|
'request_human_input',
|
|
171
173
|
'set_ground_control_state',
|
|
172
174
|
'record_mission_telemetry',
|
|
175
|
+
'record_recovery',
|
|
173
176
|
]);
|
|
174
177
|
|
|
175
178
|
function currentBrowserUrl(browser) {
|
|
@@ -630,6 +633,14 @@ export const turnLoop = async (
|
|
|
630
633
|
} else if (fnName === 'record_mission_telemetry') {
|
|
631
634
|
const recorded = recordGroundTelemetry(groundControl, args, { turn });
|
|
632
635
|
result = { ok: true, recorded };
|
|
636
|
+
} else if (fnName === 'record_recovery') {
|
|
637
|
+
let screenshot = null;
|
|
638
|
+
if (args.screenshot && process.env.TESTRONAUT_SCREENSHOTS !== '0') {
|
|
639
|
+
const screenshotResult = await CHROME_TOOL_MAP.screenshot(browser, {}, agentMemory);
|
|
640
|
+
screenshot = String(screenshotResult || '').match(/(?:\.\/)?screenshots\/[^\s]+/)?.[0] || null;
|
|
641
|
+
}
|
|
642
|
+
const recorded = recordRecovery(groundControl, args, { turn, screenshot });
|
|
643
|
+
result = { ok: true, recorded };
|
|
633
644
|
} else {
|
|
634
645
|
const toolHandler = CHROME_TOOL_MAP[fnName];
|
|
635
646
|
if (typeof toolHandler !== 'function') throw new Error(`Unknown tool: ${fnName}`);
|
|
@@ -973,7 +984,7 @@ export const turnLoop = async (
|
|
|
973
984
|
// model functionCall to immediately follow user content or a
|
|
974
985
|
// functionResponse; inserting it before `msg` would create adjacent,
|
|
975
986
|
// reversed model function-call turns.
|
|
976
|
-
if (!errorMessage && ['click_text', 'click', 'expand_menu'].includes(fnName)) {
|
|
987
|
+
if (!errorMessage && ['click_text', 'click', 'expand_menu', 'reload'].includes(fnName)) {
|
|
977
988
|
domRefreshSource = fnName;
|
|
978
989
|
}
|
|
979
990
|
}
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -50,7 +50,7 @@ const formatListLimit = (v) => v === Infinity ? 'all' : v;
|
|
|
50
50
|
* @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
|
|
51
51
|
* @param {string} missionName
|
|
52
52
|
*/
|
|
53
|
-
export async function runMissions({ preMission, mission, postMission, tags = [] }, missionName) {
|
|
53
|
+
export async function runMissions({ preMission, mission, postMission, tags = [], recovery }, missionName) {
|
|
54
54
|
// 1) Config and turn budget (with guardrails)
|
|
55
55
|
const cfg = await loadConfig();
|
|
56
56
|
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
@@ -61,6 +61,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
61
61
|
const automaticDomLimitInfo = getAutomaticDomLimit(cfg);
|
|
62
62
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
63
63
|
const humanInput = getHumanInputConfig(cfg);
|
|
64
|
+
const recoveryPolicy = recovery ?? cfg?.recovery;
|
|
64
65
|
const debugEnabled = isDebugEnabled();
|
|
65
66
|
const missionTags = normalizeTags([
|
|
66
67
|
...normalizeTags(tags),
|
|
@@ -226,6 +227,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
226
227
|
debug: debugEnabled,
|
|
227
228
|
resourceGuard,
|
|
228
229
|
humanInput,
|
|
230
|
+
recovery: recoveryPolicy,
|
|
229
231
|
}
|
|
230
232
|
);
|
|
231
233
|
if (!success) {
|
|
@@ -100,10 +100,10 @@ describe('cli helpers', () => {
|
|
|
100
100
|
const { collectReportScreenshotNames, collectReportScreenshotPaths } = __test__;
|
|
101
101
|
const report = { missions: [
|
|
102
102
|
{ steps: [{ screenshotPath: './screenshots/one.png' }, { screenshotPath: './screenshots/two.jpg' }] },
|
|
103
|
-
{ steps: [{ screenshotPath: './screenshots/one.png' }, { events: [] }] },
|
|
103
|
+
{ steps: [{ screenshotPath: './screenshots/one.png' }, { events: [] }], recoveries: [{ screenshot: './screenshots/recovery.png' }] },
|
|
104
104
|
] };
|
|
105
|
-
expect(collectReportScreenshotNames(report)).toEqual(['one.png', 'two.jpg']);
|
|
106
|
-
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/one.png', 'screenshots/two.jpg']);
|
|
105
|
+
expect(collectReportScreenshotNames(report)).toEqual(['one.png', 'two.jpg', 'recovery.png']);
|
|
106
|
+
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/one.png', 'screenshots/two.jpg', 'screenshots/recovery.png']);
|
|
107
107
|
});
|
|
108
108
|
|
|
109
109
|
it('preserves run-specific screenshot paths', () => {
|
|
@@ -17,12 +17,13 @@ describe('terminal UI', () => {
|
|
|
17
17
|
ui.banner({ version: '1.2.3', provider: 'openai', model: 'gpt-test', session: 'isolated' });
|
|
18
18
|
ui.phase('launch', 'login.mission.js');
|
|
19
19
|
ui.mission(true, 'login.mission.js', ['smoke']);
|
|
20
|
-
ui.summary({ totalMissions:
|
|
20
|
+
ui.summary({ totalMissions: 2, passed: 1, recovered: 1, failed: 0, durationMs: 1234, reportPath: 'reports/run.html' });
|
|
21
21
|
|
|
22
22
|
expect(target.read()).toContain('🧑🚀 TESTRONAUT / MISSION RUNNER');
|
|
23
23
|
expect(target.read()).toContain('LAUNCH login.mission.js');
|
|
24
24
|
expect(target.read()).toContain('#smoke');
|
|
25
25
|
expect(target.read()).toContain('MISSION COMPLETE');
|
|
26
|
+
expect(target.read()).toContain('1 recovered');
|
|
26
27
|
expect(target.read()).toContain('Duration: 1.23s');
|
|
27
28
|
expect(target.read()).not.toContain('\u001b[');
|
|
28
29
|
});
|
|
@@ -119,6 +119,24 @@ describe('core/agent.runAgent', () => {
|
|
|
119
119
|
expect(result.telemetry).toEqual([{ kind: 'note', text: 'durable observation' }]);
|
|
120
120
|
});
|
|
121
121
|
|
|
122
|
+
it('marks a successful goal recovered and enforces permitted actions', async () => {
|
|
123
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
124
|
+
ctx.groundControl.recoveries.push({ turn: 2, action: 'reload', reason: 'message was missing' });
|
|
125
|
+
return { success: true };
|
|
126
|
+
});
|
|
127
|
+
const [recovered] = await runAgent([{ goal: 'Observe' }], 'Recovery', 3, 2, { recovery: { permitted: ['reload'] } });
|
|
128
|
+
expect(recovered.status).toBe('recovered');
|
|
129
|
+
expect(recovered.recoveries).toEqual([{ turn: 2, action: 'reload', reason: 'message was missing' }]);
|
|
130
|
+
|
|
131
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
132
|
+
ctx.groundControl.recoveries.push({ turn: 1, action: 'resend', reason: 'submission disappeared' });
|
|
133
|
+
return { success: true };
|
|
134
|
+
});
|
|
135
|
+
const [failed] = await runAgent([{ goal: 'Observe' }], 'Recovery', 3, 2, { recovery: { permitted: ['reload'] } });
|
|
136
|
+
expect(failed.status).toBe('failed');
|
|
137
|
+
expect(failed.recoveryPolicyViolation).toEqual(['resend']);
|
|
138
|
+
});
|
|
139
|
+
|
|
122
140
|
it('reuses a completed launch protocol after a passing shared-auth probe', async () => {
|
|
123
141
|
process.env.TESTRONAUT_SESSION_MODE = 'shared-auth';
|
|
124
142
|
const probeSpy = vi.spyOn(ChromeBrowser.prototype, 'probe').mockResolvedValue({
|
|
@@ -67,6 +67,18 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
67
67
|
log.mockRestore();
|
|
68
68
|
});
|
|
69
69
|
|
|
70
|
+
it('uses project recovery policy unless the mission overrides it', async () => {
|
|
71
|
+
loadConfig.mockResolvedValue({ maxTurns: 20, recovery: { permitted: ['reload'] } });
|
|
72
|
+
enforceTurnBudget.mockReturnValue({ effectiveMax: 20, limits: {}, notes: [] });
|
|
73
|
+
runAgent.mockResolvedValue([{ steps: [{ result: 'SUCCESS' }], status: 'passed' }]);
|
|
74
|
+
|
|
75
|
+
await runMissions({ mission: 'Check dashboard' }, 'Configured recovery');
|
|
76
|
+
expect(runAgent.mock.calls[0][4].recovery).toEqual({ permitted: ['reload'] });
|
|
77
|
+
|
|
78
|
+
await runMissions({ mission: 'Check dashboard', recovery: { permitted: [] } }, 'No recovery');
|
|
79
|
+
expect(runAgent.mock.calls[1][4].recovery).toEqual({ permitted: [] });
|
|
80
|
+
});
|
|
81
|
+
|
|
70
82
|
it('unwraps launch protocols while preserving preMission authoring', async () => {
|
|
71
83
|
loadConfig.mockResolvedValue({ maxTurns: 20 });
|
|
72
84
|
enforceTurnBudget.mockReturnValue({
|
|
@@ -27,9 +27,20 @@ import {
|
|
|
27
27
|
createEmptyGroundControl,
|
|
28
28
|
applyGroundControlUpdate,
|
|
29
29
|
recordGroundTelemetry,
|
|
30
|
+
recordRecovery,
|
|
30
31
|
summarizeGroundControlForPrompt
|
|
31
32
|
} from '../../tools/contextControl.js';
|
|
32
33
|
|
|
34
|
+
describe('recordRecovery', () => {
|
|
35
|
+
it('normalizes and records structured recovery evidence', () => {
|
|
36
|
+
const groundControl = createEmptyGroundControl();
|
|
37
|
+
expect(recordRecovery(groundControl, { action: ' Reload ', reason: 'Message missing', evidence: 'Appeared later' }, { turn: 3, screenshot: './screenshots/recovery.png' })).toEqual({
|
|
38
|
+
turn: 3, action: 'reload', reason: 'Message missing', evidence: 'Appeared later', screenshot: './screenshots/recovery.png',
|
|
39
|
+
});
|
|
40
|
+
expect(groundControl.recoveries).toHaveLength(1);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
33
44
|
describe('sanitizeHeavyToolHistory', () => {
|
|
34
45
|
it('stubs older get_dom tool messages but keeps the latest N intact', () => {
|
|
35
46
|
const messages = [
|
|
@@ -18,6 +18,17 @@ afterEach(() => {
|
|
|
18
18
|
});
|
|
19
19
|
|
|
20
20
|
describe('generateHtmlReport', () => {
|
|
21
|
+
it('renders recovered status and recovery evidence', () => {
|
|
22
|
+
const outPath = path.join(tmpDir, 'recovered.html');
|
|
23
|
+
generateHtmlReport({ summary: { passed: 0, recovered: 1, failed: 0 }, missions: [{
|
|
24
|
+
missionName: 'Chat', status: 'recovered', recoveries: [{ turn: 1, action: 'reload', reason: 'message missing', screenshot: './screenshots/recovery.png' }], steps: [],
|
|
25
|
+
}] }, outPath);
|
|
26
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
27
|
+
expect(html).toContain('Passed after recovery');
|
|
28
|
+
expect(html).toContain('Recovered: 1');
|
|
29
|
+
expect(html).toContain('message missing');
|
|
30
|
+
expect(html).toContain('./screenshots/recovery.png');
|
|
31
|
+
});
|
|
21
32
|
it('points local reports to hosted analytics for the same run', () => {
|
|
22
33
|
const report = { runId: 'run_analytics_123', missions: [] };
|
|
23
34
|
const outPath = path.join(tmpDir, 'analytics-cta.html');
|
|
@@ -6,6 +6,17 @@ describe('tools/toolSchema', () => {
|
|
|
6
6
|
expect(toolsSchema.map(tool => tool.function.name)).not.toContain('get_dom_chunk');
|
|
7
7
|
});
|
|
8
8
|
|
|
9
|
+
it('exposes structured recovery recording', () => {
|
|
10
|
+
const entry = toolsSchema.find(tool => tool.function.name === 'record_recovery');
|
|
11
|
+
expect(entry.function.parameters.required).toEqual(['action', 'reason']);
|
|
12
|
+
expect(entry.function.parameters.properties).toHaveProperty('screenshot');
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it('exposes reload as a first-class browser action', () => {
|
|
16
|
+
const entry = toolsSchema.find(tool => tool.function.name === 'reload');
|
|
17
|
+
expect(entry.function.parameters).toEqual({ type: 'object', properties: {}, additionalProperties: false });
|
|
18
|
+
});
|
|
19
|
+
|
|
9
20
|
it('allows get_dom calls to be scoped by selector', () => {
|
|
10
21
|
const entry = toolsSchema.find(tool => tool.function.name === 'get_dom');
|
|
11
22
|
expect(entry.function.parameters.properties.selector.type).toBe('string');
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -641,6 +641,13 @@ export class ChromeBrowser {
|
|
|
641
641
|
return `navigated to ${this.page.url()}`;
|
|
642
642
|
}
|
|
643
643
|
|
|
644
|
+
async reload() {
|
|
645
|
+
await this.page.reload({ timeout: 30_000, waitUntil: 'domcontentloaded' });
|
|
646
|
+
await this.page.waitForLoadState('domcontentloaded').catch(() => {});
|
|
647
|
+
try { await this._waitLoaderGone(); } catch {}
|
|
648
|
+
return `reloaded ${this.page.url()}`;
|
|
649
|
+
}
|
|
650
|
+
|
|
644
651
|
async probe({ url, selector, text, timeout = 5000 } = {}) {
|
|
645
652
|
if (url) {
|
|
646
653
|
await this.page.goto(url, { timeout: 30_000, waitUntil: 'domcontentloaded' });
|
|
@@ -1445,6 +1452,7 @@ export class ChromeBrowser {
|
|
|
1445
1452
|
|
|
1446
1453
|
export const CHROME_TOOL_MAP = {
|
|
1447
1454
|
navigate: (b, args) => b.navigate(args),
|
|
1455
|
+
reload: (b) => b.reload(),
|
|
1448
1456
|
fill: (b, args) => b.fill(args),
|
|
1449
1457
|
click: (b, args) => b.click(args),
|
|
1450
1458
|
get_dom: (b, args, agentMemory) => b.get_dom(args, agentMemory),
|
package/tools/contextControl.js
CHANGED
|
@@ -140,9 +140,27 @@ export function createEmptyGroundControl() {
|
|
|
140
140
|
stayWithinBaseUrl: null, // if true, avoid leaving app.baseUrl unless told to
|
|
141
141
|
},
|
|
142
142
|
telemetry: [], // array of { kind, text, status, ts, turn? }
|
|
143
|
+
recoveries: [], // array of { turn, action, reason, evidence?, screenshot? }
|
|
143
144
|
};
|
|
144
145
|
}
|
|
145
146
|
|
|
147
|
+
export function recordRecovery(groundControl, entry, meta = {}) {
|
|
148
|
+
if (!groundControl) throw new Error('recordRecovery called without Ground Control state');
|
|
149
|
+
groundControl.recoveries ||= [];
|
|
150
|
+
const action = String(entry?.action || '').trim().toLowerCase();
|
|
151
|
+
const reason = String(entry?.reason || '').trim();
|
|
152
|
+
if (!action || !reason) return null;
|
|
153
|
+
const recovery = {
|
|
154
|
+
turn: meta.turn,
|
|
155
|
+
action,
|
|
156
|
+
reason,
|
|
157
|
+
...(entry?.evidence ? { evidence: String(entry.evidence).trim() } : {}),
|
|
158
|
+
...(meta.screenshot ? { screenshot: meta.screenshot } : {}),
|
|
159
|
+
};
|
|
160
|
+
groundControl.recoveries.push(recovery);
|
|
161
|
+
return recovery;
|
|
162
|
+
}
|
|
163
|
+
|
|
146
164
|
/**
|
|
147
165
|
* Apply a partial update from the `set_ground_control_state` tool to the
|
|
148
166
|
* existing Ground Control state.
|
|
@@ -43,6 +43,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
43
43
|
.replace(/"/g, '"');
|
|
44
44
|
const badge = (status) =>
|
|
45
45
|
status === 'passed' ? '✅ Passed' :
|
|
46
|
+
status === 'recovered' ? '⚠️ Passed after recovery' :
|
|
46
47
|
status === 'failed' ? '❌ Failed' : (status || '—');
|
|
47
48
|
const providerKey = String(llm.provider ?? '').trim().toLowerCase() === 'claude'
|
|
48
49
|
? 'anthropic'
|
|
@@ -160,7 +161,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
160
161
|
const type = String(m.submissionType || 'mission').toLowerCase();
|
|
161
162
|
const typeLabel = (type === 'premission' || type === 'postmission') ? type : 'mission';
|
|
162
163
|
const prettyTitle = `${esc(m.missionName || 'Mission')} — ${typeLabel}${m.submissionName ? `: ${esc(m.submissionName)}` : ''}`;
|
|
163
|
-
const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'passed' ? 'ok' : '');
|
|
164
|
+
const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'recovered' ? 'warn' : (m.status === 'passed' ? 'ok' : ''));
|
|
165
|
+
const recoveries = Array.isArray(m.recoveries) ? m.recoveries : [];
|
|
166
|
+
const recoveryItems = recoveries.length ? `
|
|
167
|
+
<div class="recoveries">
|
|
168
|
+
<h4>Recoveries</h4>
|
|
169
|
+
${recoveries.map(recovery => `<div class="recovery"><strong>Turn ${esc(Number(recovery.turn ?? 0) + 1)} · ${esc(recovery.action)}</strong><span>${esc(recovery.reason)}</span>${recovery.evidence ? `<span>${esc(recovery.evidence)}</span>` : ''}${recovery.screenshot ? `<img src="${esc(recovery.screenshot)}" alt="Recovery evidence">` : ''}</div>`).join('')}
|
|
170
|
+
</div>` : '';
|
|
164
171
|
|
|
165
172
|
return `
|
|
166
173
|
<details class="mission-submission">
|
|
@@ -173,6 +180,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
173
180
|
</span>
|
|
174
181
|
</summary>
|
|
175
182
|
<div class="steps">
|
|
183
|
+
${recoveryItems}
|
|
176
184
|
${stepItems || '<div class="empty">No steps recorded.</div>'}
|
|
177
185
|
</div>
|
|
178
186
|
</details>
|
|
@@ -186,7 +194,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
186
194
|
return acc;
|
|
187
195
|
}, {});
|
|
188
196
|
|
|
189
|
-
const groupStatus = (subs) => subs.some(s => s.status === 'failed')
|
|
197
|
+
const groupStatus = (subs) => subs.some(s => s.status === 'failed')
|
|
198
|
+
? 'failed'
|
|
199
|
+
: subs.some(s => s.status === 'recovered') ? 'recovered' : 'passed';
|
|
190
200
|
|
|
191
201
|
const missionGroupBlock = (missionName, subs) => {
|
|
192
202
|
const status = groupStatus(subs);
|
|
@@ -213,7 +223,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
213
223
|
<summary>
|
|
214
224
|
<span class="name">${esc(missionName)}</span>
|
|
215
225
|
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
216
|
-
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
226
|
+
<span class="status ${status === 'failed' ? 'bad' : status === 'recovered' ? 'warn' : 'ok'}">${badge(status)}</span>
|
|
217
227
|
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} • ` : ''}submissions: ${subs.length} • steps: ${totalSteps} • duration: ${groupDur}s • ${esc(tokenMeta(groupModelUsage, groupJevUsage))}</span>
|
|
218
228
|
<span class="toolbar">
|
|
219
229
|
<button class="btn-mini toggle" data-scope="mission" aria-label="Expand">▼</button>
|
|
@@ -234,9 +244,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
234
244
|
const st = groupStatus(grouped[name]);
|
|
235
245
|
acc.total += 1;
|
|
236
246
|
acc.passed += st === 'passed' ? 1 : 0;
|
|
247
|
+
acc.recovered += st === 'recovered' ? 1 : 0;
|
|
237
248
|
acc.failed += st === 'failed' ? 1 : 0;
|
|
238
249
|
return acc;
|
|
239
|
-
}, { total: 0, passed: 0, failed: 0 });
|
|
250
|
+
}, { total: 0, passed: 0, recovered: 0, failed: 0 });
|
|
240
251
|
|
|
241
252
|
const html = `<!DOCTYPE html>
|
|
242
253
|
<html>
|
|
@@ -379,9 +390,14 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
379
390
|
.name{ flex:1; }
|
|
380
391
|
.status.ok{ color: var(--ok); font-weight:700; }
|
|
381
392
|
.status.bad{ color: var(--bad); font-weight:700; }
|
|
393
|
+
.status.warn{ color: var(--amber); font-weight:700; }
|
|
382
394
|
.meta{ color: var(--text-muted); font-size:12px; }
|
|
383
395
|
|
|
384
396
|
.steps{ padding: 0 12px 12px; }
|
|
397
|
+
.recoveries{ margin:12px 0; padding:12px; border:1px solid rgba(251,191,36,.4); border-radius:12px; background:rgba(251,191,36,.1); }
|
|
398
|
+
.recoveries h4{ margin:0 0 8px; color:var(--amber); }
|
|
399
|
+
.recovery{ display:grid; gap:4px; margin-top:8px; color:var(--text-muted); }
|
|
400
|
+
.recovery strong{ color:var(--text); }
|
|
385
401
|
.step summary{
|
|
386
402
|
background: rgba(255,255,255,.04);
|
|
387
403
|
border:1px solid var(--hairline);
|
|
@@ -497,6 +513,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
497
513
|
<div class="summary">
|
|
498
514
|
<div class="pill">Missions: ${esc(summary.totalMissions ?? totals.total)}</div>
|
|
499
515
|
<div class="pill ok">Passed: ${esc(summary.passed ?? totals.passed)}</div>
|
|
516
|
+
<div class="pill warn">Recovered: ${esc(summary.recovered ?? totals.recovered)}</div>
|
|
500
517
|
<div class="pill bad">Failed: ${esc(summary.failed ?? totals.failed)}</div>
|
|
501
518
|
${optimization?.configured ? `<div class="pill">Optimization: ${esc(optimization.objective)} (${esc(optimization.mode)})</div>` : ''}
|
|
502
519
|
${Object.entries(reportModelUsage.byModel).length > 1
|
package/tools/toolSchema.js
CHANGED
|
@@ -26,6 +26,14 @@ const toolsSchema = [
|
|
|
26
26
|
},
|
|
27
27
|
},
|
|
28
28
|
},
|
|
29
|
+
{
|
|
30
|
+
type: 'function',
|
|
31
|
+
function: {
|
|
32
|
+
name: 'reload',
|
|
33
|
+
description: 'Reload the current page. When reload is used to recover from an unexpected state, also call record_recovery with action "reload".',
|
|
34
|
+
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
35
|
+
},
|
|
36
|
+
},
|
|
29
37
|
{
|
|
30
38
|
type: 'function',
|
|
31
39
|
function: {
|
|
@@ -528,6 +536,24 @@ const toolsSchema = [
|
|
|
528
536
|
}
|
|
529
537
|
}
|
|
530
538
|
},
|
|
539
|
+
{
|
|
540
|
+
type: 'function',
|
|
541
|
+
function: {
|
|
542
|
+
name: 'record_recovery',
|
|
543
|
+
description: 'Record a deviation from the normal user journey that was needed to continue, such as reloading, resending, or dismissing an unexpected dialog.',
|
|
544
|
+
parameters: {
|
|
545
|
+
type: 'object',
|
|
546
|
+
properties: {
|
|
547
|
+
action: { type: 'string', description: 'Short stable action name, for example "reload", "resend", or "dismiss_dialog".' },
|
|
548
|
+
reason: { type: 'string', description: 'Why the recovery was necessary and what was observed before it.' },
|
|
549
|
+
evidence: { type: 'string', description: 'Optional short supporting observation.' },
|
|
550
|
+
screenshot: { type: 'boolean', description: 'Capture and attach a screenshot at the time the recovery is recorded.' },
|
|
551
|
+
},
|
|
552
|
+
required: ['action', 'reason'],
|
|
553
|
+
additionalProperties: false,
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
},
|
|
531
557
|
{
|
|
532
558
|
type: 'function',
|
|
533
559
|
function: {
|