testronaut 1.3.0 ā 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/cli.js +124 -16
- package/bin/init.js +2 -0
- package/bin/initHelpers.js +8 -0
- package/core/agent.js +21 -13
- package/core/config.js +121 -0
- package/core/missionDiscovery.js +55 -0
- package/core/turnLoop.js +169 -8
- package/package.json +3 -2
- package/runner/testronaut.js +71 -2
- package/tests/binTests/cli.helpers.test.js +45 -0
- package/tests/binTests/init.test.js +6 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +3 -0
- package/tests/coreTests/config.test.js +137 -0
- package/tests/coreTests/missionDiscovery.test.js +81 -0
- package/tests/coreTests/turnLoop.test.js +100 -3
- package/tests/runnerTests/testronaut.test.js +76 -6
- package/tests/toolsTests/domControl.test.js +35 -0
- package/tests/toolsTests/generateHtmlReport.test.js +90 -0
- package/tests/toolsTests/toolSchema.test.js +24 -0
- package/tools/chromeBrowser.js +211 -4
- package/tools/domControl.js +10 -3
- package/tools/generateHtmlReport.js +38 -4
- package/tools/toolSchema.js +43 -1
package/core/turnLoop.js
CHANGED
|
@@ -63,10 +63,56 @@ console.log(`š§ Using LLM ā provider: ${PROVIDER_ID}, model: ${MODEL_ID}`);
|
|
|
63
63
|
|
|
64
64
|
const llm = getLLM(PROVIDER_ID);
|
|
65
65
|
|
|
66
|
+
// Track resource/download coverage across turns (guard against partial loops).
|
|
67
|
+
function ensureDocProgress(agentMemory, cfg) {
|
|
68
|
+
if (!cfg?.enabled) return null;
|
|
69
|
+
if (!agentMemory.docProgress) {
|
|
70
|
+
agentMemory.docProgress = {
|
|
71
|
+
items: [],
|
|
72
|
+
downloaded: new Set(),
|
|
73
|
+
lastSummary: '',
|
|
74
|
+
lastScriptCount: 0,
|
|
75
|
+
patterns: cfg,
|
|
76
|
+
};
|
|
77
|
+
} else {
|
|
78
|
+
agentMemory.docProgress.patterns = cfg;
|
|
79
|
+
}
|
|
80
|
+
return agentMemory.docProgress;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Parse injected <pre data-testronaut-doc-list> summary into structured items.
|
|
84
|
+
function parseDocListFromDom(html) {
|
|
85
|
+
try {
|
|
86
|
+
const match = html.match(/<pre[^>]*data-testronaut-doc-list[^>]*>([\s\S]*?)<\/pre>/i);
|
|
87
|
+
if (!match) return { items: [], scriptDocs: 0 };
|
|
88
|
+
const text = match[1];
|
|
89
|
+
const lines = text.split('\n').map(l => l.trim()).filter(Boolean).slice(1);
|
|
90
|
+
const items = lines.map(l => {
|
|
91
|
+
const m = l.match(/-\s*\[(.*?)\]\s*(.*?)\s+(\/document\/\d+[^\s]*)?/i);
|
|
92
|
+
return {
|
|
93
|
+
id: m?.[1] || '',
|
|
94
|
+
title: m?.[2] || l.replace(/^-/, '').trim(),
|
|
95
|
+
href: m?.[3] || '',
|
|
96
|
+
};
|
|
97
|
+
}).filter(it => it.id || it.title || it.href);
|
|
98
|
+
return { items, scriptDocs: items.length };
|
|
99
|
+
} catch {
|
|
100
|
+
return { items: [], scriptDocs: 0 };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Extract numeric document IDs from common /document/<id> URLs.
|
|
105
|
+
function extractDocIdFromUrl(url = '') {
|
|
106
|
+
const m = url.match(/\/document\/(\d+)/i);
|
|
107
|
+
return m ? m[1] : '';
|
|
108
|
+
}
|
|
109
|
+
|
|
66
110
|
// Rolling token counters used for self-throttling
|
|
67
111
|
let totalTokensUsed = 0;
|
|
68
112
|
let turnTimestamps = [];
|
|
69
113
|
let shouldBackoff;
|
|
114
|
+
const DEFAULT_TURN_RETRY_LIMIT = 2; // number of retries (not counting initial attempt)
|
|
115
|
+
const TURN_RETRY_BASE_DELAY_MS = 500;
|
|
70
116
|
|
|
71
117
|
// Certain tools mutate the browser or produce side effects that are
|
|
72
118
|
// *useful for humans* (reports/screenshots), but do not carry semantic
|
|
@@ -171,16 +217,31 @@ export const turnLoop = async (
|
|
|
171
217
|
currentStep = {},
|
|
172
218
|
ctx = {} // { steps, missionName, groundControl }
|
|
173
219
|
) => {
|
|
174
|
-
const { steps = [], missionName, groundControl = createEmptyGroundControl() } = ctx;
|
|
220
|
+
const { steps = [], missionName, groundControl = createEmptyGroundControl(), retryLimit: retryLimitRaw } = ctx;
|
|
221
|
+
const resourceGuardCfg = ctx.resourceGuard || {
|
|
222
|
+
enabled: true,
|
|
223
|
+
hrefIncludes: ['/document/', '/file/', '/download', '/attachment/'],
|
|
224
|
+
dataTypes: ['document', 'file', 'item', 'row'],
|
|
225
|
+
};
|
|
226
|
+
const stepsArchive = ctx.stepsArchive || steps;
|
|
227
|
+
const retryLimitClamped = Math.min(10, Math.max(1, Number.isFinite(retryLimitRaw) ? retryLimitRaw : DEFAULT_TURN_RETRY_LIMIT)); // retries allowed (excludes initial)
|
|
228
|
+
const maxAttempts = retryLimitClamped + 1; // includes initial attempt
|
|
175
229
|
ctx.groundControl = groundControl;
|
|
176
230
|
let agentMemory = { lastMenuExpanded: false };
|
|
231
|
+
ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
232
|
+
let turnRetries = 0;
|
|
233
|
+
let stepSeq = ctx._stepSeq || 0;
|
|
177
234
|
|
|
178
235
|
// Centralized recorder: push to in-memory buffer AND fire optional callback for streaming
|
|
179
236
|
// Idempotent recorder ā prevents accidental double-push
|
|
180
237
|
const recordStep = (step) => {
|
|
181
238
|
if (!step || step.__recorded) return;
|
|
239
|
+
step._seq = stepSeq++;
|
|
182
240
|
step.__recorded = true;
|
|
183
241
|
try { steps.push(step); } catch {}
|
|
242
|
+
if (stepsArchive && stepsArchive !== steps) {
|
|
243
|
+
try { stepsArchive.push(step); } catch {}
|
|
244
|
+
}
|
|
184
245
|
try { ctx?.onStep?.(step); } catch {}
|
|
185
246
|
};
|
|
186
247
|
|
|
@@ -190,7 +251,19 @@ export const turnLoop = async (
|
|
|
190
251
|
for (let turn = currentTurn; turn < maxTurns; turn++) {
|
|
191
252
|
console.log(`\nš Turn ${turn + 1}/${maxTurns}`);
|
|
192
253
|
let response;
|
|
193
|
-
const
|
|
254
|
+
const attempt = turnRetries + 1;
|
|
255
|
+
const step = {
|
|
256
|
+
turn,
|
|
257
|
+
retryAttempt: attempt,
|
|
258
|
+
retryLimit: retryLimitClamped, // number of retries allowed (excludes initial)
|
|
259
|
+
events: [],
|
|
260
|
+
result: 'š” In Progress',
|
|
261
|
+
missionName,
|
|
262
|
+
};
|
|
263
|
+
if (attempt > 1) {
|
|
264
|
+
const retryNumber = attempt - 1;
|
|
265
|
+
step.events.push(`š Re-attempt ${retryNumber}/${retryLimitClamped} for turn`);
|
|
266
|
+
}
|
|
194
267
|
|
|
195
268
|
try {
|
|
196
269
|
// Refresh token usage window (rolling 60 seconds)
|
|
@@ -211,7 +284,7 @@ export const turnLoop = async (
|
|
|
211
284
|
step.events.push('ā Tool call structure invalid and no placeholders inserted.');
|
|
212
285
|
step.result = 'ā Failure';
|
|
213
286
|
recordStep(step);
|
|
214
|
-
return steps;
|
|
287
|
+
return { success: false, steps: stepsArchive };
|
|
215
288
|
}
|
|
216
289
|
|
|
217
290
|
// Detect orphaned assistant tool calls with no matching tool responses
|
|
@@ -231,7 +304,7 @@ export const turnLoop = async (
|
|
|
231
304
|
step.events.push('š Detected assistant tool calls without matching tool responses');
|
|
232
305
|
step.result = 'ā Failure';
|
|
233
306
|
recordStep(step);
|
|
234
|
-
return steps;
|
|
307
|
+
return { success: false, steps: stepsArchive };
|
|
235
308
|
}
|
|
236
309
|
|
|
237
310
|
// Before calling the model, trim/sanitize the conversation context.
|
|
@@ -273,7 +346,7 @@ export const turnLoop = async (
|
|
|
273
346
|
step.events.push('ā Too many retries. Exiting.');
|
|
274
347
|
step.result = 'ā Failure';
|
|
275
348
|
recordStep(step);
|
|
276
|
-
return steps;
|
|
349
|
+
return { success: false, steps: stepsArchive };
|
|
277
350
|
}
|
|
278
351
|
retryCount += 1;
|
|
279
352
|
turn -= 1;
|
|
@@ -283,7 +356,7 @@ export const turnLoop = async (
|
|
|
283
356
|
step.events.push(`ā Bad request: ${err.message}`);
|
|
284
357
|
step.result = 'ā Failure';
|
|
285
358
|
recordStep(step);
|
|
286
|
-
return steps;
|
|
359
|
+
return { success: false, steps: stepsArchive };
|
|
287
360
|
} else {
|
|
288
361
|
throw err;
|
|
289
362
|
}
|
|
@@ -319,6 +392,7 @@ export const turnLoop = async (
|
|
|
319
392
|
if (msg.tool_calls?.length) {
|
|
320
393
|
console.log('Processing tool calls...');
|
|
321
394
|
const toolResponses = [];
|
|
395
|
+
let hadToolIssues = false;
|
|
322
396
|
|
|
323
397
|
for (const call of msg.tool_calls) {
|
|
324
398
|
const fnName = call.function.name;
|
|
@@ -335,6 +409,7 @@ export const turnLoop = async (
|
|
|
335
409
|
} catch (e) {
|
|
336
410
|
errorMessage = `ERROR: ${e.message}`;
|
|
337
411
|
result = errorMessage;
|
|
412
|
+
hadToolIssues = true;
|
|
338
413
|
}
|
|
339
414
|
|
|
340
415
|
// Capture and log any file upload/download events (for reports)
|
|
@@ -348,6 +423,20 @@ export const turnLoop = async (
|
|
|
348
423
|
: `š„ Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
|
|
349
424
|
step.events.push(msgLine);
|
|
350
425
|
console.log(msgLine);
|
|
426
|
+
|
|
427
|
+
if (resourceGuardCfg.enabled && (maybeJson._testronaut_file_event === 'download' || maybeJson._testronaut_file_event === 'upload')) {
|
|
428
|
+
const prog = ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
429
|
+
if (prog) {
|
|
430
|
+
const id = extractDocIdFromUrl(maybeJson.url || maybeJson.selector || maybeJson.fileName);
|
|
431
|
+
if (id) prog.downloaded.add(id);
|
|
432
|
+
// also try title match
|
|
433
|
+
if (maybeJson.fileName) {
|
|
434
|
+
const byTitle = prog.items.find(i => maybeJson.fileName.includes(i.title));
|
|
435
|
+
if (byTitle?.id) prog.downloaded.add(byTitle.id);
|
|
436
|
+
}
|
|
437
|
+
step.events.push(`š Resource progress: ${prog.downloaded.size}/${prog.items.length || prog.lastScriptCount}`);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
351
440
|
}
|
|
352
441
|
} catch {
|
|
353
442
|
// non-JSON results ignored
|
|
@@ -388,6 +477,35 @@ export const turnLoop = async (
|
|
|
388
477
|
contentForModel = result;
|
|
389
478
|
}
|
|
390
479
|
|
|
480
|
+
// Keep doc list progress updated when we fetch DOM
|
|
481
|
+
if (!errorMessage && resourceGuardCfg.enabled) {
|
|
482
|
+
const prog = ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
483
|
+
if (prog) {
|
|
484
|
+
if (fnName === 'get_dom') {
|
|
485
|
+
const { items, scriptDocs } = parseDocListFromDom(result);
|
|
486
|
+
if (items.length) {
|
|
487
|
+
prog.items = items;
|
|
488
|
+
prog.lastScriptCount = scriptDocs || items.length;
|
|
489
|
+
prog.lastSummary = `docs:${items.length}`;
|
|
490
|
+
step.events.push(`š Detected document list (${items.length} items)`);
|
|
491
|
+
}
|
|
492
|
+
} else if (fnName === 'list_local_files') {
|
|
493
|
+
try {
|
|
494
|
+
const parsed = JSON.parse(result);
|
|
495
|
+
const files = parsed?.files || [];
|
|
496
|
+
if (Array.isArray(files) && files.length) {
|
|
497
|
+
prog.items = files.map(f => ({ id: f, title: f, href: f }));
|
|
498
|
+
prog.lastScriptCount = files.length;
|
|
499
|
+
prog.lastSummary = `files:${files.length}`;
|
|
500
|
+
step.events.push(`š Detected local files (${files.length} items)`);
|
|
501
|
+
}
|
|
502
|
+
} catch {
|
|
503
|
+
// ignore parse errors
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
391
509
|
toolResponses.push({
|
|
392
510
|
role: 'tool',
|
|
393
511
|
tool_call_id: call.id,
|
|
@@ -416,7 +534,21 @@ export const turnLoop = async (
|
|
|
416
534
|
|
|
417
535
|
// Merge new assistant + tool responses back into conversation
|
|
418
536
|
messages.push(msg, ...toolResponses);
|
|
419
|
-
|
|
537
|
+
if (hadToolIssues && turnRetries < retryLimitClamped) {
|
|
538
|
+
step.result = 'ā³ Retrying turn';
|
|
539
|
+
const retryNumber = attempt - 1;
|
|
540
|
+
step.events.push(`š Re-attempt ${retryNumber}/${retryLimitClamped} after tool issues`);
|
|
541
|
+
recordStep(step);
|
|
542
|
+
turnRetries += 1;
|
|
543
|
+
const delay = Math.min(TURN_RETRY_BASE_DELAY_MS * 2 ** (turnRetries - 1), 2000);
|
|
544
|
+
await wait(delay);
|
|
545
|
+
turn -= 1; // re-use the same turn index
|
|
546
|
+
continue;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// reset retries after a clean turn or after exhausting retries
|
|
550
|
+
turnRetries = 0;
|
|
551
|
+
step.result = hadToolIssues ? 'ā ļø Turn Issues' : 'ā
Passed';
|
|
420
552
|
recordStep(step);
|
|
421
553
|
continue;
|
|
422
554
|
}
|
|
@@ -426,10 +558,29 @@ export const turnLoop = async (
|
|
|
426
558
|
// āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā
|
|
427
559
|
const finalResponse = finalResponseHandler(msg);
|
|
428
560
|
if (finalResponse !== null) {
|
|
561
|
+
const prog = ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
562
|
+
if (prog?.items.length && prog.downloaded.size < prog.items.length) {
|
|
563
|
+
const remaining = prog.items
|
|
564
|
+
.filter(i => !prog.downloaded.has(i.id))
|
|
565
|
+
.map(i => i.title || i.id)
|
|
566
|
+
.slice(0, 10);
|
|
567
|
+
const remainText = remaining.length ? remaining.join('; ') : 'unknown items';
|
|
568
|
+
const guardMsg = `Auto-guard: downloaded ${prog.downloaded.size}/${prog.items.length}. Remaining: ${remainText}`;
|
|
569
|
+
console.log(`[guard] ${guardMsg}`);
|
|
570
|
+
step.events.push(guardMsg);
|
|
571
|
+
step.result = 'š” In Progress';
|
|
572
|
+
recordStep(step);
|
|
573
|
+
// Nudge model to continue
|
|
574
|
+
messages.push({ role: 'assistant', content: guardMsg });
|
|
575
|
+
// Continue loop without exiting
|
|
576
|
+
turnRetries = 0;
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
429
579
|
step.events.push(finalResponse.finalMessage);
|
|
430
580
|
step.result = finalResponse.success ? 'ā
Mission Success' : 'ā Mission Failure';
|
|
431
581
|
recordStep(step);
|
|
432
|
-
|
|
582
|
+
turnRetries = 0;
|
|
583
|
+
return { success: finalResponse.success, finalMessage: finalResponse.finalMessage, steps: stepsArchive };
|
|
433
584
|
}
|
|
434
585
|
|
|
435
586
|
// āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā
|
|
@@ -443,5 +594,15 @@ export const turnLoop = async (
|
|
|
443
594
|
// This is a meaningful turn even without tool calls ā record it.
|
|
444
595
|
step.result = step.result || 'š” In Progress';
|
|
445
596
|
recordStep(step);
|
|
597
|
+
turnRetries = 0;
|
|
446
598
|
}
|
|
599
|
+
|
|
600
|
+
return { success: false, steps: stepsArchive };
|
|
601
|
+
};
|
|
602
|
+
|
|
603
|
+
// Expose small helper bundle for unit tests (no production use).
|
|
604
|
+
export const __docProgressInternals = {
|
|
605
|
+
ensureDocProgress,
|
|
606
|
+
parseDocListFromDom,
|
|
607
|
+
extractDocIdFromUrl,
|
|
447
608
|
};
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.2",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
|
-
"testronaut": "
|
|
6
|
+
"testronaut": "bin/cli.js"
|
|
7
7
|
},
|
|
8
8
|
"exports": {
|
|
9
9
|
".": "./runner/testronaut.js",
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"@google/generative-ai": "^0.24.1",
|
|
25
25
|
"cheerio": "^1.1.0",
|
|
26
26
|
"dotenv": "^17.2.0",
|
|
27
|
+
"fast-glob": "^3.3.3",
|
|
27
28
|
"inquirer": "^12.9.1",
|
|
28
29
|
"node-fetch": "^3.3.2",
|
|
29
30
|
"openai": "^5.10.2",
|
package/runner/testronaut.js
CHANGED
|
@@ -23,9 +23,20 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
import fs from 'fs';
|
|
27
|
+
import path from 'path';
|
|
26
28
|
import { runAgent } from '../core/agent.js';
|
|
27
29
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
28
|
-
import { loadConfig, enforceTurnBudget } from '../core/config.js';
|
|
30
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig } from '../core/config.js';
|
|
31
|
+
|
|
32
|
+
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
33
|
+
const isDebugEnabled = () => {
|
|
34
|
+
const raw = String(process.env.TESTRONAUT_DEBUG || '').trim().toLowerCase();
|
|
35
|
+
return ['1', 'true', 'yes', 'on'].includes(raw);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
// Presentable string for DOM list limit logging/debug output.
|
|
39
|
+
const formatListLimit = (v) => v === Infinity ? 'all' : v;
|
|
29
40
|
|
|
30
41
|
/**
|
|
31
42
|
* Run a mission flow.
|
|
@@ -38,9 +49,58 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
38
49
|
const cfg = await loadConfig();
|
|
39
50
|
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
40
51
|
const maxTurns = effectiveMax;
|
|
52
|
+
const retryInfo = getRetryLimit(cfg);
|
|
53
|
+
const retryLimit = retryInfo.value;
|
|
54
|
+
const domListLimitInfo = getDomListLimit(cfg);
|
|
55
|
+
const resourceGuard = getResourceGuardConfig(cfg);
|
|
56
|
+
const debugEnabled = isDebugEnabled();
|
|
41
57
|
if (notes.length) {
|
|
42
58
|
console.warn(notes.join('\n'));
|
|
43
59
|
}
|
|
60
|
+
if (retryInfo.clamped) {
|
|
61
|
+
console.warn(`ā ļø Retry limit clamped to ${retryLimit} (allowed range 1-10).`);
|
|
62
|
+
}
|
|
63
|
+
if (domListLimitInfo?.clamped) {
|
|
64
|
+
console.warn(`ā ļø DOM list limit clamped to ${domListLimitInfo.value} (allowed 0-100 or "all"/"none").`);
|
|
65
|
+
}
|
|
66
|
+
if (domListLimitInfo?.mode === 'all') {
|
|
67
|
+
console.warn('ā ļø DOM list limit is set to "all". This can dramatically increase token usage and may break flows on heavy pages.');
|
|
68
|
+
}
|
|
69
|
+
if (debugEnabled) {
|
|
70
|
+
const cfgDomRaw = cfg?.dom?.listItemLimit ?? cfg?.dom?.listLimit ?? cfg?.domListLimit;
|
|
71
|
+
console.log('[debug] Debug mode enabled');
|
|
72
|
+
console.log('[debug] Turn limits', {
|
|
73
|
+
maxTurns,
|
|
74
|
+
softMax: limits.softMaxTurns,
|
|
75
|
+
hardMax: limits.hardMaxTurns,
|
|
76
|
+
hardMin: limits.hardMinTurns,
|
|
77
|
+
});
|
|
78
|
+
console.log('[debug] Retry limit', { retryLimit, source: retryInfo.source, clamped: retryInfo.clamped });
|
|
79
|
+
console.log('[debug] DOM list limit', {
|
|
80
|
+
env: process.env.TESTRONAUT_DOM_LIST_LIMIT ?? 'unset',
|
|
81
|
+
config: cfgDomRaw ?? 'unset',
|
|
82
|
+
resolved: formatListLimit(domListLimitInfo?.value),
|
|
83
|
+
mode: domListLimitInfo?.mode,
|
|
84
|
+
source: domListLimitInfo?.source,
|
|
85
|
+
clamped: domListLimitInfo?.clamped,
|
|
86
|
+
});
|
|
87
|
+
console.log('[debug] Resource guard', {
|
|
88
|
+
enabled: resourceGuard.enabled,
|
|
89
|
+
hrefIncludes: resourceGuard.hrefIncludes,
|
|
90
|
+
dataTypes: resourceGuard.dataTypes,
|
|
91
|
+
});
|
|
92
|
+
// Helpful for upload missions: show missions/files contents
|
|
93
|
+
const filesDir = path.resolve(process.cwd(), 'missions/files');
|
|
94
|
+
try {
|
|
95
|
+
const entries = fs.readdirSync(filesDir);
|
|
96
|
+
const pdfs = entries.filter(f => f.toLowerCase().endsWith('.pdf'));
|
|
97
|
+
console.log(`[debug] missions/files dir: ${filesDir}`);
|
|
98
|
+
console.log(`[debug] missions/files entries (${entries.length}):`, entries);
|
|
99
|
+
if (pdfs.length) console.log(`[debug] missions/files PDFs (${pdfs.length}):`, pdfs);
|
|
100
|
+
} catch (err) {
|
|
101
|
+
console.log(`[debug] missions/files not readable: ${err.message}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
44
104
|
|
|
45
105
|
// 2) Normalize submissions
|
|
46
106
|
const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
|
|
@@ -123,7 +183,13 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
123
183
|
);
|
|
124
184
|
|
|
125
185
|
// 3) Execute
|
|
126
|
-
const success = await runAgent(
|
|
186
|
+
const success = await runAgent(
|
|
187
|
+
goals,
|
|
188
|
+
missionName,
|
|
189
|
+
maxTurns,
|
|
190
|
+
retryLimit,
|
|
191
|
+
{ domListLimit: domListLimitInfo?.value, debug: debugEnabled, resourceGuard }
|
|
192
|
+
);
|
|
127
193
|
if (!success) {
|
|
128
194
|
console.log(`ā Aborting after failed goal.`);
|
|
129
195
|
return;
|
|
@@ -136,3 +202,6 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
136
202
|
console.log('\nā
Mission flow complete.');
|
|
137
203
|
return success;
|
|
138
204
|
}
|
|
205
|
+
|
|
206
|
+
// Exposed for unit tests.
|
|
207
|
+
export const __test__ = { isDebugEnabled, formatListLimit };
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
|
|
5
|
+
// Import helper exports from cli.js
|
|
6
|
+
import { __test__ } from '../../bin/cli.js';
|
|
7
|
+
|
|
8
|
+
describe('cli helpers', () => {
|
|
9
|
+
it('guessMimeType returns expected types', () => {
|
|
10
|
+
const { guessMimeType } = __test__;
|
|
11
|
+
expect(guessMimeType('file.html')).toContain('text/html');
|
|
12
|
+
expect(guessMimeType('file.json')).toContain('application/json');
|
|
13
|
+
expect(guessMimeType('file.unknown')).toBe('application/octet-stream');
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('safeJoin prevents traversal', () => {
|
|
17
|
+
const { safeJoin } = __test__;
|
|
18
|
+
const root = '/tmp/root';
|
|
19
|
+
expect(safeJoin(root, '/sub/ok.txt')).toBe(path.resolve('/tmp/root/sub/ok.txt'));
|
|
20
|
+
expect(safeJoin(root, '/../etc/passwd')).toBeNull();
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it('findLatestReportPair finds latest html/json', () => {
|
|
24
|
+
const { findLatestReportPair } = __test__;
|
|
25
|
+
const tmp = fs.mkdtempSync(path.join(process.cwd(), 'testronaut-report-'));
|
|
26
|
+
const r1 = path.join(tmp, 'run_100.html');
|
|
27
|
+
const r2 = path.join(tmp, 'run_200.html');
|
|
28
|
+
fs.writeFileSync(r1, 'a');
|
|
29
|
+
fs.writeFileSync(r2, 'b');
|
|
30
|
+
fs.writeFileSync(path.join(tmp, 'run_200.json'), '{}');
|
|
31
|
+
|
|
32
|
+
const latest = findLatestReportPair(tmp);
|
|
33
|
+
expect(latest?.htmlFile).toBe('run_200.html');
|
|
34
|
+
expect(latest?.jsonFile).toBe('run_200.json');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('parses booleans in a tolerant way', () => {
|
|
38
|
+
const { parseBool } = __test__;
|
|
39
|
+
expect(parseBool('true')).toBe(true);
|
|
40
|
+
expect(parseBool('0')).toBe(false);
|
|
41
|
+
expect(parseBool('off')).toBe(false);
|
|
42
|
+
expect(parseBool('')).toBeNull();
|
|
43
|
+
expect(parseBool('maybe')).toBeNull();
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -77,6 +77,8 @@ describe('initializeTestronautProject', () => {
|
|
|
77
77
|
expect(cfg.initialized).toBe(true);
|
|
78
78
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
79
79
|
expect(typeof cfg.maxTurns).toBe('number');
|
|
80
|
+
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
81
|
+
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
80
82
|
|
|
81
83
|
// .env
|
|
82
84
|
const envPath = path.join(temp, '.env');
|
|
@@ -108,6 +110,8 @@ describe('initializeTestronautProject', () => {
|
|
|
108
110
|
const cfg = readJson(path.join(temp, 'testronaut-config.json'));
|
|
109
111
|
expect(cfg.provider).toBe('gemini');
|
|
110
112
|
expect(cfg.model).toBe('gemini-2.5-flash');
|
|
113
|
+
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
114
|
+
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
111
115
|
|
|
112
116
|
const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
|
|
113
117
|
expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
|
|
@@ -185,5 +189,7 @@ describe('initializeTestronautProject', () => {
|
|
|
185
189
|
expect(cfg.projectName).toBe('my-project'); // preserved
|
|
186
190
|
expect(cfg.outputDir).toBe('missions/mission_reports'); // default filled in
|
|
187
191
|
expect(cfg.maxTurns).toBe(20); // default filled in
|
|
192
|
+
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
193
|
+
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
188
194
|
});
|
|
189
195
|
});
|
|
@@ -8,5 +8,8 @@ describe('defaultConfig', () => {
|
|
|
8
8
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
9
9
|
expect(cfg.projectName).toBe('my-app');
|
|
10
10
|
expect(cfg.maxTurns).toBe(20);
|
|
11
|
+
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
12
|
+
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
13
|
+
expect(cfg.resourceGuard?.hrefIncludes).toContain('/document/');
|
|
11
14
|
});
|
|
12
15
|
});
|
|
@@ -8,8 +8,11 @@ import { readFile } from 'node:fs/promises';
|
|
|
8
8
|
import {
|
|
9
9
|
loadConfig,
|
|
10
10
|
getMaxTurns,
|
|
11
|
+
getRetryLimit,
|
|
11
12
|
resolveTurnLimits,
|
|
12
13
|
enforceTurnBudget,
|
|
14
|
+
getDomListLimit,
|
|
15
|
+
getResourceGuardConfig,
|
|
13
16
|
} from '../../core/config.js';
|
|
14
17
|
|
|
15
18
|
describe('core/config', () => {
|
|
@@ -64,6 +67,140 @@ describe('core/config', () => {
|
|
|
64
67
|
});
|
|
65
68
|
});
|
|
66
69
|
|
|
70
|
+
describe('getRetryLimit', () => {
|
|
71
|
+
const OLD_ENV = { ...process.env };
|
|
72
|
+
beforeEach(() => {
|
|
73
|
+
process.env = { ...OLD_ENV };
|
|
74
|
+
delete process.env.TESTRONAUT_RETRY_LIMIT;
|
|
75
|
+
});
|
|
76
|
+
afterEach(() => {
|
|
77
|
+
process.env = { ...OLD_ENV };
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('returns default when missing', () => {
|
|
81
|
+
const res = getRetryLimit({});
|
|
82
|
+
expect(res.value).toBe(2);
|
|
83
|
+
expect(res.source).toBe('default');
|
|
84
|
+
expect(res.clamped).toBe(false);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('uses config value and clamps to range', () => {
|
|
88
|
+
expect(getRetryLimit({ retryLimit: 5 }).value).toBe(5);
|
|
89
|
+
const high = getRetryLimit({ retryLimit: 99 });
|
|
90
|
+
expect(high.value).toBe(10);
|
|
91
|
+
expect(high.clamped).toBe(true);
|
|
92
|
+
const low = getRetryLimit({ retryLimit: 0 });
|
|
93
|
+
expect(low.value).toBe(1);
|
|
94
|
+
expect(low.clamped).toBe(true);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it('prefers env over config and clamps', () => {
|
|
98
|
+
process.env.TESTRONAUT_RETRY_LIMIT = '8';
|
|
99
|
+
const res = getRetryLimit({ retryLimit: 3 });
|
|
100
|
+
expect(res.value).toBe(8);
|
|
101
|
+
expect(res.source).toBe('env');
|
|
102
|
+
|
|
103
|
+
process.env.TESTRONAUT_RETRY_LIMIT = '0';
|
|
104
|
+
const clampedLow = getRetryLimit({ retryLimit: 3 });
|
|
105
|
+
expect(clampedLow.value).toBe(1);
|
|
106
|
+
expect(clampedLow.clamped).toBe(true);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe('getDomListLimit', () => {
|
|
111
|
+
const OLD_ENV = { ...process.env };
|
|
112
|
+
beforeEach(() => {
|
|
113
|
+
process.env = { ...OLD_ENV };
|
|
114
|
+
delete process.env.TESTRONAUT_DOM_LIST_LIMIT;
|
|
115
|
+
});
|
|
116
|
+
afterEach(() => {
|
|
117
|
+
process.env = { ...OLD_ENV };
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('returns default when missing', () => {
|
|
121
|
+
const res = getDomListLimit({});
|
|
122
|
+
expect(res.value).toBe(3);
|
|
123
|
+
expect(res.mode).toBe('number');
|
|
124
|
+
expect(res.source).toBe('default');
|
|
125
|
+
expect(res.clamped).toBe(false);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('uses config value (nested) and clamps to range', () => {
|
|
129
|
+
const res = getDomListLimit({ dom: { listItemLimit: 250 } });
|
|
130
|
+
expect(res.value).toBe(100);
|
|
131
|
+
expect(res.mode).toBe('number');
|
|
132
|
+
expect(res.source).toBe('config');
|
|
133
|
+
expect(res.clamped).toBe(true);
|
|
134
|
+
|
|
135
|
+
const nestedAlt = getDomListLimit({ dom: { listLimit: '9' } });
|
|
136
|
+
expect(nestedAlt.value).toBe(9);
|
|
137
|
+
expect(nestedAlt.source).toBe('config');
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('prefers env override and accepts keywords', () => {
|
|
141
|
+
process.env.TESTRONAUT_DOM_LIST_LIMIT = 'all';
|
|
142
|
+
const allRes = getDomListLimit({ domListLimit: 2 });
|
|
143
|
+
expect(allRes.value).toBe(Infinity);
|
|
144
|
+
expect(allRes.mode).toBe('all');
|
|
145
|
+
expect(allRes.source).toBe('env');
|
|
146
|
+
|
|
147
|
+
process.env.TESTRONAUT_DOM_LIST_LIMIT = 'none';
|
|
148
|
+
const noneRes = getDomListLimit({ domListLimit: 5 });
|
|
149
|
+
expect(noneRes.value).toBe(0);
|
|
150
|
+
expect(noneRes.mode).toBe('none');
|
|
151
|
+
expect(noneRes.source).toBe('env');
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('accepts numeric strings', () => {
|
|
155
|
+
const res = getDomListLimit({ domListLimit: '7' });
|
|
156
|
+
expect(res.value).toBe(7);
|
|
157
|
+
expect(res.mode).toBe('number');
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
describe('getResourceGuardConfig', () => {
|
|
162
|
+
const OLD_ENV = { ...process.env };
|
|
163
|
+
beforeEach(() => {
|
|
164
|
+
process.env = { ...OLD_ENV };
|
|
165
|
+
delete process.env.TESTRONAUT_RESOURCE_GUARD;
|
|
166
|
+
delete process.env.TESTRONAUT_RESOURCE_HREF_PATTERNS;
|
|
167
|
+
delete process.env.TESTRONAUT_RESOURCE_DATA_TYPES;
|
|
168
|
+
});
|
|
169
|
+
afterEach(() => {
|
|
170
|
+
process.env = { ...OLD_ENV };
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('returns defaults when nothing is set', () => {
|
|
174
|
+
const res = getResourceGuardConfig({});
|
|
175
|
+
expect(res.enabled).toBe(true);
|
|
176
|
+
expect(res.hrefIncludes.length).toBeGreaterThan(0);
|
|
177
|
+
expect(res.dataTypes).toContain('document');
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it('respects env toggles and lists', () => {
|
|
181
|
+
process.env.TESTRONAUT_RESOURCE_GUARD = 'false';
|
|
182
|
+
process.env.TESTRONAUT_RESOURCE_HREF_PATTERNS = '/a,/b';
|
|
183
|
+
process.env.TESTRONAUT_RESOURCE_DATA_TYPES = 'doc,file';
|
|
184
|
+
const res = getResourceGuardConfig({ resourceGuard: { enabled: true } });
|
|
185
|
+
expect(res.enabled).toBe(false);
|
|
186
|
+
expect(res.hrefIncludes).toEqual(['/a', '/b']);
|
|
187
|
+
expect(res.dataTypes).toEqual(['doc', 'file']);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('falls back to config arrays when env missing', () => {
|
|
191
|
+
const res = getResourceGuardConfig({
|
|
192
|
+
resourceGuard: {
|
|
193
|
+
enabled: 'yes',
|
|
194
|
+
hrefIncludes: ['/x', '/y'],
|
|
195
|
+
dataTypes: ['row'],
|
|
196
|
+
},
|
|
197
|
+
});
|
|
198
|
+
expect(res.enabled).toBe(true);
|
|
199
|
+
expect(res.hrefIncludes).toEqual(['/x', '/y']);
|
|
200
|
+
expect(res.dataTypes).toEqual(['row']);
|
|
201
|
+
});
|
|
202
|
+
});
|
|
203
|
+
|
|
67
204
|
describe('enforceTurnBudget (lenient)', () => {
|
|
68
205
|
it('clamps > hardMaxTurns', () => {
|
|
69
206
|
const cfg = { maxTurns: 1000 };
|