testronaut 1.12.0 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/bin/cli.js +17 -3
- package/bin/terminalUi.js +9 -1
- package/core/agent.js +6 -2
- package/core/missionDiscovery.js +56 -6
- package/core/turnLoop.js +58 -1
- package/package.json +1 -1
- package/tests/binTests/cli.integration.test.js +4 -1
- package/tests/binTests/terminalUi.test.js +2 -1
- package/tests/coreTests/agent.test.js +2 -0
- package/tests/coreTests/missionDiscovery.test.js +44 -0
- package/tests/coreTests/turnLoop.test.js +47 -1
- package/tests/toolsTests/generateHtmlReport.test.js +20 -0
- package/tools/generateHtmlReport.js +10 -5
package/README.md
CHANGED
|
@@ -239,6 +239,34 @@ await runMissions({
|
|
|
239
239
|
}, "Contact Form Flow");
|
|
240
240
|
```
|
|
241
241
|
|
|
242
|
+
Control the order of an automatically discovered suite in
|
|
243
|
+
`testronaut-config.json`:
|
|
244
|
+
|
|
245
|
+
```json
|
|
246
|
+
{
|
|
247
|
+
"missions": {
|
|
248
|
+
"include": ["**/*.mission.js"],
|
|
249
|
+
"order": [
|
|
250
|
+
"login.mission.js",
|
|
251
|
+
"account/**/*.mission.js",
|
|
252
|
+
"*",
|
|
253
|
+
"logout.mission.js"
|
|
254
|
+
]
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Order entries are applied from top to bottom. Each pattern's matches remain
|
|
260
|
+
alphabetical, `*` places missions not claimed by another entry, and unlisted
|
|
261
|
+
missions are appended alphabetically. An exact filename that is not discovered
|
|
262
|
+
produces a warning. Without `missions.order`, discovery remains alphabetical.
|
|
263
|
+
Mission filenames supplied directly on the command line run in argument order.
|
|
264
|
+
|
|
265
|
+
`maxTurns` counts working turns. If a mission spends the full budget without a
|
|
266
|
+
verdict, Testronaut makes one additional tool-free closure request so the report
|
|
267
|
+
ends with an explicit success or failure. JSON and HTML reports include elapsed
|
|
268
|
+
milliseconds for each turn and mission plus the duration of the complete suite.
|
|
269
|
+
|
|
242
270
|
---
|
|
243
271
|
|
|
244
272
|
## 🧰 Developer Mode (Staging API)
|
package/bin/cli.js
CHANGED
|
@@ -951,7 +951,12 @@ if (args.includes('serve') || args.includes('view')) {
|
|
|
951
951
|
await new Promise(() => {}); // ✅ never resolves; Ctrl+C will terminate
|
|
952
952
|
}
|
|
953
953
|
|
|
954
|
-
const {
|
|
954
|
+
const {
|
|
955
|
+
root: missionsRoot,
|
|
956
|
+
files: discoveredMissions,
|
|
957
|
+
warnings: missionDiscoveryWarnings = [],
|
|
958
|
+
} = await discoverMissionFiles({ cwd: process.cwd() });
|
|
959
|
+
for (const warning of missionDiscoveryWarnings) console.warn(`⚠️ ${warning}`);
|
|
955
960
|
const tagConfig = await loadConfig(process.cwd());
|
|
956
961
|
const optimizationPolicy = resolveOptimizationPolicy({ config: tagConfig });
|
|
957
962
|
if (!optimizationPolicy.valid) {
|
|
@@ -1144,14 +1149,19 @@ const runFile = async (filePath) => {
|
|
|
1144
1149
|
|
|
1145
1150
|
if (typeof missionsModule.executeMission === 'function') {
|
|
1146
1151
|
if (!jsonOutput) terminalUi.phase('launch', path.relative(process.cwd(), modulePath) || filePath);
|
|
1152
|
+
const missionFileStartTime = Date.now();
|
|
1147
1153
|
const result = await missionsModule.executeMission();
|
|
1154
|
+
const missionFileEndTime = Date.now();
|
|
1148
1155
|
if (result == null) {
|
|
1149
1156
|
console.error(`❌ Mission did not return a result: ${filePath}`);
|
|
1150
1157
|
return false;
|
|
1151
1158
|
}
|
|
1152
1159
|
allResults.push({
|
|
1153
1160
|
file: path.relative(process.cwd(), modulePath) || path.basename(modulePath),
|
|
1154
|
-
result
|
|
1161
|
+
result,
|
|
1162
|
+
startTime: missionFileStartTime,
|
|
1163
|
+
endTime: missionFileEndTime,
|
|
1164
|
+
durationMs: Math.max(0, missionFileEndTime - missionFileStartTime),
|
|
1155
1165
|
});
|
|
1156
1166
|
const missionResults = Array.isArray(result) ? result : [result];
|
|
1157
1167
|
return !missionResults.some(item => item?.status === 'failed');
|
|
@@ -1206,6 +1216,9 @@ const flatMissions = allResults.flatMap(entry => {
|
|
|
1206
1216
|
...m,
|
|
1207
1217
|
steps,
|
|
1208
1218
|
file: entry.file,
|
|
1219
|
+
startTime: m.startTime ?? entry.startTime,
|
|
1220
|
+
endTime: m.endTime ?? entry.endTime,
|
|
1221
|
+
durationMs: m.durationMs ?? entry.durationMs,
|
|
1209
1222
|
};
|
|
1210
1223
|
const jevShadow = summarizeJevUsage(steps);
|
|
1211
1224
|
return jevShadow.evaluations ? { ...mission, jevShadow } : mission;
|
|
@@ -1226,6 +1239,7 @@ const report = {
|
|
|
1226
1239
|
cli: { version: CLI_VERSION },
|
|
1227
1240
|
startTime: startTime.toISOString(),
|
|
1228
1241
|
endTime: endTime.toISOString(),
|
|
1242
|
+
durationMs: Math.max(0, endTime.getTime() - startTime.getTime()),
|
|
1229
1243
|
llm: {
|
|
1230
1244
|
provider: llmProvider,
|
|
1231
1245
|
model: llmModel,
|
|
@@ -1283,7 +1297,7 @@ fs.writeFileSync(path.join(outputDir, `${runId}.json`), JSON.stringify(report, n
|
|
|
1283
1297
|
const htmlReportPath = path.join(outputDir, `${runId}.html`);
|
|
1284
1298
|
generateHtmlReport(report, htmlReportPath);
|
|
1285
1299
|
if (jsonOutput) writeOutput(report);
|
|
1286
|
-
else terminalUi.summary({ ...report.summary, reportPath: path.relative(process.cwd(), htmlReportPath) });
|
|
1300
|
+
else terminalUi.summary({ ...report.summary, durationMs: report.durationMs, reportPath: path.relative(process.cwd(), htmlReportPath) });
|
|
1287
1301
|
|
|
1288
1302
|
try {
|
|
1289
1303
|
if (!process.env.TN_KEEP_TMP && fs.existsSync(TMP_DIR)) {
|
package/bin/terminalUi.js
CHANGED
|
@@ -9,6 +9,12 @@ const ANSI = {
|
|
|
9
9
|
cream: '\u001b[38;5;230m',
|
|
10
10
|
};
|
|
11
11
|
|
|
12
|
+
export function formatDuration(durationMs) {
|
|
13
|
+
if (!Number.isFinite(durationMs)) return null;
|
|
14
|
+
const seconds = Math.max(0, durationMs) / 1000;
|
|
15
|
+
return seconds < 60 ? `${seconds.toFixed(2)}s` : `${Math.floor(seconds / 60)}m ${(seconds % 60).toFixed(1)}s`;
|
|
16
|
+
}
|
|
17
|
+
|
|
12
18
|
export function shouldUseColor(stream = process.stdout, env = process.env) {
|
|
13
19
|
return Boolean(stream?.isTTY) && !('NO_COLOR' in env) && env.TERM !== 'dumb';
|
|
14
20
|
}
|
|
@@ -38,13 +44,15 @@ export function createTerminalUi({
|
|
|
38
44
|
const tagText = tags.length ? paint(ANSI.dim, ` ${tags.map(tag => `#${tag}`).join(' ')}`) : '';
|
|
39
45
|
write(` ${marker} ${file}${tagText}`);
|
|
40
46
|
},
|
|
41
|
-
summary({ totalMissions = 0, passed = 0, failed = 0, reportPath }) {
|
|
47
|
+
summary({ totalMissions = 0, passed = 0, failed = 0, durationMs, reportPath }) {
|
|
42
48
|
const status = failed === 0
|
|
43
49
|
? paint(`${ANSI.bold}${ANSI.teal}`, 'MISSION COMPLETE')
|
|
44
50
|
: paint(`${ANSI.bold}${ANSI.red}`, 'MISSION NEEDS REVIEW');
|
|
45
51
|
write('');
|
|
46
52
|
write(`${label('COMPLETE')}${status}`);
|
|
47
53
|
write(` ${paint(ANSI.teal, `${passed} passed`)} ${paint(failed ? ANSI.red : ANSI.dim, `${failed} failed`)} ${paint(ANSI.dim, `${totalMissions} total`)}`);
|
|
54
|
+
const duration = formatDuration(durationMs);
|
|
55
|
+
if (duration) write(paint(ANSI.dim, ` Duration: ${duration}`));
|
|
48
56
|
if (reportPath) write(paint(ANSI.dim, ` Report: ${reportPath}`));
|
|
49
57
|
},
|
|
50
58
|
};
|
package/core/agent.js
CHANGED
|
@@ -59,6 +59,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
|
|
|
59
59
|
const groundControl = createEmptyGroundControl();
|
|
60
60
|
|
|
61
61
|
for (const goal of goals) {
|
|
62
|
+
const goalStartTime = Date.now();
|
|
62
63
|
const telemetryStart = groundControl.telemetry.length;
|
|
63
64
|
const steps = [];
|
|
64
65
|
const stepsArchive = [];
|
|
@@ -114,6 +115,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
|
|
|
114
115
|
stepFile,
|
|
115
116
|
startTime: now,
|
|
116
117
|
endTime: now,
|
|
118
|
+
durationMs: 0,
|
|
117
119
|
telemetry: [],
|
|
118
120
|
launchProtocol: { id: protocol.id, status: 'reused', probe: 'passed' },
|
|
119
121
|
});
|
|
@@ -289,6 +291,7 @@ Email invitation and magic links:
|
|
|
289
291
|
}
|
|
290
292
|
|
|
291
293
|
const compact = dedupeSteps(stepsArchive.length ? stepsArchive : steps);
|
|
294
|
+
const goalEndTime = Date.now();
|
|
292
295
|
|
|
293
296
|
// Snapshot steps (avoid retaining references to mutable arrays)
|
|
294
297
|
missionResults.push({
|
|
@@ -298,8 +301,9 @@ Email invitation and magic links:
|
|
|
298
301
|
status: result?.success ? 'passed' : 'failed',
|
|
299
302
|
steps: JSON.parse(JSON.stringify(compact)), // last 20 (by memory design)
|
|
300
303
|
stepFile, // full history is in this JSONL (one step per line)
|
|
301
|
-
startTime:
|
|
302
|
-
endTime:
|
|
304
|
+
startTime: goalStartTime,
|
|
305
|
+
endTime: goalEndTime,
|
|
306
|
+
durationMs: Math.max(0, goalEndTime - goalStartTime),
|
|
303
307
|
telemetry: JSON.parse(JSON.stringify(groundControl.telemetry.slice(telemetryStart))),
|
|
304
308
|
...(protocolResult ? { launchProtocol: protocolResult } : {}),
|
|
305
309
|
});
|
package/core/missionDiscovery.js
CHANGED
|
@@ -6,6 +6,54 @@ import { loadConfig } from './config.js';
|
|
|
6
6
|
const DEFAULT_ROOT = 'missions';
|
|
7
7
|
const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
|
|
8
8
|
|
|
9
|
+
function orderMissionFiles(files, order, root) {
|
|
10
|
+
const alphabetical = [...files].sort();
|
|
11
|
+
if (!Array.isArray(order) || order.length === 0) {
|
|
12
|
+
return { files: alphabetical, warnings: [] };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const remaining = new Set(alphabetical);
|
|
16
|
+
const ordered = [];
|
|
17
|
+
const warnings = [];
|
|
18
|
+
|
|
19
|
+
for (let index = 0; index < order.length; index += 1) {
|
|
20
|
+
const rawPattern = order[index];
|
|
21
|
+
const pattern = String(rawPattern ?? '').trim();
|
|
22
|
+
if (!pattern) continue;
|
|
23
|
+
|
|
24
|
+
if (pattern === '*') {
|
|
25
|
+
const reserved = new Set(order.slice(index + 1).flatMap(laterPattern => {
|
|
26
|
+
const later = String(laterPattern ?? '').trim();
|
|
27
|
+
if (!later || later === '*') return [];
|
|
28
|
+
return fg.sync(later, { cwd: root, onlyFiles: true });
|
|
29
|
+
}));
|
|
30
|
+
for (const file of alphabetical) {
|
|
31
|
+
if (!reserved.has(file) && remaining.delete(file)) ordered.push(file);
|
|
32
|
+
}
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const matches = fg.sync(pattern, { cwd: root, onlyFiles: true })
|
|
37
|
+
.filter(file => remaining.has(file))
|
|
38
|
+
.sort();
|
|
39
|
+
for (const file of matches) {
|
|
40
|
+
remaining.delete(file);
|
|
41
|
+
ordered.push(file);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (!fg.isDynamicPattern(pattern) && !alphabetical.includes(pattern)) {
|
|
45
|
+
warnings.push(`Configured mission order entry was not discovered: ${pattern}`);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// A partial order only moves the named groups; all other missions retain
|
|
50
|
+
// deterministic legacy ordering after the configured entries.
|
|
51
|
+
for (const file of alphabetical) {
|
|
52
|
+
if (remaining.delete(file)) ordered.push(file);
|
|
53
|
+
}
|
|
54
|
+
return { files: ordered, warnings };
|
|
55
|
+
}
|
|
56
|
+
|
|
9
57
|
/**
|
|
10
58
|
* Discover mission files based on testronaut-config.json.
|
|
11
59
|
* - When no missions block exists, preserve legacy behavior:
|
|
@@ -14,7 +62,7 @@ const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
|
|
|
14
62
|
*
|
|
15
63
|
* @param {object} opts
|
|
16
64
|
* @param {string} [opts.cwd=process.cwd()] - project root for config + resolution
|
|
17
|
-
* @returns {Promise<{ root:string, files:string[] }>} root is absolute; files are relative to root
|
|
65
|
+
* @returns {Promise<{ root:string, files:string[], warnings:string[] }>} root is absolute; files are relative to root
|
|
18
66
|
*/
|
|
19
67
|
export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
|
|
20
68
|
const defaultRoot = path.resolve(cwd, DEFAULT_ROOT);
|
|
@@ -24,17 +72,17 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
|
|
|
24
72
|
// Legacy behavior remains non-recursive, now with first-class TypeScript support.
|
|
25
73
|
if (!missionsCfg) {
|
|
26
74
|
if (!fs.existsSync(defaultRoot)) {
|
|
27
|
-
return { root: defaultRoot, files: [] };
|
|
75
|
+
return { root: defaultRoot, files: [], warnings: [] };
|
|
28
76
|
}
|
|
29
77
|
const files = fs.readdirSync(defaultRoot)
|
|
30
78
|
.filter(f => f.endsWith('.mission.js') || f.endsWith('.mission.ts'))
|
|
31
79
|
.sort();
|
|
32
|
-
return { root: defaultRoot, files };
|
|
80
|
+
return { root: defaultRoot, files, warnings: [] };
|
|
33
81
|
}
|
|
34
82
|
|
|
35
83
|
const root = path.resolve(cwd, missionsCfg.root || DEFAULT_ROOT);
|
|
36
84
|
if (!fs.existsSync(root)) {
|
|
37
|
-
return { root, files: [] };
|
|
85
|
+
return { root, files: [], warnings: [] };
|
|
38
86
|
}
|
|
39
87
|
|
|
40
88
|
const include = Array.isArray(missionsCfg.include) && missionsCfg.include.length
|
|
@@ -49,7 +97,9 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
|
|
|
49
97
|
});
|
|
50
98
|
|
|
51
99
|
const unique = Array.from(new Set(matches));
|
|
52
|
-
unique.
|
|
100
|
+
const ordered = orderMissionFiles(unique, missionsCfg.order, root);
|
|
53
101
|
|
|
54
|
-
return { root,
|
|
102
|
+
return { root, ...ordered };
|
|
55
103
|
}
|
|
104
|
+
|
|
105
|
+
export const __test__ = { orderMissionFiles };
|
package/core/turnLoop.js
CHANGED
|
@@ -409,6 +409,8 @@ export const turnLoop = async (
|
|
|
409
409
|
// Idempotent recorder – prevents accidental double-push
|
|
410
410
|
const recordStep = (step) => {
|
|
411
411
|
if (!step || step.__recorded) return;
|
|
412
|
+
step.endTime = Date.now();
|
|
413
|
+
step.durationMs = Math.max(0, step.endTime - (step.startTime ?? step.endTime));
|
|
412
414
|
step._seq = stepSeq++;
|
|
413
415
|
step.__recorded = true;
|
|
414
416
|
try { steps.push(step); } catch {}
|
|
@@ -427,6 +429,7 @@ export const turnLoop = async (
|
|
|
427
429
|
const attempt = turnRetries + 1;
|
|
428
430
|
const step = {
|
|
429
431
|
turn,
|
|
432
|
+
startTime: Date.now(),
|
|
430
433
|
retryAttempt: attempt,
|
|
431
434
|
retryLimit: retryLimitClamped, // number of retries allowed (excludes initial)
|
|
432
435
|
events: [],
|
|
@@ -1061,7 +1064,61 @@ export const turnLoop = async (
|
|
|
1061
1064
|
turnRetries = 0;
|
|
1062
1065
|
}
|
|
1063
1066
|
|
|
1064
|
-
|
|
1067
|
+
// The configured budget covers working turns. Once those are consumed, make
|
|
1068
|
+
// one final tool-free request so every mission ends with an explicit verdict.
|
|
1069
|
+
const closureStep = {
|
|
1070
|
+
turn: maxTurns,
|
|
1071
|
+
startTime: Date.now(),
|
|
1072
|
+
isClosureTurn: true,
|
|
1073
|
+
retryAttempt: 1,
|
|
1074
|
+
retryLimit: 0,
|
|
1075
|
+
events: ['⏱️ Working-turn budget exhausted; requesting final mission verdict.'],
|
|
1076
|
+
result: '🟡 Awaiting Closure',
|
|
1077
|
+
missionName,
|
|
1078
|
+
provider: PROVIDER_ID,
|
|
1079
|
+
model: MODEL_ID,
|
|
1080
|
+
};
|
|
1081
|
+
console.log(`\n🏁 Turn budget exhausted after ${maxTurns} working turn(s). Requesting closure.`);
|
|
1082
|
+
messages.push({
|
|
1083
|
+
role: 'user',
|
|
1084
|
+
content: 'The working-turn budget is exhausted. Do not call tools or continue the task. Based only on the evidence already collected, respond with exactly SUCCESS: followed by a short reason if the goal was achieved, otherwise FAILURE: followed by a short reason.',
|
|
1085
|
+
});
|
|
1086
|
+
|
|
1087
|
+
try {
|
|
1088
|
+
const closureResponse = await llm.chat({ model: MODEL_ID, messages, tools: [] });
|
|
1089
|
+
const msg = closureResponse?.message ?? { role: 'assistant', content: '' };
|
|
1090
|
+
const usage = closureResponse?.usage;
|
|
1091
|
+
if (usage) {
|
|
1092
|
+
closureStep.tokensUsed = usage.total_tokens || 0;
|
|
1093
|
+
closureStep.inputTokens = usage.input_tokens ?? usage.prompt_tokens ?? usage.providerRaw?.input_tokens ?? usage.providerRaw?.prompt_tokens;
|
|
1094
|
+
closureStep.outputTokens = usage.output_tokens ?? usage.completion_tokens ?? usage.providerRaw?.output_tokens ?? usage.providerRaw?.completion_tokens;
|
|
1095
|
+
}
|
|
1096
|
+
closureStep.summary = summarizeTurnIntentFromMessage(msg, { emoji: true });
|
|
1097
|
+
const finalResponse = finalResponseHandler(msg);
|
|
1098
|
+
if (finalResponse) {
|
|
1099
|
+
closureStep.events.push(finalResponse.finalMessage);
|
|
1100
|
+
closureStep.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
|
|
1101
|
+
recordStep(closureStep);
|
|
1102
|
+
return {
|
|
1103
|
+
success: finalResponse.success,
|
|
1104
|
+
finalMessage: finalResponse.finalMessage,
|
|
1105
|
+
steps: stepsArchive,
|
|
1106
|
+
completedBy: 'budget_closure',
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
closureStep.events.push('FAILURE: The closure response did not provide an explicit SUCCESS or FAILURE verdict.');
|
|
1110
|
+
} catch (error) {
|
|
1111
|
+
closureStep.events.push(`FAILURE: Unable to obtain the closure verdict: ${error?.message || error}`);
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
closureStep.result = '❌ Mission Failure';
|
|
1115
|
+
recordStep(closureStep);
|
|
1116
|
+
return {
|
|
1117
|
+
success: false,
|
|
1118
|
+
finalMessage: closureStep.events.at(-1),
|
|
1119
|
+
steps: stepsArchive,
|
|
1120
|
+
completedBy: 'budget_closure',
|
|
1121
|
+
};
|
|
1065
1122
|
};
|
|
1066
1123
|
|
|
1067
1124
|
// Expose small helper bundle for unit tests (no production use).
|
package/package.json
CHANGED
|
@@ -98,7 +98,10 @@ describe('CLI inspection commands', () => {
|
|
|
98
98
|
const result = run(cwd, ['--tags', 'authentication', '--json', '--quiet']);
|
|
99
99
|
|
|
100
100
|
expect(result.status).toBe(0);
|
|
101
|
-
|
|
101
|
+
const report = JSON.parse(result.stdout);
|
|
102
|
+
expect(report.missions[0].missionName).toBe('https://example.test');
|
|
103
|
+
expect(report.durationMs).toEqual(expect.any(Number));
|
|
104
|
+
expect(report.missions[0].durationMs).toEqual(expect.any(Number));
|
|
102
105
|
});
|
|
103
106
|
|
|
104
107
|
it('records shared-auth session mode in the report', () => {
|
|
@@ -17,12 +17,13 @@ describe('terminal UI', () => {
|
|
|
17
17
|
ui.banner({ version: '1.2.3', provider: 'openai', model: 'gpt-test', session: 'isolated' });
|
|
18
18
|
ui.phase('launch', 'login.mission.js');
|
|
19
19
|
ui.mission(true, 'login.mission.js', ['smoke']);
|
|
20
|
-
ui.summary({ totalMissions: 1, passed: 1, failed: 0, reportPath: 'reports/run.html' });
|
|
20
|
+
ui.summary({ totalMissions: 1, passed: 1, failed: 0, durationMs: 1234, reportPath: 'reports/run.html' });
|
|
21
21
|
|
|
22
22
|
expect(target.read()).toContain('🧑🚀 TESTRONAUT / MISSION RUNNER');
|
|
23
23
|
expect(target.read()).toContain('LAUNCH login.mission.js');
|
|
24
24
|
expect(target.read()).toContain('#smoke');
|
|
25
25
|
expect(target.read()).toContain('MISSION COMPLETE');
|
|
26
|
+
expect(target.read()).toContain('Duration: 1.23s');
|
|
26
27
|
expect(target.read()).not.toContain('\u001b[');
|
|
27
28
|
});
|
|
28
29
|
|
|
@@ -61,6 +61,8 @@ describe('core/agent.runAgent', () => {
|
|
|
61
61
|
expect(res).toHaveLength(2);
|
|
62
62
|
expect(res[0].status).toBe('passed');
|
|
63
63
|
expect(res[1].status).toBe('passed');
|
|
64
|
+
expect(res[0].durationMs).toEqual(expect.any(Number));
|
|
65
|
+
expect(res[0].endTime).toBeGreaterThanOrEqual(res[0].startTime);
|
|
64
66
|
});
|
|
65
67
|
|
|
66
68
|
it('stops early on first failure and returns partial results', async () => {
|
|
@@ -78,4 +78,48 @@ describe('discoverMissionFiles', () => {
|
|
|
78
78
|
const res = await discoverMissionFiles({ cwd: tmp });
|
|
79
79
|
expect(res.files).toEqual(['keep.mission.js']);
|
|
80
80
|
});
|
|
81
|
+
|
|
82
|
+
it('orders configured patterns and places remaining missions at the wildcard', async () => {
|
|
83
|
+
const tmp = makeTempDir();
|
|
84
|
+
temps.push(tmp);
|
|
85
|
+
const missionsDir = path.join(tmp, 'missions');
|
|
86
|
+
fs.mkdirSync(path.join(missionsDir, 'account'), { recursive: true });
|
|
87
|
+
for (const file of ['login.mission.js', 'logout.mission.js', 'search.mission.js']) {
|
|
88
|
+
fs.writeFileSync(path.join(missionsDir, file), '// ok');
|
|
89
|
+
}
|
|
90
|
+
fs.writeFileSync(path.join(missionsDir, 'account', 'profile.mission.js'), '// ok');
|
|
91
|
+
|
|
92
|
+
writeJson(path.join(tmp, 'testronaut-config.json'), {
|
|
93
|
+
missions: {
|
|
94
|
+
include: ['**/*.mission.js'],
|
|
95
|
+
order: ['login.mission.js', 'account/**/*.mission.js', '*', 'logout.mission.js'],
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
const res = await discoverMissionFiles({ cwd: tmp });
|
|
100
|
+
expect(res.files).toEqual([
|
|
101
|
+
'login.mission.js',
|
|
102
|
+
'account/profile.mission.js',
|
|
103
|
+
'search.mission.js',
|
|
104
|
+
'logout.mission.js',
|
|
105
|
+
]);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('warns about missing exact order entries while ignoring unmatched globs', async () => {
|
|
109
|
+
const tmp = makeTempDir();
|
|
110
|
+
temps.push(tmp);
|
|
111
|
+
fs.mkdirSync(path.join(tmp, 'missions'));
|
|
112
|
+
fs.writeFileSync(path.join(tmp, 'missions', 'actual.mission.js'), '// ok');
|
|
113
|
+
writeJson(path.join(tmp, 'testronaut-config.json'), {
|
|
114
|
+
missions: {
|
|
115
|
+
order: ['missing.mission.js', 'optional/**/*.mission.js', '*'],
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
const res = await discoverMissionFiles({ cwd: tmp });
|
|
120
|
+
expect(res.files).toEqual(['actual.mission.js']);
|
|
121
|
+
expect(res.warnings).toEqual([
|
|
122
|
+
'Configured mission order entry was not discovered: missing.mission.js',
|
|
123
|
+
]);
|
|
124
|
+
});
|
|
81
125
|
});
|
|
@@ -96,7 +96,7 @@ vi.mock('../../tools/turnLoopUtils.js', async () => {
|
|
|
96
96
|
finalResponseHandler: (msg) => {
|
|
97
97
|
const content = (msg?.content || '').toString();
|
|
98
98
|
if (content.includes('FINAL')) {
|
|
99
|
-
return { finalMessage:
|
|
99
|
+
return { finalMessage: content, success: !content.includes('FAILURE') };
|
|
100
100
|
}
|
|
101
101
|
return null;
|
|
102
102
|
},
|
|
@@ -182,6 +182,52 @@ describe('turnLoop', () => {
|
|
|
182
182
|
expect(shared.chromeToolSpies.get_dom).not.toHaveBeenCalled();
|
|
183
183
|
});
|
|
184
184
|
|
|
185
|
+
it('uses one tool-free closure turn after the working-turn budget is exhausted', async () => {
|
|
186
|
+
shared.chatMock
|
|
187
|
+
.mockResolvedValueOnce({
|
|
188
|
+
message: { role: 'assistant', content: 'Still investigating' },
|
|
189
|
+
usage: { total_tokens: 10 },
|
|
190
|
+
headers: {},
|
|
191
|
+
})
|
|
192
|
+
.mockResolvedValueOnce({
|
|
193
|
+
message: { role: 'assistant', content: 'FINAL: FAILURE: budget exhausted before completion' },
|
|
194
|
+
usage: { total_tokens: 4 },
|
|
195
|
+
headers: {},
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
const result = await turnLoop(browser, baseMessages(), 1, 0, 0, {}, {
|
|
199
|
+
steps: [], missionName: 'demo',
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
expect(result).toMatchObject({ success: false, completedBy: 'budget_closure' });
|
|
203
|
+
expect(shared.chatMock).toHaveBeenCalledTimes(2);
|
|
204
|
+
expect(shared.chatMock.mock.calls[1][0].tools).toEqual([]);
|
|
205
|
+
expect(shared.chatMock.mock.calls[1][0].messages.at(-1).content).toContain('working-turn budget is exhausted');
|
|
206
|
+
expect(result.steps.at(-1)).toMatchObject({
|
|
207
|
+
isClosureTurn: true,
|
|
208
|
+
result: '❌ Mission Failure',
|
|
209
|
+
tokensUsed: 4,
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it('records elapsed milliseconds for normal and closure turns', async () => {
|
|
214
|
+
shared.chatMock
|
|
215
|
+
.mockResolvedValueOnce({ message: { role: 'assistant', content: 'Continue' }, usage: {} })
|
|
216
|
+
.mockResolvedValueOnce({ message: { role: 'assistant', content: 'FINAL: FAILURE: incomplete' }, usage: {} });
|
|
217
|
+
|
|
218
|
+
const result = await turnLoop(browser, baseMessages(), 1, 0, 0, {}, {
|
|
219
|
+
steps: [], missionName: 'timed',
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
expect(result.steps).toHaveLength(2);
|
|
223
|
+
for (const step of result.steps) {
|
|
224
|
+
expect(step.startTime).toEqual(expect.any(Number));
|
|
225
|
+
expect(step.endTime).toEqual(expect.any(Number));
|
|
226
|
+
expect(step.durationMs).toEqual(expect.any(Number));
|
|
227
|
+
expect(step.durationMs).toBeGreaterThanOrEqual(0);
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
|
|
185
231
|
it('records Jev shadow telemetry without changing the final result', async () => {
|
|
186
232
|
process.env.TESTRONAUT_JEV_SHADOW = '1';
|
|
187
233
|
process.env.TESTRONAUT_JEV_STRATEGIES = '1';
|
|
@@ -18,6 +18,26 @@ afterEach(() => {
|
|
|
18
18
|
});
|
|
19
19
|
|
|
20
20
|
describe('generateHtmlReport', () => {
|
|
21
|
+
it('renders explicit suite, mission, and turn durations', () => {
|
|
22
|
+
const report = {
|
|
23
|
+
runId: 'timed_run',
|
|
24
|
+
durationMs: 6500,
|
|
25
|
+
missions: [{
|
|
26
|
+
missionName: 'Timed Mission',
|
|
27
|
+
submissionType: 'mission',
|
|
28
|
+
status: 'passed',
|
|
29
|
+
durationMs: 2500,
|
|
30
|
+
steps: [{ turn: 0, result: '✅ Mission Success', durationMs: 1250 }],
|
|
31
|
+
}],
|
|
32
|
+
};
|
|
33
|
+
const outPath = path.join(tmpDir, 'timed.html');
|
|
34
|
+
generateHtmlReport(report, outPath);
|
|
35
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
36
|
+
expect(html).toContain('<strong>Duration:</strong> 6.50s');
|
|
37
|
+
expect(html).toContain('duration: 2.50s');
|
|
38
|
+
expect(html).toContain('Turn 1 • 1.25s');
|
|
39
|
+
});
|
|
40
|
+
|
|
21
41
|
it('writes an HTML report containing mission and retry metadata', () => {
|
|
22
42
|
const report = {
|
|
23
43
|
runId: 'run_test',
|
|
@@ -30,8 +30,9 @@ import { summarizeJevUsage } from '../core/jevUsage.js';
|
|
|
30
30
|
export function generateHtmlReport(report, outputPath) {
|
|
31
31
|
const { runId, startTime, endTime, missions = [], summary = {}, llm = {}, cli = {}, optimization } = report;
|
|
32
32
|
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
33
|
-
const durationSec =
|
|
34
|
-
(
|
|
33
|
+
const durationSec = Number.isFinite(report.durationMs)
|
|
34
|
+
? (report.durationMs / 1000).toFixed(2)
|
|
35
|
+
: (startTime && endTime)
|
|
35
36
|
? ((new Date(endTime) - new Date(startTime)) / 1000).toFixed(2)
|
|
36
37
|
: '—';
|
|
37
38
|
|
|
@@ -91,8 +92,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
91
92
|
const jevLogo = '<svg viewBox="0 0 24 24" role="img" aria-hidden="true"><rect width="24" height="24" rx="4" fill="#e547b5"/><g fill="none" stroke="#181a1b" stroke-width="2.25" stroke-linecap="square" stroke-linejoin="round"><path d="M12 2.5 6.1 6v10.1l3.2 1.9 2.7-1.6V9.6l3-1.8V4.3z"/><path d="m9.3 9.6 2.7 1.6 3-1.8 3 1.8v7L12 21.5l-3-1.8V13z"/><path d="m12 11.2 3 1.8 3-1.8M15 13v5.2l-6 3.3M9.3 9.6 12 8l3 1.8"/></g></svg>';
|
|
92
93
|
|
|
93
94
|
const submissionBlock = (m) => {
|
|
94
|
-
const mDurationSec =
|
|
95
|
-
m.
|
|
95
|
+
const mDurationSec = Number.isFinite(m.durationMs)
|
|
96
|
+
? (m.durationMs / 1000).toFixed(2)
|
|
97
|
+
: m.endTime && m.startTime
|
|
96
98
|
? ((m.endTime - m.startTime) / 1000).toFixed(2)
|
|
97
99
|
: (typeof m.durationSeconds === 'number' ? m.durationSeconds.toFixed(2) : '—');
|
|
98
100
|
|
|
@@ -129,6 +131,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
129
131
|
: step.modelRouting?.applied
|
|
130
132
|
? ' • routed by Jev'
|
|
131
133
|
: '';
|
|
134
|
+
const stepDuration = Number.isFinite(step.durationMs)
|
|
135
|
+
? ` • ${(step.durationMs / 1000).toFixed(2)}s`
|
|
136
|
+
: '';
|
|
132
137
|
|
|
133
138
|
const plan = (typeof step.summary === 'string' && step.summary.trim())
|
|
134
139
|
? step.summary.trim()
|
|
@@ -140,7 +145,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
140
145
|
return `
|
|
141
146
|
<details class="step" ${ok ? '' : 'open'}>
|
|
142
147
|
<summary>
|
|
143
|
-
<span class="turn"
|
|
148
|
+
<span class="turn">${step.isClosureTurn ? 'Closure' : `Turn ${esc((step.turn ?? idx) + 1)}`}${esc(attemptLabel)}${stepDuration}</span>
|
|
144
149
|
${humanInput}
|
|
145
150
|
${planSpan}
|
|
146
151
|
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|