testronaut 1.0.8 ā 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/agent.js +47 -57
- package/core/turnLoop.js +75 -105
- package/package.json +9 -2
- package/runner/missionLaunch.js +1 -1
- package/runner/testronaut.js +9 -36
- package/tools/chromeBrowser.js +10 -12
- package/tools/generateHtmlReport.js +109 -39
- package/tools/toolSchema.js +18 -0
- package/tools/turnLoopUtils.js +2 -2
package/core/agent.js
CHANGED
|
@@ -2,41 +2,17 @@ import 'dotenv/config';
|
|
|
2
2
|
import { ChromeBrowser } from '../tools/chromeBrowser.js';
|
|
3
3
|
import { turnLoop } from './turnLoop.js';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
// export async function runAgent(goal, maxTurns = 10) {
|
|
7
|
-
// console.log(`\nš Running Agent with goal: ${goal}`);
|
|
8
|
-
// const browser = new ChromeBrowser();
|
|
9
|
-
// await browser.start();
|
|
10
|
-
|
|
11
|
-
// const messages = [
|
|
12
|
-
// {
|
|
13
|
-
// role: 'system',
|
|
14
|
-
// content: `
|
|
15
|
-
// You are an autonomous web agent. Use function calls to complete the user's goal.
|
|
16
|
-
// If you are unsure of the selectors for inputs or buttons, call 'get_dom' to retrieve page HTML,
|
|
17
|
-
// analyze it, then make your best guess based on labels, names, types, and placeholder values.
|
|
18
|
-
// After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
|
|
19
|
-
// `.trim(),
|
|
20
|
-
// },
|
|
21
|
-
// { role: 'user', content: goal },
|
|
22
|
-
// ];
|
|
23
|
-
|
|
24
|
-
// try {
|
|
25
|
-
// const result = await turnLoop(browser, messages, maxTurns);
|
|
26
|
-
// !result && console.log('š Agent ran out of turns.')
|
|
27
|
-
// } finally {
|
|
28
|
-
// await browser.close();
|
|
29
|
-
// }
|
|
30
|
-
// }
|
|
31
|
-
|
|
32
5
|
export async function runAgent(goals, missionName, maxTurns = 20, ) {
|
|
33
6
|
const browser = new ChromeBrowser();
|
|
34
7
|
await browser.start();
|
|
35
8
|
const startTime = Date.now();
|
|
9
|
+
let result;
|
|
36
10
|
|
|
37
11
|
try {
|
|
12
|
+
const missionResults = [];
|
|
38
13
|
for (const goal of goals) {
|
|
39
14
|
// console.log(`\nš Running Agent with goal:\n${goal}\n`);
|
|
15
|
+
const steps = [];
|
|
40
16
|
const messages = [
|
|
41
17
|
{
|
|
42
18
|
role: 'system',
|
|
@@ -56,43 +32,57 @@ export async function runAgent(goals, missionName, maxTurns = 20, ) {
|
|
|
56
32
|
{ role: 'user', content: goal.goal },
|
|
57
33
|
];
|
|
58
34
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
35
|
+
result = await turnLoop(
|
|
36
|
+
browser,
|
|
37
|
+
messages,
|
|
38
|
+
maxTurns,
|
|
39
|
+
0, /* currentTurn */
|
|
40
|
+
0, /* retryCount */
|
|
41
|
+
null, /* currentStep */
|
|
42
|
+
{
|
|
43
|
+
steps, // š pass in
|
|
44
|
+
missionName, // š pass in (for tagging)
|
|
69
45
|
}
|
|
70
|
-
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
missionResults.push({
|
|
49
|
+
missionName,
|
|
50
|
+
status: result.success ? 'passed' : 'failed',
|
|
51
|
+
steps: JSON.parse(JSON.stringify(steps)), // š deep-clone to avoid refs
|
|
52
|
+
startTime: Date.now() - 1, // set real values if you have them
|
|
53
|
+
endTime: Date.now(),
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
if (!result.success) {
|
|
57
|
+
console.log('š Agent stopped due to failed goal.\n');
|
|
58
|
+
return missionResults;
|
|
71
59
|
}
|
|
60
|
+
|
|
61
|
+
// if (!result.success) {
|
|
62
|
+
// console.log('š Agent stopped due to failed goal.\n');
|
|
63
|
+
// const endTime = Date.now();
|
|
64
|
+
// return {
|
|
65
|
+
// missionName: missionName,
|
|
66
|
+
// status: 'failed',
|
|
67
|
+
// steps: result.steps,
|
|
68
|
+
// startTime: startTime,
|
|
69
|
+
// endTime: endTime,
|
|
70
|
+
// };
|
|
71
|
+
// }
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
console.log('ā
All goals completed successfully.\n');
|
|
75
|
-
const endTime = Date.now();
|
|
76
|
-
return
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
;
|
|
75
|
+
// const endTime = Date.now();
|
|
76
|
+
return missionResults;
|
|
77
|
+
// return {
|
|
78
|
+
// missionName: missionName,
|
|
79
|
+
// status: 'passed',
|
|
80
|
+
// steps: result.steps,
|
|
81
|
+
// startTime: startTime,
|
|
82
|
+
// endTime: endTime,
|
|
83
|
+
// };
|
|
84
84
|
|
|
85
85
|
} finally {
|
|
86
86
|
await browser.close();
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
// (async () => {
|
|
92
|
-
// await runAgent(`
|
|
93
|
-
// Visit ${process.env.URL}.
|
|
94
|
-
// Log in using ${process.env.USERNAME} and password ${process.env.PASSWORD}.
|
|
95
|
-
// After clicking the login button, use 'check_text' to verify that "${process.env.AFTER_LOGIN_CHECK}" appears on the page.
|
|
96
|
-
// If found, report SUCCESS. Otherwise, report FAILURE.
|
|
97
|
-
// `);
|
|
98
|
-
// })();
|
package/core/turnLoop.js
CHANGED
|
@@ -18,6 +18,8 @@ const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
|
18
18
|
|
|
19
19
|
let totalTokensUsed = 0;
|
|
20
20
|
let turnTimestamps = [];
|
|
21
|
+
let shouldBackoff;
|
|
22
|
+
let steps = [];
|
|
21
23
|
|
|
22
24
|
const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool } = {}) => {
|
|
23
25
|
if (skipIfLastTool && skipIfLastTool.includes(messages.at(-1)?.name)) {
|
|
@@ -54,55 +56,43 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
54
56
|
});
|
|
55
57
|
}
|
|
56
58
|
|
|
57
|
-
export const turnLoop = async (
|
|
59
|
+
export const turnLoop = async (
|
|
60
|
+
browser,
|
|
61
|
+
messages,
|
|
62
|
+
maxTurns,
|
|
63
|
+
currentTurn = 0,
|
|
64
|
+
retryCount = 0,
|
|
65
|
+
currentStep = {},
|
|
66
|
+
ctx = {} // š { steps, missionName }
|
|
67
|
+
) => {
|
|
68
|
+
const { steps = [], missionName } = ctx;
|
|
58
69
|
let agentMemory = {
|
|
59
70
|
lastMenuExpanded: false
|
|
60
71
|
};
|
|
61
72
|
|
|
62
73
|
for (let turn = currentTurn; turn < maxTurns; turn++) {
|
|
63
74
|
console.log(`\nš Turn ${turn + 1}/${maxTurns}`);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
75
|
+
let response;
|
|
76
|
+
// const currentStep = { turn, events: [], result: 'š” In Progress' };
|
|
77
|
+
const currentStep = { turn, events: [], result: 'š” In Progress', missionName }; // š tag it
|
|
78
|
+
try {
|
|
68
79
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
turnTimestamps = pruned.turnTimestamps;
|
|
72
|
-
totalTokensUsed = pruned.totalTokensUsed;
|
|
80
|
+
// Recalculate the rolling window before checking cooldown
|
|
81
|
+
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
73
82
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
turnTimestamps = cooldownResult.turnTimestamps;
|
|
78
|
-
if (cooldownResult.shouldBackoff) {
|
|
79
|
-
return await turnLoop(browser, messages, maxTurns, turn);
|
|
80
|
-
}
|
|
81
|
-
// if (await tokenUseCoolOff(totalTokensUsed, turnTimestamps)) {return await turnLoop(browser, messages, maxTurns, turn)}
|
|
83
|
+
// Cool off check
|
|
84
|
+
({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(totalTokensUsed, turnTimestamps));
|
|
85
|
+
if(shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
|
|
82
86
|
|
|
83
|
-
|
|
84
|
-
insertPlaceholders: true
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
// console.log(JSON.stringify(messages, null, 2));
|
|
87
|
+
//Tool Validator
|
|
88
|
+
if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
|
|
89
|
+
console.error('ā Tool call structure invalid and no placeholders inserted.');
|
|
90
|
+
currentStep.events.push('ā Tool call structure invalid and no placeholders inserted.')
|
|
91
|
+
currentStep.result = 'ā Failure';
|
|
92
|
+
steps.push(currentStep);
|
|
93
|
+
return steps;
|
|
94
|
+
}
|
|
92
95
|
|
|
93
|
-
let response;
|
|
94
|
-
try {
|
|
95
|
-
// console.log(
|
|
96
|
-
// 'š Verifying message structure before OpenAI call:\n',
|
|
97
|
-
// messages.map(m => ({
|
|
98
|
-
// role: m.role,
|
|
99
|
-
// name: m.name,
|
|
100
|
-
// tool_call_id: m.tool_call_id,
|
|
101
|
-
// tool_calls: m.tool_calls?.map(t => t.id),
|
|
102
|
-
// content: m.content?.slice?.(0, 100), // truncate for readability
|
|
103
|
-
// }))
|
|
104
|
-
// );
|
|
105
|
-
|
|
106
96
|
const unrespondedCalls = messages.filter(
|
|
107
97
|
(msg, i) =>
|
|
108
98
|
msg.role === 'assistant' &&
|
|
@@ -117,7 +107,10 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
|
|
|
117
107
|
if (unrespondedCalls.length) {
|
|
118
108
|
console.error('š Detected assistant tool calls without matching tool responses:');
|
|
119
109
|
console.dir(unrespondedCalls, { depth: 5 });
|
|
120
|
-
|
|
110
|
+
currentStep.events.push('š Detected assistant tool calls without matching tool responses')
|
|
111
|
+
currentStep.result = 'ā Failure';
|
|
112
|
+
steps.push(currentStep);
|
|
113
|
+
return steps;
|
|
121
114
|
}
|
|
122
115
|
|
|
123
116
|
|
|
@@ -134,59 +127,51 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
|
|
|
134
127
|
|
|
135
128
|
if (retryCount >= 5) {
|
|
136
129
|
console.error('ā Too many retries. Exiting.');
|
|
137
|
-
|
|
130
|
+
currentStep.events.push('ā Too many retries. Exiting.')
|
|
131
|
+
currentStep.result = 'ā Failure';
|
|
132
|
+
steps.push(currentStep);
|
|
133
|
+
return steps;
|
|
138
134
|
}
|
|
139
135
|
|
|
140
|
-
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1);
|
|
136
|
+
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, currentStep);
|
|
141
137
|
} else if (err.status === 400) {
|
|
142
138
|
console.error('ā Bad request:', err.message);
|
|
143
|
-
console.log("current tools", toolsSchema);
|
|
144
|
-
console.log("messages: ", messages)
|
|
145
|
-
|
|
139
|
+
// console.log("current tools", toolsSchema);
|
|
140
|
+
// console.log("messages: ", messages)
|
|
141
|
+
currentStep.events.push(`ā Bad request: ${err.message}`)
|
|
142
|
+
currentStep.result = 'ā Failure';
|
|
143
|
+
steps.push(currentStep);
|
|
144
|
+
return steps;
|
|
146
145
|
} else {
|
|
147
146
|
throw err;
|
|
148
147
|
}
|
|
149
148
|
}
|
|
150
149
|
console.log('Response received from OpenAI');
|
|
151
150
|
const usage = response.usage;
|
|
152
|
-
// if (usage) {
|
|
153
|
-
// console.log(`š Token Usage This Turn ā Prompt: ${usage.prompt_tokens}, Completion: ${usage.completion_tokens}, Total: ${usage.total_tokens}`);
|
|
154
|
-
// totalTokensUsed += usage?.total_tokens || 0;
|
|
155
|
-
// turnTimestamps.push(now);
|
|
156
|
-
// console.log(`š Running Total Tokens Used: ${totalTokensUsed}`);
|
|
157
|
-
// }
|
|
158
151
|
if (usage) {
|
|
159
152
|
const tokensUsed = usage.total_tokens || 0;
|
|
160
153
|
console.log(`š Token Usage This Turn ā Total: ${tokensUsed}`);
|
|
154
|
+
currentStep.tokensUsed = tokensUsed;
|
|
161
155
|
recordTokenUsage(turnTimestamps, tokensUsed);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
totalTokensUsed = pruned.totalTokensUsed;
|
|
156
|
+
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
157
|
+
|
|
165
158
|
console.log(`š Running Total Tokens Used (Rolling 60s): ${totalTokensUsed}`);
|
|
159
|
+
currentStep.totalTokensUsed = totalTokensUsed;
|
|
166
160
|
}
|
|
167
161
|
|
|
168
162
|
const msg = response.choices[0].message;
|
|
169
163
|
|
|
170
|
-
// if (!msg.tool_calls?.length) {
|
|
171
|
-
// console.error('ā Assistant message had empty or missing tool_calls array');
|
|
172
|
-
// return false;
|
|
173
|
-
// }
|
|
174
|
-
|
|
175
164
|
if (msg.tool_calls?.length) {
|
|
176
165
|
console.log('Processing tool calls...');
|
|
177
|
-
// messages.push(msg);
|
|
178
166
|
const toolResponses = [];
|
|
179
167
|
let lastToolName = null;
|
|
180
168
|
|
|
181
169
|
for (const call of msg.tool_calls) {
|
|
182
|
-
const toolCallId = call.id;
|
|
183
|
-
// const hasResponse = messages.some(m =>
|
|
184
|
-
// m.role === 'tool' && m.tool_call_id === toolCallId
|
|
185
|
-
// );
|
|
186
170
|
const fnName = call.function.name;
|
|
187
171
|
lastToolName = fnName;
|
|
188
172
|
const args = JSON.parse(call.function.arguments || '{}');
|
|
189
173
|
console.log(`[model] ā ${fnName}`, args);
|
|
174
|
+
currentStep.events.push(`[model] ā ${fnName} ${args}`);
|
|
190
175
|
// console.log('Calling tool:', fnName);
|
|
191
176
|
let result;
|
|
192
177
|
let errorMessage = null;
|
|
@@ -194,15 +179,6 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
|
|
|
194
179
|
// console.log("agentMemory: ", agentMemory);
|
|
195
180
|
result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
|
|
196
181
|
|
|
197
|
-
// Smart DOM-ready wait after major actions like click
|
|
198
|
-
// if (['click', 'click_text', 'fill'].includes(fnName)) {
|
|
199
|
-
// try {
|
|
200
|
-
// await browser.page.waitForLoadState?.('domcontentloaded', { timeout: 5000 });
|
|
201
|
-
// await browser.page.waitForTimeout?.(2000); // allow animations/navigation
|
|
202
|
-
// } catch (err) {
|
|
203
|
-
// console.warn(`ā ļø DOM readiness wait failed after ${fnName}: ${err.message}`);
|
|
204
|
-
// }
|
|
205
|
-
// }
|
|
206
182
|
if (typeof result !== 'string') {
|
|
207
183
|
result = JSON.stringify(result ?? '');
|
|
208
184
|
}
|
|
@@ -211,27 +187,20 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
|
|
|
211
187
|
result = errorMessage;
|
|
212
188
|
}
|
|
213
189
|
|
|
214
|
-
// š Diagnostic: log whether this tool call succeeded or failed
|
|
215
190
|
console.log(`[tool ] ā ${fnName} result:`, errorMessage ? 'ā Failed' : 'ā
Success');
|
|
216
|
-
|
|
191
|
+
currentStep.events.push(`[tool ] ā ${fnName} result: ${errorMessage ? 'ā Failed' : 'ā
Success'}`)
|
|
217
192
|
const truncated = result.length > 100 ? result.slice(0, 100) + 'ā¦' : result;
|
|
193
|
+
console.log(`[tool ] ā ${truncated}`);
|
|
218
194
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
// role: 'tool',
|
|
229
|
-
// tool_call_id: call.id,
|
|
230
|
-
// name: fnName,
|
|
231
|
-
// type: 'function',
|
|
232
|
-
// content: result,
|
|
233
|
-
// });
|
|
234
|
-
|
|
195
|
+
if (fnName === 'screenshot') {
|
|
196
|
+
const match = result.match(/screenshot.*?saved at: (.+\.png)/i);
|
|
197
|
+
if (match && match[1]) {
|
|
198
|
+
currentStep.screenshotPath = match[1];
|
|
199
|
+
currentStep.events.push(`š¼ļø Screenshot captured: ${match[1]}`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
currentStep.events.push(`[tool ] ā ${truncated}`)
|
|
235
204
|
toolResponses.push({
|
|
236
205
|
role: 'tool',
|
|
237
206
|
tool_call_id: call.id,
|
|
@@ -246,40 +215,41 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
|
|
|
246
215
|
|
|
247
216
|
if (fnName === 'click_text' || fnName === 'click' || fnName === 'expand_menu') {
|
|
248
217
|
console.log(`[auto] ā Injecting DOM after menu expand...`);
|
|
218
|
+
currentStep.events.push(`[auto] ā Injecting DOM after menu expand...`);
|
|
249
219
|
const domHtml = await CHROME_TOOL_MAP.get_dom(browser, {
|
|
250
220
|
limit: 100000,
|
|
251
221
|
exclude: true,
|
|
252
|
-
focus: [],
|
|
222
|
+
focus: [],
|
|
253
223
|
}, agentMemory);
|
|
254
224
|
await tokenEstimate('gpt-4o', domHtml);
|
|
255
|
-
// fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
|
|
256
225
|
await pushDOMAssistant(browser, messages, agentMemory, {
|
|
257
226
|
skipIfLastTool: ['get_dom', 'check_text']
|
|
258
227
|
});
|
|
259
228
|
console.log(`[auto] ā DOM size after ${fnName}: ${domHtml.length} chars`);
|
|
229
|
+
currentStep.events.push(`[auto] ā DOM size after ${fnName}: ${domHtml.length} chars`)
|
|
260
230
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
231
|
|
|
264
|
-
|
|
265
|
-
// console.error(`ā Tool call ${toolCallId} for ${call.function.name} was not followed by a response`);
|
|
266
|
-
// return false;
|
|
267
|
-
// }
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
// Only after ALL tool responses are ready
|
|
232
|
+
}
|
|
271
233
|
messages.push(msg, ...toolResponses);
|
|
272
|
-
|
|
234
|
+
currentStep.result = 'ā
Passed';
|
|
235
|
+
steps.push(currentStep);
|
|
273
236
|
continue;
|
|
274
237
|
}
|
|
275
238
|
|
|
276
|
-
const
|
|
277
|
-
if (
|
|
239
|
+
const finalResponse = finalResponseHandler(msg);
|
|
240
|
+
if (finalResponse !== null) {
|
|
241
|
+
currentStep.events.push(finalResponse.finalMessage);
|
|
242
|
+
currentStep.result = finalResponse.success ? 'ā
Mission Success': 'ā Mission Failure';
|
|
243
|
+
steps.push(currentStep);
|
|
244
|
+
return { success: finalResponse.finalMessage, steps };
|
|
245
|
+
}
|
|
246
|
+
|
|
278
247
|
|
|
279
248
|
await pushDOMAssistant(browser, messages, agentMemory,{
|
|
280
249
|
skipIfLastTool: ['get_dom', 'check_text']
|
|
281
250
|
});
|
|
282
251
|
|
|
283
252
|
console.log(`[auto] ā Injected DOM for next reasoning step`);
|
|
253
|
+
currentStep.events.push(`[auto] ā Injected DOM for next reasoning step`)
|
|
284
254
|
}
|
|
285
255
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.10",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"testronaut": "./bin/cli.js"
|
|
@@ -25,5 +25,12 @@
|
|
|
25
25
|
"openai": "^5.10.2",
|
|
26
26
|
"playwright": "^1.54.1",
|
|
27
27
|
"prompts": "^2.4.2"
|
|
28
|
-
}
|
|
28
|
+
},
|
|
29
|
+
"keywords": [
|
|
30
|
+
"cli",
|
|
31
|
+
"testing",
|
|
32
|
+
"automation",
|
|
33
|
+
"node",
|
|
34
|
+
"qa"
|
|
35
|
+
]
|
|
29
36
|
}
|
package/runner/missionLaunch.js
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -16,38 +16,6 @@ export async function launch(preMissionSetup) {
|
|
|
16
16
|
await runSuite(objectives, preMissionSetup);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
// export async function runMissions({ preMission, mission, postMission }) {
|
|
21
|
-
// const goals = [];
|
|
22
|
-
|
|
23
|
-
// if (preMission) {
|
|
24
|
-
// console.log('\nš§ Running pre-mission setup:', preMission);
|
|
25
|
-
// goals.push({ goal: preMission, label: 'pre-mission' });
|
|
26
|
-
// }
|
|
27
|
-
|
|
28
|
-
// if (mission) {
|
|
29
|
-
// console.log('\nš§ Running main mission:', mission);
|
|
30
|
-
// goals.push({ goal: mission, label: 'mission' });
|
|
31
|
-
// }
|
|
32
|
-
|
|
33
|
-
// if (postMission) {
|
|
34
|
-
// console.log('\nš§ Running post-mission cleanup:', postMission);
|
|
35
|
-
// goals.push({ goal: postMission, label: 'post-mission' });
|
|
36
|
-
// }
|
|
37
|
-
|
|
38
|
-
// console.log('\nš§ Running mission flow:\n', goals);
|
|
39
|
-
|
|
40
|
-
// // for (const { goal, label } of goals) {
|
|
41
|
-
// // console.log(`\nš§ Running ${label}:\n`);
|
|
42
|
-
// const success = await runAgent(goals);
|
|
43
|
-
// if (!success) {
|
|
44
|
-
// console.log(`ā Aborting after failed goal.`);
|
|
45
|
-
// return;
|
|
46
|
-
// }
|
|
47
|
-
// // }
|
|
48
|
-
// console.log('\nā
Mission flow complete.');
|
|
49
|
-
// }
|
|
50
|
-
|
|
51
19
|
export async function runMissions({ preMission, mission, postMission }, missionName) {
|
|
52
20
|
const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
|
|
53
21
|
|
|
@@ -58,17 +26,14 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
58
26
|
const goals = [];
|
|
59
27
|
|
|
60
28
|
if (pre.length) {
|
|
61
|
-
// console.log('\nš§ Running pre-mission setup:', pre);
|
|
62
29
|
goals.push(...pre.map(fn => ({ goal: fn, label: 'pre-mission' })));
|
|
63
30
|
}
|
|
64
31
|
|
|
65
32
|
if (main.length) {
|
|
66
|
-
// console.log('\nš§ Running main mission(s):', main);
|
|
67
33
|
goals.push(...main.map(fn => ({ goal: fn, label: 'mission' })));
|
|
68
34
|
}
|
|
69
35
|
|
|
70
36
|
if (post.length) {
|
|
71
|
-
// console.log('\nš§ Running post-mission cleanup:', post);
|
|
72
37
|
goals.push(...post.map(fn => ({ goal: fn, label: 'post-mission' })));
|
|
73
38
|
}
|
|
74
39
|
|
|
@@ -82,7 +47,15 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
82
47
|
console.log(`ā Aborting after failed goal.`);
|
|
83
48
|
return;
|
|
84
49
|
}
|
|
50
|
+
// console.log("success[0]: ", success[0]);
|
|
51
|
+
// console.log("success[0].steps.length: ", success[0].steps.length);
|
|
52
|
+
// console.log("success[0].steps[success[0].steps.length - 1].result: ", success[0].steps[success[0].steps.length - 1].result)
|
|
53
|
+
const missionStatus = success[0].steps[success[0].steps.length - 1].result;
|
|
54
|
+
// console.log("missionStatus: ", missionStatus)
|
|
55
|
+
if (missionStatus.toLowerCase().includes('failure')) {
|
|
56
|
+
// console.log("we detected the failure!")
|
|
57
|
+
success[0].status = 'failed';
|
|
58
|
+
}
|
|
85
59
|
console.log('\nā
Mission flow complete.');
|
|
86
60
|
return success;
|
|
87
|
-
|
|
88
61
|
}
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -90,9 +90,7 @@ export class ChromeBrowser {
|
|
|
90
90
|
if (innerText.includes(text)) {
|
|
91
91
|
await el.scrollIntoViewIfNeeded().catch(() => {});
|
|
92
92
|
await el.click();
|
|
93
|
-
await this.page.screenshot({ path: `missions/mission_reports/screenshot_after_click_${Date.now()}.png` });
|
|
94
93
|
await this.page.waitForTimeout(5000);
|
|
95
|
-
// await this.page.screenshot({ path: `screenshot_after_waiting_click_${Date.now()}.png` });
|
|
96
94
|
return `clicked text "${text}"`;
|
|
97
95
|
|
|
98
96
|
}
|
|
@@ -119,7 +117,6 @@ export class ChromeBrowser {
|
|
|
119
117
|
console.log(`[click_text] ā Clicking visible element with bounding box`, box);
|
|
120
118
|
await el.click();
|
|
121
119
|
await this.page.waitForTimeout(delayMs);
|
|
122
|
-
await this.page.screenshot({ path: `missions/mission_reports/screenshot_after_click_${Date.now()}.png` });
|
|
123
120
|
return `clicked visible text "${text}"`;
|
|
124
121
|
}
|
|
125
122
|
}
|
|
@@ -147,16 +144,18 @@ export class ChromeBrowser {
|
|
|
147
144
|
await this.page.waitForSelector(selector, { timeout: 30_000 });
|
|
148
145
|
await this.page.click(selector);
|
|
149
146
|
await this.page.waitForTimeout(delayMs);
|
|
150
|
-
|
|
151
|
-
const expandedDom = await this.page.content();
|
|
152
|
-
// const filename = `missions/mission_reports/expand_menu_dump_${Date.now()}.html`;
|
|
153
|
-
// fs.writeFileSync(filename, expandedDom);
|
|
154
|
-
// console.log(`[expand_menu] ā Dumped DOM after expand click to ${filename}`);
|
|
155
|
-
await this.page.screenshot({ path: `missions/mission_reports/screenshot_expand_menu_${Date.now()}.png` });
|
|
156
|
-
|
|
157
147
|
return `expanded menu using ${selector}`;
|
|
158
148
|
}
|
|
159
149
|
|
|
150
|
+
async screenshot({ name = 'screenshot' }) {
|
|
151
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
152
|
+
const screenshotPath = `missions/mission_reports/screenshots/${name}_${timestamp}.png`;
|
|
153
|
+
|
|
154
|
+
await this.page.screenshot({ path: screenshotPath });
|
|
155
|
+
|
|
156
|
+
// ā
Return relative path for HTML report
|
|
157
|
+
return `Screenshot saved at: ./screenshots/${name}_${timestamp}.png`;
|
|
158
|
+
}
|
|
160
159
|
|
|
161
160
|
|
|
162
161
|
async close() {
|
|
@@ -178,7 +177,6 @@ export const CHROME_TOOL_MAP = {
|
|
|
178
177
|
return await browser.click_text(args);
|
|
179
178
|
}
|
|
180
179
|
},
|
|
181
|
-
|
|
182
180
|
expand_menu: (b, args) => b.expand_menu(args),
|
|
183
|
-
|
|
181
|
+
screenshot: (b, args) => b.screenshot(args),
|
|
184
182
|
};
|
|
@@ -1,52 +1,122 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
2
3
|
|
|
3
4
|
export function generateHtmlReport(report, outputPath) {
|
|
4
|
-
const { runId, startTime, endTime,
|
|
5
|
+
const { runId, startTime, endTime, missions = [], summary = {} } = report;
|
|
6
|
+
const durationSec = ((new Date(endTime) - new Date(startTime)) / 1000).toFixed(2);
|
|
7
|
+
|
|
8
|
+
// tiny helpers
|
|
9
|
+
const esc = (s) => String(s ?? '')
|
|
10
|
+
.replace(/&/g, '&').replace(/</g, '<')
|
|
11
|
+
.replace(/>/g, '>');
|
|
12
|
+
const badge = (status) =>
|
|
13
|
+
status === 'passed' ? 'ā
Passed' :
|
|
14
|
+
status === 'failed' ? 'ā Failed' : (status || 'ā');
|
|
15
|
+
|
|
16
|
+
const missionBlock = (m, idx) => {
|
|
17
|
+
const mDurationSec = m.endTime && m.startTime
|
|
18
|
+
? ((m.endTime - m.startTime) / 1000).toFixed(2)
|
|
19
|
+
: 'ā';
|
|
5
20
|
|
|
6
|
-
const missionHtml = missions.map(m => {
|
|
7
|
-
const status = m.status?.toUpperCase?.() || 'UNKNOWN';
|
|
8
|
-
const missionName = m.missionName || 'Unnamed';
|
|
9
|
-
const file = m.file || 'N/A';
|
|
10
21
|
const steps = Array.isArray(m.steps) ? m.steps : [];
|
|
22
|
+
const stepItems = steps.map(step => {
|
|
23
|
+
const events = Array.isArray(step.events) ? step.events : [];
|
|
24
|
+
const imgTag = step.screenshotPath
|
|
25
|
+
? `<img src="${esc(step.screenshotPath)}" alt="screenshot turn ${esc(step.turn)}">`
|
|
26
|
+
: '';
|
|
27
|
+
return `
|
|
28
|
+
<details class="step" ${/Mission Success|Passed/.test(step.result||'') ? '' : 'open'}>
|
|
29
|
+
<summary>
|
|
30
|
+
<span class="turn">Turn ${esc((step.turn ?? 0) + 1)}</span>
|
|
31
|
+
<span class="step-result ${/ā
/.test(step.result||'') ? 'ok' : 'bad'}">${esc(step.result || '')}</span>
|
|
32
|
+
<span class="tokens">tokens: ${esc(step.tokensUsed ?? 'ā')} / total: ${esc(step.totalTokensUsed ?? 'ā')}</span>
|
|
33
|
+
</summary>
|
|
34
|
+
<pre class="events">${esc(events.join('\n')) || '(no events)'}</pre>
|
|
35
|
+
${imgTag}
|
|
36
|
+
</details>
|
|
37
|
+
`;
|
|
38
|
+
}).join('');
|
|
11
39
|
|
|
12
40
|
return `
|
|
13
|
-
<
|
|
14
|
-
<
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
${steps.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
</div>
|
|
41
|
+
<details class="mission">
|
|
42
|
+
<summary>
|
|
43
|
+
<span class="name">${esc(m.missionName || path.basename(m.file || `mission_${idx+1}`))}</span>
|
|
44
|
+
<span class="status ${m.status === 'passed' ? 'ok' : (m.status === 'failed' ? 'bad' : '')}">${badge(m.status)}</span>
|
|
45
|
+
<span class="meta">steps: ${steps.length} ⢠duration: ${mDurationSec}s ⢠file: ${esc(m.file || '')}</span>
|
|
46
|
+
</summary>
|
|
47
|
+
<div class="steps">
|
|
48
|
+
${stepItems || '<div class="empty">No steps recorded.</div>'}
|
|
49
|
+
</div>
|
|
50
|
+
</details>
|
|
24
51
|
`;
|
|
25
|
-
}
|
|
52
|
+
};
|
|
26
53
|
|
|
27
54
|
const html = `
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
55
|
+
<!DOCTYPE html>
|
|
56
|
+
<html>
|
|
57
|
+
<head>
|
|
58
|
+
<meta charset="UTF-8" />
|
|
59
|
+
<title>Testronaut Report ā ${esc(runId)}</title>
|
|
60
|
+
<style>
|
|
61
|
+
:root {
|
|
62
|
+
--ok: #16a34a;
|
|
63
|
+
--bad: #dc2626;
|
|
64
|
+
--muted: #6b7280;
|
|
65
|
+
--bg: #f8fafc;
|
|
66
|
+
--card: #ffffff;
|
|
67
|
+
--border: #e5e7eb;
|
|
68
|
+
--mono: ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;
|
|
69
|
+
}
|
|
70
|
+
body { margin:0; padding:24px; font-family: ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Ubuntu; background: var(--bg); color:#111827; }
|
|
71
|
+
h1 { margin: 0 0 6px 0; font-size: 20px; }
|
|
72
|
+
.run-meta { color: var(--muted); margin-bottom: 16px; }
|
|
73
|
+
.summary { display:flex; gap:12px; margin: 12px 0 20px; }
|
|
74
|
+
.pill { border:1px solid var(--border); background: var(--card); padding:6px 10px; border-radius:999px; font-size:12px; }
|
|
75
|
+
.pill.ok { border-color: #bbf7d0; background:#ecfdf5; color: var(--ok); }
|
|
76
|
+
.pill.bad { border-color: #fecaca; background:#fef2f2; color: var(--bad); }
|
|
77
|
+
|
|
78
|
+
details { background: var(--card); border:1px solid var(--border); border-radius:10px; margin:10px 0; }
|
|
79
|
+
summary { cursor:pointer; padding:12px 14px; display:flex; align-items:center; gap:12px; }
|
|
80
|
+
.mission > summary { font-weight:600; }
|
|
81
|
+
.name { flex:1; }
|
|
82
|
+
.status.ok { color: var(--ok); }
|
|
83
|
+
.status.bad { color: var(--bad); }
|
|
84
|
+
.meta { color: var(--muted); font-size:12px; }
|
|
85
|
+
|
|
86
|
+
.steps { padding: 0 14px 12px; }
|
|
87
|
+
.step summary { background: #fafafa; border-bottom:1px solid var(--border); border-radius: 10px 10px 0 0; }
|
|
88
|
+
.turn { font-weight:600; }
|
|
89
|
+
.step-result.ok { color: var(--ok); }
|
|
90
|
+
.step-result.bad { color: var(--bad); }
|
|
91
|
+
.tokens { color: var(--muted); font-size:12px; margin-left:auto; }
|
|
92
|
+
.events { background:#0b1020; color:#e5e7eb; padding:12px; border-radius:8px; margin:12px 0 0; font: 12px/1.4 var(--mono); max-height: 280px; overflow:auto; }
|
|
93
|
+
img { display:block; max-width:100%; border:1px solid var(--border); border-radius:8px; margin-top:10px; }
|
|
94
|
+
.empty { color: var(--muted); padding: 12px; }
|
|
95
|
+
</style>
|
|
96
|
+
</head>
|
|
97
|
+
<body>
|
|
98
|
+
<h1>š§āš Testronaut Report</h1>
|
|
99
|
+
<div class="run-meta">
|
|
100
|
+
<div><strong>Run ID:</strong> ${esc(runId)}</div>
|
|
101
|
+
<div><strong>Start:</strong> ${esc(startTime)}</div>
|
|
102
|
+
<div><strong>End:</strong> ${esc(endTime)} ⢠<strong>Duration:</strong> ${durationSec}s</div>
|
|
103
|
+
</div>
|
|
104
|
+
|
|
105
|
+
<div class="summary">
|
|
106
|
+
<div class="pill">Missions: ${esc(summary.totalMissions ?? missions.length)}</div>
|
|
107
|
+
<div class="pill ok">Passed: ${esc(summary.passed ?? missions.filter(m=>m.status==='passed').length)}</div>
|
|
108
|
+
<div class="pill bad">Failed: ${esc(summary.failed ?? missions.filter(m=>m.status==='failed').length)}</div>
|
|
109
|
+
</div>
|
|
110
|
+
|
|
111
|
+
${missions.map(missionBlock).join('')}
|
|
112
|
+
|
|
113
|
+
</body>
|
|
114
|
+
</html>
|
|
48
115
|
`;
|
|
49
116
|
|
|
50
|
-
|
|
51
|
-
|
|
117
|
+
const out = outputPath ?? path.resolve('missions/mission_reports', `${runId}.html`);
|
|
118
|
+
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
119
|
+
fs.writeFileSync(out, html);
|
|
120
|
+
console.log(`š Report generated at: ${out}`);
|
|
121
|
+
return out;
|
|
52
122
|
}
|
package/tools/toolSchema.js
CHANGED
|
@@ -155,7 +155,25 @@ const toolsSchema = [
|
|
|
155
155
|
required: ['chunkIndex', 'totalChunks']
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
type: 'function',
|
|
161
|
+
function: {
|
|
162
|
+
name: 'screenshot',
|
|
163
|
+
description: 'Takes a screenshot of the current page',
|
|
164
|
+
parameters: {
|
|
165
|
+
type: 'object',
|
|
166
|
+
properties: {
|
|
167
|
+
label: {
|
|
168
|
+
type: 'string',
|
|
169
|
+
description: 'Optional label for the screenshot filename',
|
|
170
|
+
},
|
|
171
|
+
},
|
|
172
|
+
required: [],
|
|
173
|
+
},
|
|
174
|
+
},
|
|
158
175
|
}
|
|
176
|
+
|
|
159
177
|
|
|
160
178
|
];
|
|
161
179
|
|
package/tools/turnLoopUtils.js
CHANGED
|
@@ -4,13 +4,13 @@ export const finalResponseHandler = (msg) => {
|
|
|
4
4
|
console.log('\nāā FINAL AGENT RESPONSE āāāāāāāāāāāāāāāāāāā');
|
|
5
5
|
console.log(msg.content);
|
|
6
6
|
console.log('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā');
|
|
7
|
-
return true;
|
|
7
|
+
return { finalMessage: msg.content, success: true};
|
|
8
8
|
}
|
|
9
9
|
if (final?.startsWith('failure')) {
|
|
10
10
|
console.log('\nāā FINAL AGENT RESPONSE āāāāāāāāāāāāāāāāāāā');
|
|
11
11
|
console.log(msg.content);
|
|
12
12
|
console.log('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā');
|
|
13
|
-
return false;
|
|
13
|
+
return { finalMessage: msg.content, success: false};
|
|
14
14
|
}
|
|
15
15
|
return null;
|
|
16
16
|
}
|