testronaut 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +194 -0
- package/bin/cli.js +1 -1
- package/bin/init.js +141 -57
- package/bin/initHelpers.js +140 -0
- package/core/agent.js +1 -1
- package/core/turnIntent.js +51 -20
- package/core/turnLoop.js +172 -105
- package/llm/gemini/geminiProvider.js +188 -0
- package/llm/llmFactory.js +50 -0
- package/llm/modelResolver.js +65 -0
- package/llm/openAI/openaiProvider.js +54 -0
- package/package.json +9 -3
- package/tests/binTests/init.test.js +189 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +12 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +14 -0
- package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js +19 -0
- package/tests/binTests/initHelpersTests/models.test.js +21 -0
- package/tests/coreTests/turnLoop.test.js +177 -0
- package/tests/llmTests/geminiProvider.test.js +191 -0
- package/tests/llmTests/llmFactory.test.js +78 -0
- package/tests/llmTests/modelResolver.test.js +119 -0
- package/tests/llmTests/openAIProvider.test.js +117 -0
- package/tests/toolsTests/tokenControl.test.js +150 -0
- package/tools/generateHtmlReport.js +83 -0
- package/tools/tokenControl.js +107 -17
- package/vitest.config.js +8 -0
- package/openAI/modelResolver.js +0 -33
package/core/turnIntent.js
CHANGED
|
@@ -13,7 +13,36 @@ function pickArg(args, keys) {
|
|
|
13
13
|
return null;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
// core/turnIntent.js
|
|
17
|
+
const ICONS = {
|
|
18
|
+
navigate: '🧭',
|
|
19
|
+
click_text: '🖱️',
|
|
20
|
+
click: '🖱️',
|
|
21
|
+
type: '⌨️',
|
|
22
|
+
fill: '⌨️',
|
|
23
|
+
press: '⌨️',
|
|
24
|
+
hover: '🪄',
|
|
25
|
+
wait_for_selector: '⏳',
|
|
26
|
+
wait: '⏳',
|
|
27
|
+
screenshot: '📸',
|
|
28
|
+
get_dom: '🧩',
|
|
29
|
+
check_text: '🔍',
|
|
30
|
+
scroll: '↕️',
|
|
31
|
+
upload_file: '📤',
|
|
32
|
+
download_file: '📥',
|
|
33
|
+
expand_menu: '📂',
|
|
34
|
+
set_viewport: '🖥️',
|
|
35
|
+
default: '⚙️',
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const EMOJI_ON_DEFAULT =
|
|
39
|
+
process.env.TESTRONAUT_EMOJI === '1' || process.env.TESTRONAUT_EMOJI === 'true';
|
|
40
|
+
|
|
41
|
+
const withIcon = (name, s, { emoji = EMOJI_ON_DEFAULT } = {}) =>
|
|
42
|
+
emoji ? `${ICONS[name] || ICONS.default} ${s}` : s;
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
export function summarizeToolCall(fnName, args = {}, opts = {}) {
|
|
17
46
|
const safeArgs = redactArgs(fnName, args);
|
|
18
47
|
|
|
19
48
|
const selector = pickArg(safeArgs, ['selector','xpath','css','role','testId']);
|
|
@@ -29,33 +58,35 @@ export function summarizeToolCall(fnName, args = {}) {
|
|
|
29
58
|
const tmo = timeout ? ` [≤${timeout}ms]` : '';
|
|
30
59
|
|
|
31
60
|
switch (fnName) {
|
|
32
|
-
case 'navigate':
|
|
33
|
-
case 'click_text':
|
|
34
|
-
case 'click':
|
|
61
|
+
case 'navigate': return withIcon('navigate', `Navigate to ${truncateMiddle(url || '(no url)', 80)}`, opts);
|
|
62
|
+
case 'click_text': return withIcon('click_text', `Click text${val}`, opts);
|
|
63
|
+
case 'click': return withIcon('click', `Click element${sel}`, opts);
|
|
35
64
|
case 'type':
|
|
36
|
-
case 'fill':
|
|
37
|
-
case 'press':
|
|
38
|
-
case 'hover':
|
|
39
|
-
case 'wait_for_selector':
|
|
40
|
-
|
|
41
|
-
case '
|
|
42
|
-
case '
|
|
43
|
-
case '
|
|
44
|
-
case '
|
|
45
|
-
case '
|
|
46
|
-
case '
|
|
47
|
-
case '
|
|
48
|
-
|
|
65
|
+
case 'fill': return withIcon('type', `Type${val}${selector ? ` into ${truncateMiddle(selector, 60)}` : ''}`, opts);
|
|
66
|
+
case 'press': return withIcon('press', `Press key${val || ''}${selector ? ` in ${truncateMiddle(selector, 60)}` : ''}`, opts);
|
|
67
|
+
case 'hover': return withIcon('hover', `Hover over${sel || val || ''}`, opts);
|
|
68
|
+
case 'wait_for_selector':
|
|
69
|
+
return withIcon('wait_for_selector', `Wait for${sel}${tmo}`, opts);
|
|
70
|
+
case 'wait': return withIcon('wait', `Wait${tmo || ''}`, opts);
|
|
71
|
+
case 'screenshot': return withIcon('screenshot', `Capture screenshot${selector ? ` of ${truncateMiddle(selector, 60)}` : ''}`, opts);
|
|
72
|
+
case 'get_dom': return withIcon('get_dom', `Inspect DOM (focused extract)`, opts);
|
|
73
|
+
case 'check_text': return withIcon('check_text', `Check page for${val || ' specific text'}`, opts);
|
|
74
|
+
case 'scroll': return withIcon('scroll', `Scroll to coords (${x ?? '?'}, ${y ?? '?'})`, opts);
|
|
75
|
+
case 'upload_file': return withIcon('upload_file', `Upload file ${truncateMiddle(file || '(unknown)', 60)}${selector ? ` via ${truncateMiddle(selector, 60)}` : ''}`, opts);
|
|
76
|
+
case 'download_file':
|
|
77
|
+
return withIcon('download_file', `Download file to ${truncateMiddle(file || '(unknown)', 60)}`, opts);
|
|
78
|
+
case 'expand_menu': return withIcon('expand_menu', `Expand menu${sel || ''}`, opts);
|
|
79
|
+
case 'set_viewport': return withIcon('set_viewport', `Set viewport ${safeArgs?.width}×${safeArgs?.height}`, opts);
|
|
49
80
|
default:
|
|
50
|
-
return `Run ${fnName} with ${truncateMiddle(JSON.stringify(safeArgs ?? {}), 80)}
|
|
81
|
+
return withIcon('default', `Run ${fnName} with ${truncateMiddle(JSON.stringify(safeArgs ?? {}), 80)}`, opts);
|
|
51
82
|
}
|
|
52
83
|
}
|
|
53
84
|
|
|
54
|
-
export function summarizeTurnIntentFromMessage(msg) {
|
|
85
|
+
export function summarizeTurnIntentFromMessage(msg, opts = {}) {
|
|
55
86
|
try {
|
|
56
87
|
if (msg?.tool_calls?.length) {
|
|
57
88
|
return msg.tool_calls
|
|
58
|
-
.map(c => summarizeToolCall(c.function?.name || 'unknown', JSON.parse(c.function?.arguments || '{}')))
|
|
89
|
+
.map(c => summarizeToolCall(c.function?.name || 'unknown', JSON.parse(c.function?.arguments || '{}'), opts))
|
|
59
90
|
.join(' → ');
|
|
60
91
|
}
|
|
61
92
|
if (msg?.content) return `Reasoning: ${truncateMiddle(msg.content, 120)}`;
|
package/core/turnLoop.js
CHANGED
|
@@ -1,4 +1,33 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* turnLoop.js
|
|
3
|
+
* ------------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Core execution loop for Testronaut's autonomous agent testing framework.
|
|
6
|
+
* Handles reasoning turns, model responses, tool execution, DOM updates,
|
|
7
|
+
* backoff logic, and final mission success/failure detection.
|
|
8
|
+
*
|
|
9
|
+
* Responsibilities:
|
|
10
|
+
* - Coordinate between browser (Playwright/Puppeteer) and chosen LLM.
|
|
11
|
+
* - Maintain conversation history and agent "memory" between turns.
|
|
12
|
+
* - Detect and execute tool/function calls emitted by the model.
|
|
13
|
+
* - Track and enforce token-per-minute rate limits with adaptive cooldowns.
|
|
14
|
+
* - Handle DOM re-injection after browser actions to support reasoning continuity.
|
|
15
|
+
* - Summarize each turn’s intent and record detailed step logs.
|
|
16
|
+
*
|
|
17
|
+
* Related tests:
|
|
18
|
+
* Located in `tests/coreTests/`
|
|
19
|
+
* (see integration tests covering multi-turn reasoning, tool calling, and backoff).
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - CLI mission runners (`missionRunner.js`, `runMission.js`)
|
|
23
|
+
* - Autonomous agent framework during mission playback.
|
|
24
|
+
*
|
|
25
|
+
* Notes:
|
|
26
|
+
* - Provider-agnostic: routes all LLM calls through the `llmFactory` adapter layer.
|
|
27
|
+
* - OpenAI and Gemini both normalize responses to an OpenAI-like message format.
|
|
28
|
+
* - Token control is model-sensitive but provider-neutral.
|
|
29
|
+
*/
|
|
30
|
+
|
|
2
31
|
import toolsSchema from '../tools/toolSchema.js';
|
|
3
32
|
import { CHROME_TOOL_MAP } from '../tools/chromeBrowser.js';
|
|
4
33
|
import fs from 'fs';
|
|
@@ -12,24 +41,31 @@ import {
|
|
|
12
41
|
tokenUseCoolOff,
|
|
13
42
|
recordTokenUsage,
|
|
14
43
|
pruneOldTokenUsage,
|
|
15
|
-
getCurrentTokenLimit,
|
|
16
44
|
updateLimitsFromHeaders
|
|
17
45
|
} from '../tools/tokenControl.js';
|
|
18
|
-
import {
|
|
46
|
+
import { resolveProviderModel } from '../llm/modelResolver.js';
|
|
47
|
+
import { getLLM } from '../llm/llmFactory.js';
|
|
19
48
|
import { summarizeTurnIntentFromMessage } from './turnIntent.js';
|
|
20
49
|
import { redactArgs } from './redaction.js';
|
|
21
50
|
|
|
51
|
+
// ─────────────────────────────────────────────
|
|
52
|
+
// STEP 0: Resolve provider + model and initialize adapter
|
|
53
|
+
// ─────────────────────────────────────────────
|
|
54
|
+
const { provider: PROVIDER_ID, model: MODEL_ID } = resolveProviderModel();
|
|
55
|
+
console.log(`🧠 Using LLM → provider: ${PROVIDER_ID}, model: ${MODEL_ID}`);
|
|
22
56
|
|
|
23
|
-
const
|
|
24
|
-
console.log(`🧠 Using OpenAI model: ${MODEL_ID}`);
|
|
25
|
-
|
|
26
|
-
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
57
|
+
const llm = getLLM(PROVIDER_ID);
|
|
27
58
|
|
|
59
|
+
// Rolling token counters used for self-throttling
|
|
28
60
|
let totalTokensUsed = 0;
|
|
29
61
|
let turnTimestamps = [];
|
|
30
62
|
let shouldBackoff;
|
|
31
63
|
let steps = [];
|
|
32
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Utility: format byte counts into human-readable strings.
|
|
67
|
+
* Used when reporting upload/download events in mission logs.
|
|
68
|
+
*/
|
|
33
69
|
function formatBytes(n) {
|
|
34
70
|
if (!Number.isFinite(n)) return `${n}`;
|
|
35
71
|
const u = ['B','KB','MB','GB','TB'];
|
|
@@ -38,7 +74,17 @@ function formatBytes(n) {
|
|
|
38
74
|
return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`;
|
|
39
75
|
}
|
|
40
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Pushes a new "get_dom" tool call and result pair into the message history.
|
|
79
|
+
* This keeps the model updated on the browser's DOM after actions like click/expand.
|
|
80
|
+
*
|
|
81
|
+
* @param {object} browser - The browser automation instance.
|
|
82
|
+
* @param {array} messages - The running conversation message array.
|
|
83
|
+
* @param {object} agentMemory - Memory object storing recent agent state.
|
|
84
|
+
* @param {{skipIfLastTool?: string[]}} opts - Optional tool skip conditions.
|
|
85
|
+
*/
|
|
41
86
|
const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool } = {}) => {
|
|
87
|
+
// Avoid redundant DOM pushes immediately following tools that already include DOM context.
|
|
42
88
|
if (skipIfLastTool && skipIfLastTool.includes(messages.at(-1)?.name)) {
|
|
43
89
|
console.log(`[skip] Skipping DOM push after redundant tool: ${messages.at(-1)?.name}`);
|
|
44
90
|
return;
|
|
@@ -46,6 +92,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
46
92
|
|
|
47
93
|
const domCallId = `get_dom_${Date.now()}`;
|
|
48
94
|
|
|
95
|
+
// Inject a synthetic assistant tool call
|
|
49
96
|
messages.push({
|
|
50
97
|
role: 'assistant',
|
|
51
98
|
tool_calls: [
|
|
@@ -61,9 +108,9 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
61
108
|
});
|
|
62
109
|
|
|
63
110
|
const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
|
|
64
|
-
await tokenEstimate(
|
|
65
|
-
// fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
|
|
111
|
+
await tokenEstimate(MODEL_ID, domHtml);
|
|
66
112
|
|
|
113
|
+
// Push the corresponding tool response back into the conversation
|
|
67
114
|
messages.push({
|
|
68
115
|
role: 'tool',
|
|
69
116
|
tool_call_id: domCallId,
|
|
@@ -71,8 +118,24 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
71
118
|
type: 'function',
|
|
72
119
|
content: typeof domHtml === 'string' ? domHtml : JSON.stringify(domHtml),
|
|
73
120
|
});
|
|
74
|
-
}
|
|
75
|
-
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Main turn loop driver.
|
|
125
|
+
*
|
|
126
|
+
* Runs the agent through iterative reasoning turns until:
|
|
127
|
+
* - Max turn count is reached, or
|
|
128
|
+
* - The model emits a "final" message (success/failure).
|
|
129
|
+
*
|
|
130
|
+
* @param {object} browser - Browser automation interface (Playwright/Puppeteer).
|
|
131
|
+
* @param {array} messages - Conversation messages so far.
|
|
132
|
+
* @param {number} maxTurns - Maximum allowed reasoning turns.
|
|
133
|
+
* @param {number} currentTurn - Turn index to start from (default 0).
|
|
134
|
+
* @param {number} retryCount - Internal retry counter for rate limiting.
|
|
135
|
+
* @param {object} currentStep - Step data being populated for the current turn.
|
|
136
|
+
* @param {object} ctx - Shared context (steps array, mission name).
|
|
137
|
+
* @returns {Promise<object[]>} - Collected step results or final success message.
|
|
138
|
+
*/
|
|
76
139
|
export const turnLoop = async (
|
|
77
140
|
browser,
|
|
78
141
|
messages,
|
|
@@ -80,36 +143,38 @@ export const turnLoop = async (
|
|
|
80
143
|
currentTurn = 0,
|
|
81
144
|
retryCount = 0,
|
|
82
145
|
currentStep = {},
|
|
83
|
-
ctx = {} //
|
|
146
|
+
ctx = {} // { steps, missionName }
|
|
84
147
|
) => {
|
|
85
148
|
const { steps = [], missionName } = ctx;
|
|
86
|
-
let agentMemory = {
|
|
87
|
-
lastMenuExpanded: false
|
|
88
|
-
};
|
|
149
|
+
let agentMemory = { lastMenuExpanded: false };
|
|
89
150
|
|
|
151
|
+
// ─────────────────────────────────────────────
|
|
152
|
+
// STEP 1: Begin reasoning cycle
|
|
153
|
+
// ─────────────────────────────────────────────
|
|
90
154
|
for (let turn = currentTurn; turn < maxTurns; turn++) {
|
|
91
155
|
console.log(`\n🔄 Turn ${turn + 1}/${maxTurns}`);
|
|
92
156
|
let response;
|
|
93
|
-
|
|
94
|
-
const currentStep = { turn, events: [], result: '🟡 In Progress', missionName }; // 👈 tag it
|
|
95
|
-
try {
|
|
157
|
+
const step = { turn, events: [], result: '🟡 In Progress', missionName };
|
|
96
158
|
|
|
97
|
-
|
|
159
|
+
try {
|
|
160
|
+
// Refresh token usage window (rolling 60 seconds)
|
|
98
161
|
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
99
162
|
|
|
100
|
-
//
|
|
101
|
-
({ totalTokensUsed, turnTimestamps, shouldBackoff } =
|
|
102
|
-
|
|
163
|
+
// Adaptive cooldown if nearing provider rate limit
|
|
164
|
+
({ totalTokensUsed, turnTimestamps, shouldBackoff } =
|
|
165
|
+
await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
|
|
166
|
+
if (shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
|
|
103
167
|
|
|
104
|
-
//
|
|
168
|
+
// Verify message structure integrity before requesting next model step
|
|
105
169
|
if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
|
|
106
170
|
console.error('❌ Tool call structure invalid and no placeholders inserted.');
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
steps.push(
|
|
171
|
+
step.events.push('❌ Tool call structure invalid and no placeholders inserted.');
|
|
172
|
+
step.result = '❌ Failure';
|
|
173
|
+
steps.push(step);
|
|
110
174
|
return steps;
|
|
111
175
|
}
|
|
112
176
|
|
|
177
|
+
// Detect orphaned assistant tool calls with no matching tool responses
|
|
113
178
|
const unrespondedCalls = messages.filter(
|
|
114
179
|
(msg, i) =>
|
|
115
180
|
msg.role === 'assistant' &&
|
|
@@ -120,25 +185,30 @@ export const turnLoop = async (
|
|
|
120
185
|
)
|
|
121
186
|
)
|
|
122
187
|
);
|
|
123
|
-
|
|
124
188
|
if (unrespondedCalls.length) {
|
|
125
189
|
console.error('🛑 Detected assistant tool calls without matching tool responses:');
|
|
126
190
|
console.dir(unrespondedCalls, { depth: 5 });
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
steps.push(
|
|
191
|
+
step.events.push('🛑 Detected assistant tool calls without matching tool responses');
|
|
192
|
+
step.result = '❌ Failure';
|
|
193
|
+
steps.push(step);
|
|
130
194
|
return steps;
|
|
131
195
|
}
|
|
132
|
-
|
|
133
196
|
|
|
134
|
-
|
|
197
|
+
// ─────────────────────────────────────────────
|
|
198
|
+
// STEP 2: Request next reasoning turn from model
|
|
199
|
+
// ─────────────────────────────────────────────
|
|
200
|
+
const { message, usage, headers } = await llm.chat({
|
|
135
201
|
model: MODEL_ID,
|
|
136
202
|
messages,
|
|
137
203
|
tools: toolsSchema,
|
|
138
204
|
});
|
|
205
|
+
response = { message, usage, headers };
|
|
206
|
+
|
|
139
207
|
} catch (err) {
|
|
208
|
+
// ─────────────────────────────────────────────
|
|
209
|
+
// STEP 3: Error and rate-limit handling
|
|
210
|
+
// ─────────────────────────────────────────────
|
|
140
211
|
if (err.status === 429) {
|
|
141
|
-
// If headers include limits, learn them for this model
|
|
142
212
|
try { updateLimitsFromHeaders(MODEL_ID, err.headers || err.response?.headers || {}); } catch {}
|
|
143
213
|
const delay = Math.min(60000, 2 ** retryCount * 2000);
|
|
144
214
|
console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
|
|
@@ -146,105 +216,100 @@ export const turnLoop = async (
|
|
|
146
216
|
|
|
147
217
|
if (retryCount >= 5) {
|
|
148
218
|
console.error('❌ Too many retries. Exiting.');
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
steps.push(
|
|
219
|
+
step.events.push('❌ Too many retries. Exiting.');
|
|
220
|
+
step.result = '❌ Failure';
|
|
221
|
+
steps.push(step);
|
|
152
222
|
return steps;
|
|
153
223
|
}
|
|
154
|
-
|
|
155
|
-
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, currentStep);
|
|
224
|
+
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
|
|
156
225
|
} else if (err.status === 400) {
|
|
157
226
|
console.error('❌ Bad request:', err.message);
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
currentStep.result = '❌ Failure';
|
|
162
|
-
steps.push(currentStep);
|
|
227
|
+
step.events.push(`❌ Bad request: ${err.message}`);
|
|
228
|
+
step.result = '❌ Failure';
|
|
229
|
+
steps.push(step);
|
|
163
230
|
return steps;
|
|
164
231
|
} else {
|
|
165
232
|
throw err;
|
|
166
233
|
}
|
|
167
234
|
}
|
|
168
|
-
|
|
235
|
+
|
|
236
|
+
// ─────────────────────────────────────────────
|
|
237
|
+
// STEP 4: Process model response
|
|
238
|
+
// ─────────────────────────────────────────────
|
|
239
|
+
console.log(`Response received from ${PROVIDER_ID}`);
|
|
169
240
|
const usage = response.usage;
|
|
170
241
|
if (usage) {
|
|
171
242
|
const tokensUsed = usage.total_tokens || 0;
|
|
172
243
|
console.log(`📊 Token Usage This Turn → Total: ${tokensUsed}`);
|
|
173
|
-
|
|
244
|
+
step.tokensUsed = tokensUsed;
|
|
174
245
|
recordTokenUsage(turnTimestamps, tokensUsed);
|
|
175
246
|
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
176
|
-
|
|
177
247
|
console.log(`📈 Running Total Tokens Used (Rolling 60s): ${totalTokensUsed}`);
|
|
178
|
-
|
|
248
|
+
step.totalTokensUsed = totalTokensUsed;
|
|
179
249
|
}
|
|
180
|
-
|
|
181
|
-
const msg = response.choices[0].message;
|
|
182
250
|
|
|
183
|
-
const
|
|
184
|
-
currentStep.summary = planLine;
|
|
185
|
-
currentStep.events.unshift(`📝 Plan: ${planLine}`);
|
|
186
|
-
console.log(`📝 Plan: ${planLine}`);
|
|
251
|
+
const msg = response.message ?? { role: 'assistant', content: '' };
|
|
187
252
|
|
|
253
|
+
// Summarize model’s plan for the turn (plain + emoji variants)
|
|
254
|
+
const planPlain = summarizeTurnIntentFromMessage(msg, { emoji: true });
|
|
255
|
+
const planDisplay = summarizeTurnIntentFromMessage(msg);
|
|
256
|
+
step.summary = planPlain;
|
|
257
|
+
step.events.unshift(`📝 Plan: ${planDisplay}`);
|
|
258
|
+
console.log(`📝 Plan: ${planDisplay}`);
|
|
259
|
+
|
|
260
|
+
// ─────────────────────────────────────────────
|
|
261
|
+
// STEP 5: Handle tool/function calls
|
|
262
|
+
// ─────────────────────────────────────────────
|
|
188
263
|
if (msg.tool_calls?.length) {
|
|
189
264
|
console.log('Processing tool calls...');
|
|
190
265
|
const toolResponses = [];
|
|
191
|
-
let lastToolName = null;
|
|
192
266
|
|
|
193
267
|
for (const call of msg.tool_calls) {
|
|
194
268
|
const fnName = call.function.name;
|
|
195
|
-
lastToolName = fnName;
|
|
196
269
|
const args = JSON.parse(call.function.arguments || '{}');
|
|
197
270
|
const safeArgs = redactArgs(fnName, args);
|
|
198
271
|
console.log(`[model] → ${fnName}`, safeArgs);
|
|
199
|
-
|
|
200
|
-
|
|
272
|
+
step.events.push(`[model] → ${fnName} ${safeArgs}`);
|
|
273
|
+
|
|
201
274
|
let result;
|
|
202
275
|
let errorMessage = null;
|
|
203
276
|
try {
|
|
204
|
-
// console.log("agentMemory: ", agentMemory);
|
|
205
277
|
result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
}
|
|
210
|
-
} catch (err) {
|
|
211
|
-
errorMessage = `ERROR: ${err.message}`;
|
|
278
|
+
if (typeof result !== 'string') result = JSON.stringify(result ?? '');
|
|
279
|
+
} catch (e) {
|
|
280
|
+
errorMessage = `ERROR: ${e.message}`;
|
|
212
281
|
result = errorMessage;
|
|
213
282
|
}
|
|
214
283
|
|
|
215
|
-
//
|
|
284
|
+
// Capture and log any file upload/download events
|
|
216
285
|
try {
|
|
217
286
|
const maybeJson = JSON.parse(result);
|
|
218
287
|
if (maybeJson && maybeJson._testronaut_file_event) {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
} else if (maybeJson._testronaut_file_event === 'download') {
|
|
227
|
-
const msgLine = `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
|
|
228
|
-
currentStep.events.push(msgLine);
|
|
229
|
-
console.log(msgLine);
|
|
230
|
-
}
|
|
288
|
+
step.files = step.files || [];
|
|
289
|
+
step.files.push(maybeJson);
|
|
290
|
+
const msgLine = maybeJson._testronaut_file_event === 'upload'
|
|
291
|
+
? `📤 Uploaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.method}`
|
|
292
|
+
: `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
|
|
293
|
+
step.events.push(msgLine);
|
|
294
|
+
console.log(msgLine);
|
|
231
295
|
}
|
|
232
|
-
} catch { /*
|
|
296
|
+
} catch { /* non-JSON results ignored */ }
|
|
233
297
|
|
|
234
298
|
console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
|
|
235
|
-
currentStep.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`)
|
|
236
299
|
const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
|
|
237
|
-
|
|
300
|
+
step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
|
|
301
|
+
step.events.push(`[tool ] ← ${truncated}`);
|
|
238
302
|
|
|
303
|
+
// Screenshot detection
|
|
239
304
|
if (fnName === 'screenshot') {
|
|
240
305
|
const match = result.match(/screenshot.*?saved at: (.+\.png)/i);
|
|
241
306
|
if (match && match[1]) {
|
|
242
|
-
|
|
243
|
-
|
|
307
|
+
step.screenshotPath = match[1];
|
|
308
|
+
step.events.push(`🖼️ Screenshot captured: ${match[1]}`);
|
|
244
309
|
}
|
|
245
310
|
}
|
|
246
|
-
|
|
247
|
-
|
|
311
|
+
|
|
312
|
+
// Push corresponding tool message back to LLM
|
|
248
313
|
toolResponses.push({
|
|
249
314
|
role: 'tool',
|
|
250
315
|
tool_call_id: call.id,
|
|
@@ -253,13 +318,10 @@ export const turnLoop = async (
|
|
|
253
318
|
content: result,
|
|
254
319
|
});
|
|
255
320
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
if (fnName === 'click_text' || fnName === 'click' || fnName === 'expand_menu') {
|
|
261
|
-
console.log(`[auto] → Injecting DOM after menu expand...`);
|
|
262
|
-
currentStep.events.push(`[auto] → Injecting DOM after menu expand...`);
|
|
321
|
+
// After interactive DOM actions, refresh model context
|
|
322
|
+
if (['click_text', 'click', 'expand_menu'].includes(fnName)) {
|
|
323
|
+
console.log(`[auto] → Injecting DOM after ${fnName}...`);
|
|
324
|
+
step.events.push(`[auto] → Injecting DOM after ${fnName}...`);
|
|
263
325
|
const domHtml = await CHROME_TOOL_MAP.get_dom(browser, {
|
|
264
326
|
limit: 100000,
|
|
265
327
|
exclude: true,
|
|
@@ -267,33 +329,38 @@ export const turnLoop = async (
|
|
|
267
329
|
}, agentMemory);
|
|
268
330
|
await tokenEstimate(MODEL_ID, domHtml);
|
|
269
331
|
await pushDOMAssistant(browser, messages, agentMemory, {
|
|
270
|
-
skipIfLastTool: ['get_dom', 'check_text']
|
|
332
|
+
skipIfLastTool: ['get_dom', 'check_text'],
|
|
271
333
|
});
|
|
272
334
|
console.log(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
|
|
273
|
-
|
|
335
|
+
step.events.push(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
|
|
274
336
|
}
|
|
275
|
-
|
|
276
337
|
}
|
|
338
|
+
|
|
339
|
+
// Merge new assistant + tool responses back into conversation
|
|
277
340
|
messages.push(msg, ...toolResponses);
|
|
278
|
-
|
|
279
|
-
steps.push(
|
|
341
|
+
step.result = '✅ Passed';
|
|
342
|
+
steps.push(step);
|
|
280
343
|
continue;
|
|
281
344
|
}
|
|
282
345
|
|
|
346
|
+
// ─────────────────────────────────────────────
|
|
347
|
+
// STEP 6: Detect final mission state
|
|
348
|
+
// ─────────────────────────────────────────────
|
|
283
349
|
const finalResponse = finalResponseHandler(msg);
|
|
284
350
|
if (finalResponse !== null) {
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
steps.push(
|
|
351
|
+
step.events.push(finalResponse.finalMessage);
|
|
352
|
+
step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
|
|
353
|
+
steps.push(step);
|
|
288
354
|
return { success: finalResponse.finalMessage, steps };
|
|
289
355
|
}
|
|
290
356
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
357
|
+
// ─────────────────────────────────────────────
|
|
358
|
+
// STEP 7: Fallback → Push DOM for next reasoning cycle
|
|
359
|
+
// ─────────────────────────────────────────────
|
|
360
|
+
await pushDOMAssistant(browser, messages, agentMemory, {
|
|
361
|
+
skipIfLastTool: ['get_dom', 'check_text'],
|
|
294
362
|
});
|
|
295
|
-
|
|
296
363
|
console.log(`[auto] → Injected DOM for next reasoning step`);
|
|
297
|
-
|
|
364
|
+
step.events.push(`[auto] → Injected DOM for next reasoning step`);
|
|
298
365
|
}
|
|
299
|
-
}
|
|
366
|
+
};
|