testronaut 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,7 +13,36 @@ function pickArg(args, keys) {
13
13
  return null;
14
14
  }
15
15
 
16
- export function summarizeToolCall(fnName, args = {}) {
16
+ // core/turnIntent.js
17
+ const ICONS = {
18
+ navigate: '🧭',
19
+ click_text: '🖱️',
20
+ click: '🖱️',
21
+ type: '⌨️',
22
+ fill: '⌨️',
23
+ press: '⌨️',
24
+ hover: '🪄',
25
+ wait_for_selector: '⏳',
26
+ wait: '⏳',
27
+ screenshot: '📸',
28
+ get_dom: '🧩',
29
+ check_text: '🔍',
30
+ scroll: '↕️',
31
+ upload_file: '📤',
32
+ download_file: '📥',
33
+ expand_menu: '📂',
34
+ set_viewport: '🖥️',
35
+ default: '⚙️',
36
+ };
37
+
38
+ const EMOJI_ON_DEFAULT =
39
+ process.env.TESTRONAUT_EMOJI === '1' || process.env.TESTRONAUT_EMOJI === 'true';
40
+
41
+ const withIcon = (name, s, { emoji = EMOJI_ON_DEFAULT } = {}) =>
42
+ emoji ? `${ICONS[name] || ICONS.default} ${s}` : s;
43
+
44
+
45
+ export function summarizeToolCall(fnName, args = {}, opts = {}) {
17
46
  const safeArgs = redactArgs(fnName, args);
18
47
 
19
48
  const selector = pickArg(safeArgs, ['selector','xpath','css','role','testId']);
@@ -29,33 +58,35 @@ export function summarizeToolCall(fnName, args = {}) {
29
58
  const tmo = timeout ? ` [≤${timeout}ms]` : '';
30
59
 
31
60
  switch (fnName) {
32
- case 'navigate': return `Navigate to ${truncateMiddle(url || '(no url)', 80)}`;
33
- case 'click_text': return `Click text${val}`;
34
- case 'click': return `Click element${sel}`;
61
+ case 'navigate': return withIcon('navigate', `Navigate to ${truncateMiddle(url || '(no url)', 80)}`, opts);
62
+ case 'click_text': return withIcon('click_text', `Click text${val}`, opts);
63
+ case 'click': return withIcon('click', `Click element${sel}`, opts);
35
64
  case 'type':
36
- case 'fill': return `Type${val}${selector ? ` into ${truncateMiddle(selector, 60)}` : ''}`;
37
- case 'press': return `Press key${val || ''}${selector ? ` in ${truncateMiddle(selector, 60)}` : ''}`;
38
- case 'hover': return `Hover over${sel || val || ''}`;
39
- case 'wait_for_selector': return `Wait for${sel}${tmo}`;
40
- case 'wait': return `Wait${tmo || ''}`;
41
- case 'screenshot': return `Capture screenshot${selector ? ` of ${truncateMiddle(selector, 60)}` : ''}`;
42
- case 'get_dom': return `Inspect DOM (focused extract)`;
43
- case 'check_text': return `Check page for${val || ' specific text'}`;
44
- case 'scroll': return `Scroll to coords (${x ?? '?'}, ${y ?? '?'})`;
45
- case 'upload_file': return `Upload file ${truncateMiddle(file || '(unknown)', 60)}${selector ? ` via ${truncateMiddle(selector, 60)}` : ''}`;
46
- case 'download_file': return `Download file to ${truncateMiddle(file || '(unknown)', 60)}`;
47
- case 'expand_menu': return `Expand menu${sel || ''}`;
48
- case 'set_viewport': return `Set viewport ${safeArgs?.width}×${safeArgs?.height}`;
65
+ case 'fill': return withIcon('type', `Type${val}${selector ? ` into ${truncateMiddle(selector, 60)}` : ''}`, opts);
66
+ case 'press': return withIcon('press', `Press key${val || ''}${selector ? ` in ${truncateMiddle(selector, 60)}` : ''}`, opts);
67
+ case 'hover': return withIcon('hover', `Hover over${sel || val || ''}`, opts);
68
+ case 'wait_for_selector':
69
+ return withIcon('wait_for_selector', `Wait for${sel}${tmo}`, opts);
70
+ case 'wait': return withIcon('wait', `Wait${tmo || ''}`, opts);
71
+ case 'screenshot': return withIcon('screenshot', `Capture screenshot${selector ? ` of ${truncateMiddle(selector, 60)}` : ''}`, opts);
72
+ case 'get_dom': return withIcon('get_dom', `Inspect DOM (focused extract)`, opts);
73
+ case 'check_text': return withIcon('check_text', `Check page for${val || ' specific text'}`, opts);
74
+ case 'scroll': return withIcon('scroll', `Scroll to coords (${x ?? '?'}, ${y ?? '?'})`, opts);
75
+ case 'upload_file': return withIcon('upload_file', `Upload file ${truncateMiddle(file || '(unknown)', 60)}${selector ? ` via ${truncateMiddle(selector, 60)}` : ''}`, opts);
76
+ case 'download_file':
77
+ return withIcon('download_file', `Download file to ${truncateMiddle(file || '(unknown)', 60)}`, opts);
78
+ case 'expand_menu': return withIcon('expand_menu', `Expand menu${sel || ''}`, opts);
79
+ case 'set_viewport': return withIcon('set_viewport', `Set viewport ${safeArgs?.width}×${safeArgs?.height}`, opts);
49
80
  default:
50
- return `Run ${fnName} with ${truncateMiddle(JSON.stringify(safeArgs ?? {}), 80)}`;
81
+ return withIcon('default', `Run ${fnName} with ${truncateMiddle(JSON.stringify(safeArgs ?? {}), 80)}`, opts);
51
82
  }
52
83
  }
53
84
 
54
- export function summarizeTurnIntentFromMessage(msg) {
85
+ export function summarizeTurnIntentFromMessage(msg, opts = {}) {
55
86
  try {
56
87
  if (msg?.tool_calls?.length) {
57
88
  return msg.tool_calls
58
- .map(c => summarizeToolCall(c.function?.name || 'unknown', JSON.parse(c.function?.arguments || '{}')))
89
+ .map(c => summarizeToolCall(c.function?.name || 'unknown', JSON.parse(c.function?.arguments || '{}'), opts))
59
90
  .join(' → ');
60
91
  }
61
92
  if (msg?.content) return `Reasoning: ${truncateMiddle(msg.content, 120)}`;
package/core/turnLoop.js CHANGED
@@ -1,4 +1,33 @@
1
- import OpenAI from 'openai';
1
+ /**
2
+ * turnLoop.js
3
+ * ------------
4
+ * Purpose:
5
+ * Core execution loop for Testronaut's autonomous agent testing framework.
6
+ * Handles reasoning turns, model responses, tool execution, DOM updates,
7
+ * backoff logic, and final mission success/failure detection.
8
+ *
9
+ * Responsibilities:
10
+ * - Coordinate between browser (Playwright/Puppeteer) and chosen LLM.
11
+ * - Maintain conversation history and agent "memory" between turns.
12
+ * - Detect and execute tool/function calls emitted by the model.
13
+ * - Track and enforce token-per-minute rate limits with adaptive cooldowns.
14
+ * - Handle DOM re-injection after browser actions to support reasoning continuity.
15
+ * - Summarize each turn’s intent and record detailed step logs.
16
+ *
17
+ * Related tests:
18
+ * Located in `tests/coreTests/`
19
+ * (see integration tests covering multi-turn reasoning, tool calling, and backoff).
20
+ *
21
+ * Used by:
22
+ * - CLI mission runners (`missionRunner.js`, `runMission.js`)
23
+ * - Autonomous agent framework during mission playback.
24
+ *
25
+ * Notes:
26
+ * - Provider-agnostic: routes all LLM calls through the `llmFactory` adapter layer.
27
+ * - OpenAI and Gemini both normalize responses to an OpenAI-like message format.
28
+ * - Token control is model-sensitive but provider-neutral.
29
+ */
30
+
2
31
  import toolsSchema from '../tools/toolSchema.js';
3
32
  import { CHROME_TOOL_MAP } from '../tools/chromeBrowser.js';
4
33
  import fs from 'fs';
@@ -12,24 +41,31 @@ import {
12
41
  tokenUseCoolOff,
13
42
  recordTokenUsage,
14
43
  pruneOldTokenUsage,
15
- getCurrentTokenLimit,
16
44
  updateLimitsFromHeaders
17
45
  } from '../tools/tokenControl.js';
18
- import { resolveModel } from '../openAI/modelResolver.js';
46
+ import { resolveProviderModel } from '../llm/modelResolver.js';
47
+ import { getLLM } from '../llm/llmFactory.js';
19
48
  import { summarizeTurnIntentFromMessage } from './turnIntent.js';
20
49
  import { redactArgs } from './redaction.js';
21
50
 
51
+ // ─────────────────────────────────────────────
52
+ // STEP 0: Resolve provider + model and initialize adapter
53
+ // ─────────────────────────────────────────────
54
+ const { provider: PROVIDER_ID, model: MODEL_ID } = resolveProviderModel();
55
+ console.log(`🧠 Using LLM → provider: ${PROVIDER_ID}, model: ${MODEL_ID}`);
22
56
 
23
- const MODEL_ID = resolveModel();
24
- console.log(`🧠 Using OpenAI model: ${MODEL_ID}`);
25
-
26
- const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
57
+ const llm = getLLM(PROVIDER_ID);
27
58
 
59
+ // Rolling token counters used for self-throttling
28
60
  let totalTokensUsed = 0;
29
61
  let turnTimestamps = [];
30
62
  let shouldBackoff;
31
63
  let steps = [];
32
64
 
65
+ /**
66
+ * Utility: format byte counts into human-readable strings.
67
+ * Used when reporting upload/download events in mission logs.
68
+ */
33
69
  function formatBytes(n) {
34
70
  if (!Number.isFinite(n)) return `${n}`;
35
71
  const u = ['B','KB','MB','GB','TB'];
@@ -38,7 +74,17 @@ function formatBytes(n) {
38
74
  return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`;
39
75
  }
40
76
 
77
+ /**
78
+ * Pushes a new "get_dom" tool call and result pair into the message history.
79
+ * This keeps the model updated on the browser's DOM after actions like click/expand.
80
+ *
81
+ * @param {object} browser - The browser automation instance.
82
+ * @param {array} messages - The running conversation message array.
83
+ * @param {object} agentMemory - Memory object storing recent agent state.
84
+ * @param {{skipIfLastTool?: string[]}} opts - Optional tool skip conditions.
85
+ */
41
86
  const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool } = {}) => {
87
+ // Avoid redundant DOM pushes immediately following tools that already include DOM context.
42
88
  if (skipIfLastTool && skipIfLastTool.includes(messages.at(-1)?.name)) {
43
89
  console.log(`[skip] Skipping DOM push after redundant tool: ${messages.at(-1)?.name}`);
44
90
  return;
@@ -46,6 +92,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
46
92
 
47
93
  const domCallId = `get_dom_${Date.now()}`;
48
94
 
95
+ // Inject a synthetic assistant tool call
49
96
  messages.push({
50
97
  role: 'assistant',
51
98
  tool_calls: [
@@ -61,9 +108,9 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
61
108
  });
62
109
 
63
110
  const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
64
- await tokenEstimate( MODEL_ID, domHtml);
65
- // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
111
+ await tokenEstimate(MODEL_ID, domHtml);
66
112
 
113
+ // Push the corresponding tool response back into the conversation
67
114
  messages.push({
68
115
  role: 'tool',
69
116
  tool_call_id: domCallId,
@@ -71,8 +118,24 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
71
118
  type: 'function',
72
119
  content: typeof domHtml === 'string' ? domHtml : JSON.stringify(domHtml),
73
120
  });
74
- }
75
-
121
+ };
122
+
123
+ /**
124
+ * Main turn loop driver.
125
+ *
126
+ * Runs the agent through iterative reasoning turns until:
127
+ * - Max turn count is reached, or
128
+ * - The model emits a "final" message (success/failure).
129
+ *
130
+ * @param {object} browser - Browser automation interface (Playwright/Puppeteer).
131
+ * @param {array} messages - Conversation messages so far.
132
+ * @param {number} maxTurns - Maximum allowed reasoning turns.
133
+ * @param {number} currentTurn - Turn index to start from (default 0).
134
+ * @param {number} retryCount - Internal retry counter for rate limiting.
135
+ * @param {object} currentStep - Step data being populated for the current turn.
136
+ * @param {object} ctx - Shared context (steps array, mission name).
137
+ * @returns {Promise<object[]>} - Collected step results or final success message.
138
+ */
76
139
  export const turnLoop = async (
77
140
  browser,
78
141
  messages,
@@ -80,36 +143,38 @@ export const turnLoop = async (
80
143
  currentTurn = 0,
81
144
  retryCount = 0,
82
145
  currentStep = {},
83
- ctx = {} // 👈 { steps, missionName }
146
+ ctx = {} // { steps, missionName }
84
147
  ) => {
85
148
  const { steps = [], missionName } = ctx;
86
- let agentMemory = {
87
- lastMenuExpanded: false
88
- };
149
+ let agentMemory = { lastMenuExpanded: false };
89
150
 
151
+ // ─────────────────────────────────────────────
152
+ // STEP 1: Begin reasoning cycle
153
+ // ─────────────────────────────────────────────
90
154
  for (let turn = currentTurn; turn < maxTurns; turn++) {
91
155
  console.log(`\n🔄 Turn ${turn + 1}/${maxTurns}`);
92
156
  let response;
93
- // const currentStep = { turn, events: [], result: '🟡 In Progress' };
94
- const currentStep = { turn, events: [], result: '🟡 In Progress', missionName }; // 👈 tag it
95
- try {
157
+ const step = { turn, events: [], result: '🟡 In Progress', missionName };
96
158
 
97
- // Recalculate the rolling window before checking cooldown
159
+ try {
160
+ // Refresh token usage window (rolling 60 seconds)
98
161
  ({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
99
162
 
100
- // Cool off check
101
- ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
102
- if(shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
163
+ // Adaptive cooldown if nearing provider rate limit
164
+ ({ totalTokensUsed, turnTimestamps, shouldBackoff } =
165
+ await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
166
+ if (shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
103
167
 
104
- //Tool Validator
168
+ // Verify message structure integrity before requesting next model step
105
169
  if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
106
170
  console.error('❌ Tool call structure invalid and no placeholders inserted.');
107
- currentStep.events.push('❌ Tool call structure invalid and no placeholders inserted.')
108
- currentStep.result = '❌ Failure';
109
- steps.push(currentStep);
171
+ step.events.push('❌ Tool call structure invalid and no placeholders inserted.');
172
+ step.result = '❌ Failure';
173
+ steps.push(step);
110
174
  return steps;
111
175
  }
112
176
 
177
+ // Detect orphaned assistant tool calls with no matching tool responses
113
178
  const unrespondedCalls = messages.filter(
114
179
  (msg, i) =>
115
180
  msg.role === 'assistant' &&
@@ -120,25 +185,30 @@ export const turnLoop = async (
120
185
  )
121
186
  )
122
187
  );
123
-
124
188
  if (unrespondedCalls.length) {
125
189
  console.error('🛑 Detected assistant tool calls without matching tool responses:');
126
190
  console.dir(unrespondedCalls, { depth: 5 });
127
- currentStep.events.push('🛑 Detected assistant tool calls without matching tool responses')
128
- currentStep.result = '❌ Failure';
129
- steps.push(currentStep);
191
+ step.events.push('🛑 Detected assistant tool calls without matching tool responses');
192
+ step.result = '❌ Failure';
193
+ steps.push(step);
130
194
  return steps;
131
195
  }
132
-
133
196
 
134
- response = await openai.chat.completions.create({
197
+ // ─────────────────────────────────────────────
198
+ // STEP 2: Request next reasoning turn from model
199
+ // ─────────────────────────────────────────────
200
+ const { message, usage, headers } = await llm.chat({
135
201
  model: MODEL_ID,
136
202
  messages,
137
203
  tools: toolsSchema,
138
204
  });
205
+ response = { message, usage, headers };
206
+
139
207
  } catch (err) {
208
+ // ─────────────────────────────────────────────
209
+ // STEP 3: Error and rate-limit handling
210
+ // ─────────────────────────────────────────────
140
211
  if (err.status === 429) {
141
- // If headers include limits, learn them for this model
142
212
  try { updateLimitsFromHeaders(MODEL_ID, err.headers || err.response?.headers || {}); } catch {}
143
213
  const delay = Math.min(60000, 2 ** retryCount * 2000);
144
214
  console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
@@ -146,105 +216,100 @@ export const turnLoop = async (
146
216
 
147
217
  if (retryCount >= 5) {
148
218
  console.error('❌ Too many retries. Exiting.');
149
- currentStep.events.push('❌ Too many retries. Exiting.')
150
- currentStep.result = '❌ Failure';
151
- steps.push(currentStep);
219
+ step.events.push('❌ Too many retries. Exiting.');
220
+ step.result = '❌ Failure';
221
+ steps.push(step);
152
222
  return steps;
153
223
  }
154
-
155
- return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, currentStep);
224
+ return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
156
225
  } else if (err.status === 400) {
157
226
  console.error('❌ Bad request:', err.message);
158
- // console.log("current tools", toolsSchema);
159
- // console.log("messages: ", messages)
160
- currentStep.events.push(`❌ Bad request: ${err.message}`)
161
- currentStep.result = '❌ Failure';
162
- steps.push(currentStep);
227
+ step.events.push(`❌ Bad request: ${err.message}`);
228
+ step.result = '❌ Failure';
229
+ steps.push(step);
163
230
  return steps;
164
231
  } else {
165
232
  throw err;
166
233
  }
167
234
  }
168
- console.log('Response received from OpenAI');
235
+
236
+ // ─────────────────────────────────────────────
237
+ // STEP 4: Process model response
238
+ // ─────────────────────────────────────────────
239
+ console.log(`Response received from ${PROVIDER_ID}`);
169
240
  const usage = response.usage;
170
241
  if (usage) {
171
242
  const tokensUsed = usage.total_tokens || 0;
172
243
  console.log(`📊 Token Usage This Turn → Total: ${tokensUsed}`);
173
- currentStep.tokensUsed = tokensUsed;
244
+ step.tokensUsed = tokensUsed;
174
245
  recordTokenUsage(turnTimestamps, tokensUsed);
175
246
  ({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
176
-
177
247
  console.log(`📈 Running Total Tokens Used (Rolling 60s): ${totalTokensUsed}`);
178
- currentStep.totalTokensUsed = totalTokensUsed;
248
+ step.totalTokensUsed = totalTokensUsed;
179
249
  }
180
-
181
- const msg = response.choices[0].message;
182
250
 
183
- const planLine = summarizeTurnIntentFromMessage(msg);
184
- currentStep.summary = planLine;
185
- currentStep.events.unshift(`📝 Plan: ${planLine}`);
186
- console.log(`📝 Plan: ${planLine}`);
251
+ const msg = response.message ?? { role: 'assistant', content: '' };
187
252
 
253
+ // Summarize model’s plan for the turn (plain + emoji variants)
254
+ const planPlain = summarizeTurnIntentFromMessage(msg, { emoji: true });
255
+ const planDisplay = summarizeTurnIntentFromMessage(msg);
256
+ step.summary = planPlain;
257
+ step.events.unshift(`📝 Plan: ${planDisplay}`);
258
+ console.log(`📝 Plan: ${planDisplay}`);
259
+
260
+ // ─────────────────────────────────────────────
261
+ // STEP 5: Handle tool/function calls
262
+ // ─────────────────────────────────────────────
188
263
  if (msg.tool_calls?.length) {
189
264
  console.log('Processing tool calls...');
190
265
  const toolResponses = [];
191
- let lastToolName = null;
192
266
 
193
267
  for (const call of msg.tool_calls) {
194
268
  const fnName = call.function.name;
195
- lastToolName = fnName;
196
269
  const args = JSON.parse(call.function.arguments || '{}');
197
270
  const safeArgs = redactArgs(fnName, args);
198
271
  console.log(`[model] → ${fnName}`, safeArgs);
199
- currentStep.events.push(`[model] → ${fnName} ${safeArgs}`);
200
- // console.log('Calling tool:', fnName);
272
+ step.events.push(`[model] → ${fnName} ${safeArgs}`);
273
+
201
274
  let result;
202
275
  let errorMessage = null;
203
276
  try {
204
- // console.log("agentMemory: ", agentMemory);
205
277
  result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
206
-
207
- if (typeof result !== 'string') {
208
- result = JSON.stringify(result ?? '');
209
- }
210
- } catch (err) {
211
- errorMessage = `ERROR: ${err.message}`;
278
+ if (typeof result !== 'string') result = JSON.stringify(result ?? '');
279
+ } catch (e) {
280
+ errorMessage = `ERROR: ${e.message}`;
212
281
  result = errorMessage;
213
282
  }
214
283
 
215
- // ------ Capture file upload/download events for mission report ------
284
+ // Capture and log any file upload/download events
216
285
  try {
217
286
  const maybeJson = JSON.parse(result);
218
287
  if (maybeJson && maybeJson._testronaut_file_event) {
219
- currentStep.files = currentStep.files || [];
220
- currentStep.files.push(maybeJson);
221
-
222
- if (maybeJson._testronaut_file_event === 'upload') {
223
- const msgLine = `📤 Uploaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.method}`;
224
- currentStep.events.push(msgLine);
225
- console.log(msgLine);
226
- } else if (maybeJson._testronaut_file_event === 'download') {
227
- const msgLine = `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
228
- currentStep.events.push(msgLine);
229
- console.log(msgLine);
230
- }
288
+ step.files = step.files || [];
289
+ step.files.push(maybeJson);
290
+ const msgLine = maybeJson._testronaut_file_event === 'upload'
291
+ ? `📤 Uploaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.method}`
292
+ : `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
293
+ step.events.push(msgLine);
294
+ console.log(msgLine);
231
295
  }
232
- } catch { /* not JSON / ignore */ }
296
+ } catch { /* non-JSON results ignored */ }
233
297
 
234
298
  console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
235
- currentStep.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`)
236
299
  const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
237
- console.log(`[tool ] ← ${truncated}`);
300
+ step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
301
+ step.events.push(`[tool ] ← ${truncated}`);
238
302
 
303
+ // Screenshot detection
239
304
  if (fnName === 'screenshot') {
240
305
  const match = result.match(/screenshot.*?saved at: (.+\.png)/i);
241
306
  if (match && match[1]) {
242
- currentStep.screenshotPath = match[1];
243
- currentStep.events.push(`🖼️ Screenshot captured: ${match[1]}`);
307
+ step.screenshotPath = match[1];
308
+ step.events.push(`🖼️ Screenshot captured: ${match[1]}`);
244
309
  }
245
310
  }
246
-
247
- currentStep.events.push(`[tool ] ← ${truncated}`)
311
+
312
+ // Push corresponding tool message back to LLM
248
313
  toolResponses.push({
249
314
  role: 'tool',
250
315
  tool_call_id: call.id,
@@ -253,13 +318,10 @@ export const turnLoop = async (
253
318
  content: result,
254
319
  });
255
320
 
256
- if (fnName === 'expand_menu') {
257
- agentMemory.lastMenuExpanded = true;
258
- }
259
-
260
- if (fnName === 'click_text' || fnName === 'click' || fnName === 'expand_menu') {
261
- console.log(`[auto] → Injecting DOM after menu expand...`);
262
- currentStep.events.push(`[auto] → Injecting DOM after menu expand...`);
321
+ // After interactive DOM actions, refresh model context
322
+ if (['click_text', 'click', 'expand_menu'].includes(fnName)) {
323
+ console.log(`[auto] → Injecting DOM after ${fnName}...`);
324
+ step.events.push(`[auto] → Injecting DOM after ${fnName}...`);
263
325
  const domHtml = await CHROME_TOOL_MAP.get_dom(browser, {
264
326
  limit: 100000,
265
327
  exclude: true,
@@ -267,33 +329,38 @@ export const turnLoop = async (
267
329
  }, agentMemory);
268
330
  await tokenEstimate(MODEL_ID, domHtml);
269
331
  await pushDOMAssistant(browser, messages, agentMemory, {
270
- skipIfLastTool: ['get_dom', 'check_text']
332
+ skipIfLastTool: ['get_dom', 'check_text'],
271
333
  });
272
334
  console.log(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
273
- currentStep.events.push(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`)
335
+ step.events.push(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
274
336
  }
275
-
276
337
  }
338
+
339
+ // Merge new assistant + tool responses back into conversation
277
340
  messages.push(msg, ...toolResponses);
278
- currentStep.result = '✅ Passed';
279
- steps.push(currentStep);
341
+ step.result = '✅ Passed';
342
+ steps.push(step);
280
343
  continue;
281
344
  }
282
345
 
346
+ // ─────────────────────────────────────────────
347
+ // STEP 6: Detect final mission state
348
+ // ─────────────────────────────────────────────
283
349
  const finalResponse = finalResponseHandler(msg);
284
350
  if (finalResponse !== null) {
285
- currentStep.events.push(finalResponse.finalMessage);
286
- currentStep.result = finalResponse.success ? '✅ Mission Success': '❌ Mission Failure';
287
- steps.push(currentStep);
351
+ step.events.push(finalResponse.finalMessage);
352
+ step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
353
+ steps.push(step);
288
354
  return { success: finalResponse.finalMessage, steps };
289
355
  }
290
356
 
291
-
292
- await pushDOMAssistant(browser, messages, agentMemory,{
293
- skipIfLastTool: ['get_dom', 'check_text']
357
+ // ─────────────────────────────────────────────
358
+ // STEP 7: Fallback → Push DOM for next reasoning cycle
359
+ // ─────────────────────────────────────────────
360
+ await pushDOMAssistant(browser, messages, agentMemory, {
361
+ skipIfLastTool: ['get_dom', 'check_text'],
294
362
  });
295
-
296
363
  console.log(`[auto] → Injected DOM for next reasoning step`);
297
- currentStep.events.push(`[auto] → Injected DOM for next reasoning step`)
364
+ step.events.push(`[auto] → Injected DOM for next reasoning step`);
298
365
  }
299
- }
366
+ };