testronaut 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +194 -0
- package/bin/cli.js +1 -1
- package/bin/init.js +141 -57
- package/bin/initHelpers.js +140 -0
- package/core/agent.js +1 -1
- package/core/turnLoop.js +170 -107
- package/llm/gemini/geminiProvider.js +188 -0
- package/llm/llmFactory.js +50 -0
- package/llm/modelResolver.js +65 -0
- package/llm/openAI/openaiProvider.js +54 -0
- package/package.json +9 -3
- package/tests/binTests/init.test.js +189 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +12 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +14 -0
- package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js +19 -0
- package/tests/binTests/initHelpersTests/models.test.js +21 -0
- package/tests/coreTests/turnLoop.test.js +177 -0
- package/tests/llmTests/geminiProvider.test.js +191 -0
- package/tests/llmTests/llmFactory.test.js +78 -0
- package/tests/llmTests/modelResolver.test.js +119 -0
- package/tests/llmTests/openAIProvider.test.js +117 -0
- package/tests/toolsTests/tokenControl.test.js +150 -0
- package/tools/tokenControl.js +107 -17
- package/vitest.config.js +8 -0
- package/openAI/modelResolver.js +0 -33
package/core/turnLoop.js
CHANGED
|
@@ -1,4 +1,33 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* turnLoop.js
|
|
3
|
+
* ------------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Core execution loop for Testronaut's autonomous agent testing framework.
|
|
6
|
+
* Handles reasoning turns, model responses, tool execution, DOM updates,
|
|
7
|
+
* backoff logic, and final mission success/failure detection.
|
|
8
|
+
*
|
|
9
|
+
* Responsibilities:
|
|
10
|
+
* - Coordinate between browser (Playwright/Puppeteer) and chosen LLM.
|
|
11
|
+
* - Maintain conversation history and agent "memory" between turns.
|
|
12
|
+
* - Detect and execute tool/function calls emitted by the model.
|
|
13
|
+
* - Track and enforce token-per-minute rate limits with adaptive cooldowns.
|
|
14
|
+
* - Handle DOM re-injection after browser actions to support reasoning continuity.
|
|
15
|
+
* - Summarize each turn’s intent and record detailed step logs.
|
|
16
|
+
*
|
|
17
|
+
* Related tests:
|
|
18
|
+
* Located in `tests/coreTests/`
|
|
19
|
+
* (see integration tests covering multi-turn reasoning, tool calling, and backoff).
|
|
20
|
+
*
|
|
21
|
+
* Used by:
|
|
22
|
+
* - CLI mission runners (`missionRunner.js`, `runMission.js`)
|
|
23
|
+
* - Autonomous agent framework during mission playback.
|
|
24
|
+
*
|
|
25
|
+
* Notes:
|
|
26
|
+
* - Provider-agnostic: routes all LLM calls through the `llmFactory` adapter layer.
|
|
27
|
+
* - OpenAI and Gemini both normalize responses to an OpenAI-like message format.
|
|
28
|
+
* - Token control is model-sensitive but provider-neutral.
|
|
29
|
+
*/
|
|
30
|
+
|
|
2
31
|
import toolsSchema from '../tools/toolSchema.js';
|
|
3
32
|
import { CHROME_TOOL_MAP } from '../tools/chromeBrowser.js';
|
|
4
33
|
import fs from 'fs';
|
|
@@ -12,24 +41,31 @@ import {
|
|
|
12
41
|
tokenUseCoolOff,
|
|
13
42
|
recordTokenUsage,
|
|
14
43
|
pruneOldTokenUsage,
|
|
15
|
-
getCurrentTokenLimit,
|
|
16
44
|
updateLimitsFromHeaders
|
|
17
45
|
} from '../tools/tokenControl.js';
|
|
18
|
-
import {
|
|
46
|
+
import { resolveProviderModel } from '../llm/modelResolver.js';
|
|
47
|
+
import { getLLM } from '../llm/llmFactory.js';
|
|
19
48
|
import { summarizeTurnIntentFromMessage } from './turnIntent.js';
|
|
20
49
|
import { redactArgs } from './redaction.js';
|
|
21
50
|
|
|
51
|
+
// ─────────────────────────────────────────────
|
|
52
|
+
// STEP 0: Resolve provider + model and initialize adapter
|
|
53
|
+
// ─────────────────────────────────────────────
|
|
54
|
+
const { provider: PROVIDER_ID, model: MODEL_ID } = resolveProviderModel();
|
|
55
|
+
console.log(`🧠 Using LLM → provider: ${PROVIDER_ID}, model: ${MODEL_ID}`);
|
|
22
56
|
|
|
23
|
-
const
|
|
24
|
-
console.log(`🧠 Using OpenAI model: ${MODEL_ID}`);
|
|
25
|
-
|
|
26
|
-
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
57
|
+
const llm = getLLM(PROVIDER_ID);
|
|
27
58
|
|
|
59
|
+
// Rolling token counters used for self-throttling
|
|
28
60
|
let totalTokensUsed = 0;
|
|
29
61
|
let turnTimestamps = [];
|
|
30
62
|
let shouldBackoff;
|
|
31
63
|
let steps = [];
|
|
32
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Utility: format byte counts into human-readable strings.
|
|
67
|
+
* Used when reporting upload/download events in mission logs.
|
|
68
|
+
*/
|
|
33
69
|
function formatBytes(n) {
|
|
34
70
|
if (!Number.isFinite(n)) return `${n}`;
|
|
35
71
|
const u = ['B','KB','MB','GB','TB'];
|
|
@@ -38,7 +74,17 @@ function formatBytes(n) {
|
|
|
38
74
|
return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`;
|
|
39
75
|
}
|
|
40
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Pushes a new "get_dom" tool call and result pair into the message history.
|
|
79
|
+
* This keeps the model updated on the browser's DOM after actions like click/expand.
|
|
80
|
+
*
|
|
81
|
+
* @param {object} browser - The browser automation instance.
|
|
82
|
+
* @param {array} messages - The running conversation message array.
|
|
83
|
+
* @param {object} agentMemory - Memory object storing recent agent state.
|
|
84
|
+
* @param {{skipIfLastTool?: string[]}} opts - Optional tool skip conditions.
|
|
85
|
+
*/
|
|
41
86
|
const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool } = {}) => {
|
|
87
|
+
// Avoid redundant DOM pushes immediately following tools that already include DOM context.
|
|
42
88
|
if (skipIfLastTool && skipIfLastTool.includes(messages.at(-1)?.name)) {
|
|
43
89
|
console.log(`[skip] Skipping DOM push after redundant tool: ${messages.at(-1)?.name}`);
|
|
44
90
|
return;
|
|
@@ -46,6 +92,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
46
92
|
|
|
47
93
|
const domCallId = `get_dom_${Date.now()}`;
|
|
48
94
|
|
|
95
|
+
// Inject a synthetic assistant tool call
|
|
49
96
|
messages.push({
|
|
50
97
|
role: 'assistant',
|
|
51
98
|
tool_calls: [
|
|
@@ -61,9 +108,9 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
61
108
|
});
|
|
62
109
|
|
|
63
110
|
const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
|
|
64
|
-
await tokenEstimate(
|
|
65
|
-
// fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
|
|
111
|
+
await tokenEstimate(MODEL_ID, domHtml);
|
|
66
112
|
|
|
113
|
+
// Push the corresponding tool response back into the conversation
|
|
67
114
|
messages.push({
|
|
68
115
|
role: 'tool',
|
|
69
116
|
tool_call_id: domCallId,
|
|
@@ -71,8 +118,24 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
|
|
|
71
118
|
type: 'function',
|
|
72
119
|
content: typeof domHtml === 'string' ? domHtml : JSON.stringify(domHtml),
|
|
73
120
|
});
|
|
74
|
-
}
|
|
75
|
-
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Main turn loop driver.
|
|
125
|
+
*
|
|
126
|
+
* Runs the agent through iterative reasoning turns until:
|
|
127
|
+
* - Max turn count is reached, or
|
|
128
|
+
* - The model emits a "final" message (success/failure).
|
|
129
|
+
*
|
|
130
|
+
* @param {object} browser - Browser automation interface (Playwright/Puppeteer).
|
|
131
|
+
* @param {array} messages - Conversation messages so far.
|
|
132
|
+
* @param {number} maxTurns - Maximum allowed reasoning turns.
|
|
133
|
+
* @param {number} currentTurn - Turn index to start from (default 0).
|
|
134
|
+
* @param {number} retryCount - Internal retry counter for rate limiting.
|
|
135
|
+
* @param {object} currentStep - Step data being populated for the current turn.
|
|
136
|
+
* @param {object} ctx - Shared context (steps array, mission name).
|
|
137
|
+
* @returns {Promise<object[]>} - Collected step results or final success message.
|
|
138
|
+
*/
|
|
76
139
|
export const turnLoop = async (
|
|
77
140
|
browser,
|
|
78
141
|
messages,
|
|
@@ -80,36 +143,38 @@ export const turnLoop = async (
|
|
|
80
143
|
currentTurn = 0,
|
|
81
144
|
retryCount = 0,
|
|
82
145
|
currentStep = {},
|
|
83
|
-
ctx = {} //
|
|
146
|
+
ctx = {} // { steps, missionName }
|
|
84
147
|
) => {
|
|
85
148
|
const { steps = [], missionName } = ctx;
|
|
86
|
-
let agentMemory = {
|
|
87
|
-
lastMenuExpanded: false
|
|
88
|
-
};
|
|
149
|
+
let agentMemory = { lastMenuExpanded: false };
|
|
89
150
|
|
|
151
|
+
// ─────────────────────────────────────────────
|
|
152
|
+
// STEP 1: Begin reasoning cycle
|
|
153
|
+
// ─────────────────────────────────────────────
|
|
90
154
|
for (let turn = currentTurn; turn < maxTurns; turn++) {
|
|
91
155
|
console.log(`\n🔄 Turn ${turn + 1}/${maxTurns}`);
|
|
92
156
|
let response;
|
|
93
|
-
|
|
94
|
-
const currentStep = { turn, events: [], result: '🟡 In Progress', missionName }; // 👈 tag it
|
|
95
|
-
try {
|
|
157
|
+
const step = { turn, events: [], result: '🟡 In Progress', missionName };
|
|
96
158
|
|
|
97
|
-
|
|
159
|
+
try {
|
|
160
|
+
// Refresh token usage window (rolling 60 seconds)
|
|
98
161
|
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
99
162
|
|
|
100
|
-
//
|
|
101
|
-
({ totalTokensUsed, turnTimestamps, shouldBackoff } =
|
|
102
|
-
|
|
163
|
+
// Adaptive cooldown if nearing provider rate limit
|
|
164
|
+
({ totalTokensUsed, turnTimestamps, shouldBackoff } =
|
|
165
|
+
await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
|
|
166
|
+
if (shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
|
|
103
167
|
|
|
104
|
-
//
|
|
168
|
+
// Verify message structure integrity before requesting next model step
|
|
105
169
|
if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
|
|
106
170
|
console.error('❌ Tool call structure invalid and no placeholders inserted.');
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
steps.push(
|
|
171
|
+
step.events.push('❌ Tool call structure invalid and no placeholders inserted.');
|
|
172
|
+
step.result = '❌ Failure';
|
|
173
|
+
steps.push(step);
|
|
110
174
|
return steps;
|
|
111
175
|
}
|
|
112
176
|
|
|
177
|
+
// Detect orphaned assistant tool calls with no matching tool responses
|
|
113
178
|
const unrespondedCalls = messages.filter(
|
|
114
179
|
(msg, i) =>
|
|
115
180
|
msg.role === 'assistant' &&
|
|
@@ -120,25 +185,30 @@ export const turnLoop = async (
|
|
|
120
185
|
)
|
|
121
186
|
)
|
|
122
187
|
);
|
|
123
|
-
|
|
124
188
|
if (unrespondedCalls.length) {
|
|
125
189
|
console.error('🛑 Detected assistant tool calls without matching tool responses:');
|
|
126
190
|
console.dir(unrespondedCalls, { depth: 5 });
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
steps.push(
|
|
191
|
+
step.events.push('🛑 Detected assistant tool calls without matching tool responses');
|
|
192
|
+
step.result = '❌ Failure';
|
|
193
|
+
steps.push(step);
|
|
130
194
|
return steps;
|
|
131
195
|
}
|
|
132
|
-
|
|
133
196
|
|
|
134
|
-
|
|
197
|
+
// ─────────────────────────────────────────────
|
|
198
|
+
// STEP 2: Request next reasoning turn from model
|
|
199
|
+
// ─────────────────────────────────────────────
|
|
200
|
+
const { message, usage, headers } = await llm.chat({
|
|
135
201
|
model: MODEL_ID,
|
|
136
202
|
messages,
|
|
137
203
|
tools: toolsSchema,
|
|
138
204
|
});
|
|
205
|
+
response = { message, usage, headers };
|
|
206
|
+
|
|
139
207
|
} catch (err) {
|
|
208
|
+
// ─────────────────────────────────────────────
|
|
209
|
+
// STEP 3: Error and rate-limit handling
|
|
210
|
+
// ─────────────────────────────────────────────
|
|
140
211
|
if (err.status === 429) {
|
|
141
|
-
// If headers include limits, learn them for this model
|
|
142
212
|
try { updateLimitsFromHeaders(MODEL_ID, err.headers || err.response?.headers || {}); } catch {}
|
|
143
213
|
const delay = Math.min(60000, 2 ** retryCount * 2000);
|
|
144
214
|
console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
|
|
@@ -146,109 +216,100 @@ export const turnLoop = async (
|
|
|
146
216
|
|
|
147
217
|
if (retryCount >= 5) {
|
|
148
218
|
console.error('❌ Too many retries. Exiting.');
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
steps.push(
|
|
219
|
+
step.events.push('❌ Too many retries. Exiting.');
|
|
220
|
+
step.result = '❌ Failure';
|
|
221
|
+
steps.push(step);
|
|
152
222
|
return steps;
|
|
153
223
|
}
|
|
154
|
-
|
|
155
|
-
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, currentStep);
|
|
224
|
+
return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
|
|
156
225
|
} else if (err.status === 400) {
|
|
157
226
|
console.error('❌ Bad request:', err.message);
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
currentStep.result = '❌ Failure';
|
|
162
|
-
steps.push(currentStep);
|
|
227
|
+
step.events.push(`❌ Bad request: ${err.message}`);
|
|
228
|
+
step.result = '❌ Failure';
|
|
229
|
+
steps.push(step);
|
|
163
230
|
return steps;
|
|
164
231
|
} else {
|
|
165
232
|
throw err;
|
|
166
233
|
}
|
|
167
234
|
}
|
|
168
|
-
|
|
235
|
+
|
|
236
|
+
// ─────────────────────────────────────────────
|
|
237
|
+
// STEP 4: Process model response
|
|
238
|
+
// ─────────────────────────────────────────────
|
|
239
|
+
console.log(`Response received from ${PROVIDER_ID}`);
|
|
169
240
|
const usage = response.usage;
|
|
170
241
|
if (usage) {
|
|
171
242
|
const tokensUsed = usage.total_tokens || 0;
|
|
172
243
|
console.log(`📊 Token Usage This Turn → Total: ${tokensUsed}`);
|
|
173
|
-
|
|
244
|
+
step.tokensUsed = tokensUsed;
|
|
174
245
|
recordTokenUsage(turnTimestamps, tokensUsed);
|
|
175
246
|
({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
|
|
176
|
-
|
|
177
247
|
console.log(`📈 Running Total Tokens Used (Rolling 60s): ${totalTokensUsed}`);
|
|
178
|
-
|
|
248
|
+
step.totalTokensUsed = totalTokensUsed;
|
|
179
249
|
}
|
|
180
|
-
|
|
181
|
-
const msg = response.choices[0].message;
|
|
182
250
|
|
|
183
|
-
|
|
184
|
-
const planPlain = summarizeTurnIntentFromMessage(msg, { emoji: true });
|
|
185
|
-
// Emoji (toggleable) for console/event log:
|
|
186
|
-
const planDisplay = summarizeTurnIntentFromMessage(msg, { /* emoji default via env */ });
|
|
251
|
+
const msg = response.message ?? { role: 'assistant', content: '' };
|
|
187
252
|
|
|
188
|
-
|
|
189
|
-
|
|
253
|
+
// Summarize model’s plan for the turn (plain + emoji variants)
|
|
254
|
+
const planPlain = summarizeTurnIntentFromMessage(msg, { emoji: true });
|
|
255
|
+
const planDisplay = summarizeTurnIntentFromMessage(msg);
|
|
256
|
+
step.summary = planPlain;
|
|
257
|
+
step.events.unshift(`📝 Plan: ${planDisplay}`);
|
|
190
258
|
console.log(`📝 Plan: ${planDisplay}`);
|
|
191
259
|
|
|
260
|
+
// ─────────────────────────────────────────────
|
|
261
|
+
// STEP 5: Handle tool/function calls
|
|
262
|
+
// ─────────────────────────────────────────────
|
|
192
263
|
if (msg.tool_calls?.length) {
|
|
193
264
|
console.log('Processing tool calls...');
|
|
194
265
|
const toolResponses = [];
|
|
195
|
-
let lastToolName = null;
|
|
196
266
|
|
|
197
267
|
for (const call of msg.tool_calls) {
|
|
198
268
|
const fnName = call.function.name;
|
|
199
|
-
lastToolName = fnName;
|
|
200
269
|
const args = JSON.parse(call.function.arguments || '{}');
|
|
201
270
|
const safeArgs = redactArgs(fnName, args);
|
|
202
271
|
console.log(`[model] → ${fnName}`, safeArgs);
|
|
203
|
-
|
|
204
|
-
|
|
272
|
+
step.events.push(`[model] → ${fnName} ${safeArgs}`);
|
|
273
|
+
|
|
205
274
|
let result;
|
|
206
275
|
let errorMessage = null;
|
|
207
276
|
try {
|
|
208
|
-
// console.log("agentMemory: ", agentMemory);
|
|
209
277
|
result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
}
|
|
214
|
-
} catch (err) {
|
|
215
|
-
errorMessage = `ERROR: ${err.message}`;
|
|
278
|
+
if (typeof result !== 'string') result = JSON.stringify(result ?? '');
|
|
279
|
+
} catch (e) {
|
|
280
|
+
errorMessage = `ERROR: ${e.message}`;
|
|
216
281
|
result = errorMessage;
|
|
217
282
|
}
|
|
218
283
|
|
|
219
|
-
//
|
|
284
|
+
// Capture and log any file upload/download events
|
|
220
285
|
try {
|
|
221
286
|
const maybeJson = JSON.parse(result);
|
|
222
287
|
if (maybeJson && maybeJson._testronaut_file_event) {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
} else if (maybeJson._testronaut_file_event === 'download') {
|
|
231
|
-
const msgLine = `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
|
|
232
|
-
currentStep.events.push(msgLine);
|
|
233
|
-
console.log(msgLine);
|
|
234
|
-
}
|
|
288
|
+
step.files = step.files || [];
|
|
289
|
+
step.files.push(maybeJson);
|
|
290
|
+
const msgLine = maybeJson._testronaut_file_event === 'upload'
|
|
291
|
+
? `📤 Uploaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.method}`
|
|
292
|
+
: `📥 Downloaded "${maybeJson.fileName}" (${formatBytes(maybeJson.bytes)}) via ${maybeJson.mode}`;
|
|
293
|
+
step.events.push(msgLine);
|
|
294
|
+
console.log(msgLine);
|
|
235
295
|
}
|
|
236
|
-
} catch { /*
|
|
296
|
+
} catch { /* non-JSON results ignored */ }
|
|
237
297
|
|
|
238
298
|
console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
|
|
239
|
-
currentStep.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`)
|
|
240
299
|
const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
|
|
241
|
-
|
|
300
|
+
step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
|
|
301
|
+
step.events.push(`[tool ] ← ${truncated}`);
|
|
242
302
|
|
|
303
|
+
// Screenshot detection
|
|
243
304
|
if (fnName === 'screenshot') {
|
|
244
305
|
const match = result.match(/screenshot.*?saved at: (.+\.png)/i);
|
|
245
306
|
if (match && match[1]) {
|
|
246
|
-
|
|
247
|
-
|
|
307
|
+
step.screenshotPath = match[1];
|
|
308
|
+
step.events.push(`🖼️ Screenshot captured: ${match[1]}`);
|
|
248
309
|
}
|
|
249
310
|
}
|
|
250
|
-
|
|
251
|
-
|
|
311
|
+
|
|
312
|
+
// Push corresponding tool message back to LLM
|
|
252
313
|
toolResponses.push({
|
|
253
314
|
role: 'tool',
|
|
254
315
|
tool_call_id: call.id,
|
|
@@ -257,13 +318,10 @@ export const turnLoop = async (
|
|
|
257
318
|
content: result,
|
|
258
319
|
});
|
|
259
320
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
if (fnName === 'click_text' || fnName === 'click' || fnName === 'expand_menu') {
|
|
265
|
-
console.log(`[auto] → Injecting DOM after menu expand...`);
|
|
266
|
-
currentStep.events.push(`[auto] → Injecting DOM after menu expand...`);
|
|
321
|
+
// After interactive DOM actions, refresh model context
|
|
322
|
+
if (['click_text', 'click', 'expand_menu'].includes(fnName)) {
|
|
323
|
+
console.log(`[auto] → Injecting DOM after ${fnName}...`);
|
|
324
|
+
step.events.push(`[auto] → Injecting DOM after ${fnName}...`);
|
|
267
325
|
const domHtml = await CHROME_TOOL_MAP.get_dom(browser, {
|
|
268
326
|
limit: 100000,
|
|
269
327
|
exclude: true,
|
|
@@ -271,33 +329,38 @@ export const turnLoop = async (
|
|
|
271
329
|
}, agentMemory);
|
|
272
330
|
await tokenEstimate(MODEL_ID, domHtml);
|
|
273
331
|
await pushDOMAssistant(browser, messages, agentMemory, {
|
|
274
|
-
skipIfLastTool: ['get_dom', 'check_text']
|
|
332
|
+
skipIfLastTool: ['get_dom', 'check_text'],
|
|
275
333
|
});
|
|
276
334
|
console.log(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
|
|
277
|
-
|
|
335
|
+
step.events.push(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
|
|
278
336
|
}
|
|
279
|
-
|
|
280
337
|
}
|
|
338
|
+
|
|
339
|
+
// Merge new assistant + tool responses back into conversation
|
|
281
340
|
messages.push(msg, ...toolResponses);
|
|
282
|
-
|
|
283
|
-
steps.push(
|
|
341
|
+
step.result = '✅ Passed';
|
|
342
|
+
steps.push(step);
|
|
284
343
|
continue;
|
|
285
344
|
}
|
|
286
345
|
|
|
346
|
+
// ─────────────────────────────────────────────
|
|
347
|
+
// STEP 6: Detect final mission state
|
|
348
|
+
// ─────────────────────────────────────────────
|
|
287
349
|
const finalResponse = finalResponseHandler(msg);
|
|
288
350
|
if (finalResponse !== null) {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
steps.push(
|
|
351
|
+
step.events.push(finalResponse.finalMessage);
|
|
352
|
+
step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
|
|
353
|
+
steps.push(step);
|
|
292
354
|
return { success: finalResponse.finalMessage, steps };
|
|
293
355
|
}
|
|
294
356
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
357
|
+
// ─────────────────────────────────────────────
|
|
358
|
+
// STEP 7: Fallback → Push DOM for next reasoning cycle
|
|
359
|
+
// ─────────────────────────────────────────────
|
|
360
|
+
await pushDOMAssistant(browser, messages, agentMemory, {
|
|
361
|
+
skipIfLastTool: ['get_dom', 'check_text'],
|
|
298
362
|
});
|
|
299
|
-
|
|
300
363
|
console.log(`[auto] → Injected DOM for next reasoning step`);
|
|
301
|
-
|
|
364
|
+
step.events.push(`[auto] → Injected DOM for next reasoning step`);
|
|
302
365
|
}
|
|
303
|
-
}
|
|
366
|
+
};
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* geminiProvider.js
|
|
3
|
+
* ------------------
|
|
4
|
+
* Purpose:
|
|
5
|
+
* Adapter that normalizes Testronaut's OpenAI-like chat format to
|
|
6
|
+
* Google Gemini's SDK and back. Exposes a single `chat()` method.
|
|
7
|
+
*
|
|
8
|
+
* Responsibilities:
|
|
9
|
+
* - Convert OpenAI-like messages → Gemini "contents" (roles/parts).
|
|
10
|
+
* - Map tool/function calling both ways:
|
|
11
|
+
* • assistant.tool_calls → Gemini functionCall parts
|
|
12
|
+
* • tool messages → user parts (so the model can read results)
|
|
13
|
+
* - Return an OpenAI-like assistant message and usage metadata.
|
|
14
|
+
*
|
|
15
|
+
* Message contract (OpenAI-like, internal):
|
|
16
|
+
* - messages: Array<{ role: 'system'|'user'|'assistant'|'tool', content?: string|Array, ... }>
|
|
17
|
+
* - Assistant tool calls:
|
|
18
|
+
* message.tool_calls = [{ id, type:'function', function:{ name, arguments:string }}]
|
|
19
|
+
* - Tool messages:
|
|
20
|
+
* { role:'tool', tool_call_id, name, type:'function', content:string }
|
|
21
|
+
*
|
|
22
|
+
* Related tests:
|
|
23
|
+
* Located in `tests/llmTests/geminiProvider.test.js`
|
|
24
|
+
*
|
|
25
|
+
* Used by:
|
|
26
|
+
* - llm/llmFactory.js
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Convert OpenAI-like messages → Gemini "contents" array.
|
|
33
|
+
* - System messages are coalesced and injected as a prefix into the next user turn.
|
|
34
|
+
* - Assistant tool calls are represented as model turns with functionCall parts.
|
|
35
|
+
* - Tool results are encoded as a user turn with a JSON payload part.
|
|
36
|
+
* - Text/images are mapped to Gemini `parts` (text / inlineData).
|
|
37
|
+
*/
|
|
38
|
+
function toGeminiContents(messages) {
|
|
39
|
+
const contents = [];
|
|
40
|
+
let systemPrefix = '';
|
|
41
|
+
|
|
42
|
+
for (const m of messages) {
|
|
43
|
+
if (m.role === 'system') {
|
|
44
|
+
// Collect multi-system messages; inject once on the next user message
|
|
45
|
+
const sysText = Array.isArray(m.content)
|
|
46
|
+
? m.content.map(p => (typeof p === 'string' ? p : p.text || '')).join('\n')
|
|
47
|
+
: (m.content || '');
|
|
48
|
+
systemPrefix += (systemPrefix ? '\n' : '') + sysText;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Assistant tool calls → Gemini functionCall parts on a model turn
|
|
53
|
+
if (m.role === 'assistant' && Array.isArray(m.tool_calls) && m.tool_calls.length) {
|
|
54
|
+
contents.push({
|
|
55
|
+
role: 'model',
|
|
56
|
+
parts: m.tool_calls.map(tc => ({
|
|
57
|
+
functionCall: {
|
|
58
|
+
name: tc.function?.name,
|
|
59
|
+
args: safeJsonParse(tc.function?.arguments) ?? {},
|
|
60
|
+
},
|
|
61
|
+
})),
|
|
62
|
+
});
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Tool results → encode as a user turn with a JSON part the model can parse
|
|
67
|
+
if (m.role === 'tool') {
|
|
68
|
+
const parts = [{
|
|
69
|
+
text: JSON.stringify({
|
|
70
|
+
_tool_result: true,
|
|
71
|
+
tool_call_id: m.tool_call_id,
|
|
72
|
+
name: m.name,
|
|
73
|
+
content: m.content,
|
|
74
|
+
}),
|
|
75
|
+
}];
|
|
76
|
+
contents.push({ role: 'user', parts });
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Normal user/assistant content (text and/or images)
|
|
81
|
+
const baseParts = [];
|
|
82
|
+
const asArray = Array.isArray(m.content) ? m.content : [{ type: 'text', text: m.content }];
|
|
83
|
+
for (const p of asArray) {
|
|
84
|
+
if (p?.type === 'image') {
|
|
85
|
+
baseParts.push({
|
|
86
|
+
inlineData: {
|
|
87
|
+
mimeType: p.mimeType,
|
|
88
|
+
data: Buffer.from(p.data).toString('base64'),
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
} else {
|
|
92
|
+
baseParts.push({ text: typeof p === 'string' ? p : (p.text ?? '') });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Include system prefix exactly once on the first user turn after system
|
|
97
|
+
if (systemPrefix && m.role === 'user') {
|
|
98
|
+
baseParts.unshift({ text: `[system]\n${systemPrefix}` });
|
|
99
|
+
systemPrefix = '';
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
contents.push({ role: m.role === 'assistant' ? 'model' : 'user', parts: baseParts });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return contents;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function safeJsonParse(s) {
|
|
109
|
+
try { return JSON.parse(s ?? '{}'); } catch { return null; }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Convert a Gemini candidate → OpenAI-like assistant message.
|
|
114
|
+
* - Collects text parts into `content`.
|
|
115
|
+
* - Translates functionCall parts into `tool_calls`.
|
|
116
|
+
*/
|
|
117
|
+
function fromGeminiCandidate(cand) {
|
|
118
|
+
const parts = cand?.content?.parts ?? [];
|
|
119
|
+
const tool_calls = [];
|
|
120
|
+
const texts = [];
|
|
121
|
+
|
|
122
|
+
for (const p of parts) {
|
|
123
|
+
if (p.functionCall) {
|
|
124
|
+
tool_calls.push({
|
|
125
|
+
id: cryptoRandomId(),
|
|
126
|
+
type: 'function',
|
|
127
|
+
function: {
|
|
128
|
+
name: p.functionCall.name,
|
|
129
|
+
arguments: JSON.stringify(p.functionCall.args ?? {}),
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
} else if (typeof p.text === 'string') {
|
|
133
|
+
texts.push(p.text);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const content = texts.join('');
|
|
138
|
+
return { role: 'assistant', content, tool_calls: tool_calls.length ? tool_calls : undefined };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function cryptoRandomId() {
|
|
142
|
+
return 'tool_' + Math.random().toString(36).slice(2);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export class GeminiProvider {
|
|
146
|
+
constructor({ apiKey } = {}) {
|
|
147
|
+
if (!apiKey) throw new Error('GEMINI_API_KEY is required for Gemini provider');
|
|
148
|
+
this.client = new GoogleGenerativeAI(apiKey);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Execute a chat turn via Gemini and normalize the response.
|
|
153
|
+
* @param {{model:string, messages:any[], tools?:any[]}} params
|
|
154
|
+
* @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}}>}
|
|
155
|
+
*/
|
|
156
|
+
async chat({ model, messages, tools }) {
|
|
157
|
+
// Map OpenAI-like tool schema → Gemini functionDeclarations
|
|
158
|
+
const genTools = tools?.length
|
|
159
|
+
? [{
|
|
160
|
+
functionDeclarations: tools.map(t => ({
|
|
161
|
+
name: t.function?.name ?? t.name,
|
|
162
|
+
description: t.description ?? '',
|
|
163
|
+
parameters: t.function?.parameters ?? t.parameters ?? {}, // JSON schema
|
|
164
|
+
})),
|
|
165
|
+
}]
|
|
166
|
+
: undefined;
|
|
167
|
+
|
|
168
|
+
const gmodel = this.client.getGenerativeModel({
|
|
169
|
+
model,
|
|
170
|
+
tools: genTools,
|
|
171
|
+
generationConfig: {}, // temperature/topP can be added upstream if needed
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const contents = toGeminiContents(messages);
|
|
175
|
+
const res = await gmodel.generateContent({ contents });
|
|
176
|
+
|
|
177
|
+
const cand = res?.response?.candidates?.[0];
|
|
178
|
+
const message = fromGeminiCandidate(cand);
|
|
179
|
+
|
|
180
|
+
const usageMeta = res?.response?.usageMetadata;
|
|
181
|
+
const usage = {
|
|
182
|
+
total_tokens: usageMeta?.totalTokenCount,
|
|
183
|
+
providerRaw: usageMeta,
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
return { message, usage };
|
|
187
|
+
}
|
|
188
|
+
}
|