testronaut 1.9.3 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -3
- package/TODO.md +11 -0
- package/bin/cli.js +45 -13
- package/bin/init.js +10 -5
- package/bin/initHelpers.js +9 -3
- package/core/agent.js +75 -2
- package/core/config.js +16 -0
- package/core/launchProtocol.js +42 -0
- package/core/turnLoop.js +38 -20
- package/llm/anthropic/anthropicProvider.js +2 -0
- package/llm/gemini/geminiProvider.js +36 -8
- package/llm/openAI/openaiProvider.js +2 -0
- package/package.json +1 -1
- package/runner/testronaut.js +24 -8
- package/tests/binTests/cli.integration.test.js +26 -1
- package/tests/binTests/initHelpersTests/models.test.js +7 -1
- package/tests/coreTests/agent.test.js +61 -0
- package/tests/coreTests/config.test.js +10 -0
- package/tests/coreTests/launchProtocol.test.js +38 -0
- package/tests/coreTests/turnLoop.test.js +21 -0
- package/tests/llmTests/geminiProvider.test.js +89 -7
- package/tests/runnerTests/testronaut.test.js +33 -6
- package/tests/toolsTests/chromeBrowserProbe.test.js +38 -0
- package/tests/toolsTests/contextControl.test.js +33 -1
- package/tests/toolsTests/domControl.test.js +12 -0
- package/tests/toolsTests/toolSchema.test.js +8 -0
- package/tests/toolsTests/turnLoopUtils.test.js +16 -0
- package/tools/chromeBrowser.js +50 -5
- package/tools/contextControl.js +11 -1
- package/tools/domControl.js +20 -1
- package/tools/generateHtmlReport.js +1 -1
- package/tools/tokenControl.js +3 -3
- package/tools/toolSchema.js +4 -15
- package/tools/turnLoopUtils.js +4 -3
|
@@ -28,6 +28,13 @@
|
|
|
28
28
|
|
|
29
29
|
import { GoogleGenerativeAI } from '@google/generative-ai';
|
|
30
30
|
|
|
31
|
+
// Gemini 3 validates thought signatures on function calls in the current turn.
|
|
32
|
+
// Calls synthesized by Testronaut (for example, automatic get_dom calls) were
|
|
33
|
+
// not emitted by Gemini and therefore have no real signature to preserve.
|
|
34
|
+
// Google documents this sentinel specifically for deterministic client-injected
|
|
35
|
+
// function-call history.
|
|
36
|
+
const SYNTHETIC_THOUGHT_SIGNATURE = 'skip_thought_signature_validator';
|
|
37
|
+
|
|
31
38
|
/**
|
|
32
39
|
* Convert OpenAI-like messages → Gemini "contents" array.
|
|
33
40
|
* - System messages are coalesced and injected as a prefix into the next user turn.
|
|
@@ -49,29 +56,41 @@ function toGeminiContents(messages) {
|
|
|
49
56
|
continue;
|
|
50
57
|
}
|
|
51
58
|
|
|
59
|
+
// Gemini 3 requires thought signatures to be returned exactly where the
|
|
60
|
+
// model emitted them. Prefer the untouched provider parts when available.
|
|
61
|
+
const geminiParts = m.provider_metadata?.gemini?.parts;
|
|
62
|
+
if (m.role === 'assistant' && Array.isArray(geminiParts) && geminiParts.length) {
|
|
63
|
+
contents.push({ role: 'model', parts: geminiParts });
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
|
|
52
67
|
// Assistant tool calls → Gemini functionCall parts on a model turn
|
|
53
68
|
if (m.role === 'assistant' && Array.isArray(m.tool_calls) && m.tool_calls.length) {
|
|
54
69
|
contents.push({
|
|
55
70
|
role: 'model',
|
|
56
|
-
parts: m.tool_calls.map(tc => ({
|
|
71
|
+
parts: m.tool_calls.map((tc, index) => ({
|
|
57
72
|
functionCall: {
|
|
58
73
|
name: tc.function?.name,
|
|
59
74
|
args: safeJsonParse(tc.function?.arguments) ?? {},
|
|
60
75
|
},
|
|
76
|
+
// For parallel calls Gemini expects the signature only on the first
|
|
77
|
+
// functionCall part. Genuine provider parts take the branch above and
|
|
78
|
+
// retain their original signatures instead of receiving this sentinel.
|
|
79
|
+
...(index === 0 ? { thoughtSignature: SYNTHETIC_THOUGHT_SIGNATURE } : {}),
|
|
61
80
|
})),
|
|
62
81
|
});
|
|
63
82
|
continue;
|
|
64
83
|
}
|
|
65
84
|
|
|
66
|
-
// Tool results →
|
|
85
|
+
// Tool results → Gemini function responses. This keeps function-calling
|
|
86
|
+
// history valid for Gemini 3 and gives earlier models structured results.
|
|
67
87
|
if (m.role === 'tool') {
|
|
88
|
+
const parsed = safeJsonParse(m.content);
|
|
68
89
|
const parts = [{
|
|
69
|
-
|
|
70
|
-
_tool_result: true,
|
|
71
|
-
tool_call_id: m.tool_call_id,
|
|
90
|
+
functionResponse: {
|
|
72
91
|
name: m.name,
|
|
73
|
-
|
|
74
|
-
}
|
|
92
|
+
response: parsed ?? { result: m.content },
|
|
93
|
+
},
|
|
75
94
|
}];
|
|
76
95
|
contents.push({ role: 'user', parts });
|
|
77
96
|
continue;
|
|
@@ -148,7 +167,14 @@ function fromGeminiCandidate(cand) {
|
|
|
148
167
|
}
|
|
149
168
|
|
|
150
169
|
const content = texts.join('');
|
|
151
|
-
return {
|
|
170
|
+
return {
|
|
171
|
+
role: 'assistant',
|
|
172
|
+
content,
|
|
173
|
+
tool_calls: tool_calls.length ? tool_calls : undefined,
|
|
174
|
+
// Retain all parts, including Gemini 3 thought signatures and function IDs,
|
|
175
|
+
// so the next request can replay them without lossy format conversion.
|
|
176
|
+
provider_metadata: { gemini: { parts } },
|
|
177
|
+
};
|
|
152
178
|
}
|
|
153
179
|
|
|
154
180
|
function cryptoRandomId() {
|
|
@@ -193,6 +219,8 @@ export class GeminiProvider {
|
|
|
193
219
|
const usageMeta = res?.response?.usageMetadata;
|
|
194
220
|
const usage = {
|
|
195
221
|
total_tokens: usageMeta?.totalTokenCount,
|
|
222
|
+
input_tokens: usageMeta?.promptTokenCount,
|
|
223
|
+
output_tokens: usageMeta?.candidatesTokenCount,
|
|
196
224
|
providerRaw: usageMeta,
|
|
197
225
|
};
|
|
198
226
|
|
|
@@ -63,6 +63,8 @@ export class OpenAIProvider {
|
|
|
63
63
|
const message = res.choices?.[0]?.message ?? { role: 'assistant', content: '' };
|
|
64
64
|
const usage = {
|
|
65
65
|
total_tokens: res.usage?.total_tokens,
|
|
66
|
+
input_tokens: res.usage?.prompt_tokens,
|
|
67
|
+
output_tokens: res.usage?.completion_tokens,
|
|
66
68
|
providerRaw: res.usage,
|
|
67
69
|
};
|
|
68
70
|
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -29,8 +29,11 @@ import 'dotenv/config';
|
|
|
29
29
|
import fs from 'fs';
|
|
30
30
|
import path from 'path';
|
|
31
31
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
32
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
32
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getAutomaticDomLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
33
33
|
import { normalizeTags } from '../core/tags.js';
|
|
34
|
+
import { launchProtocol, unwrapLaunchProtocol } from '../core/launchProtocol.js';
|
|
35
|
+
|
|
36
|
+
export { launchProtocol };
|
|
34
37
|
|
|
35
38
|
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
36
39
|
const isDebugEnabled = () => {
|
|
@@ -55,6 +58,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
55
58
|
const retryInfo = getRetryLimit(cfg);
|
|
56
59
|
const retryLimit = retryInfo.value;
|
|
57
60
|
const domListLimitInfo = getDomListLimit(cfg);
|
|
61
|
+
const automaticDomLimitInfo = getAutomaticDomLimit(cfg);
|
|
58
62
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
59
63
|
const humanInput = getHumanInputConfig(cfg);
|
|
60
64
|
const debugEnabled = isDebugEnabled();
|
|
@@ -99,6 +103,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
99
103
|
source: domListLimitInfo?.source,
|
|
100
104
|
clamped: domListLimitInfo?.clamped,
|
|
101
105
|
});
|
|
106
|
+
console.log('[debug] Automatic DOM limit', automaticDomLimitInfo);
|
|
102
107
|
console.log('[debug] Resource guard', {
|
|
103
108
|
enabled: resourceGuard.enabled,
|
|
104
109
|
hrefIncludes: resourceGuard.hrefIncludes,
|
|
@@ -161,12 +166,16 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
161
166
|
|
|
162
167
|
if (pre.length) {
|
|
163
168
|
goals.push(
|
|
164
|
-
...pre.map((fn, i) =>
|
|
165
|
-
goal
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
169
|
+
...pre.map((fn, i) => {
|
|
170
|
+
const { goal, protocol } = unwrapLaunchProtocol(fn);
|
|
171
|
+
return {
|
|
172
|
+
goal,
|
|
173
|
+
launchProtocol: protocol,
|
|
174
|
+
label: 'pre-mission', // legacy label (kept for logs)
|
|
175
|
+
submissionType: 'premission', // normalized
|
|
176
|
+
submissionName: pickName('premission', goal, i),
|
|
177
|
+
};
|
|
178
|
+
})
|
|
170
179
|
);
|
|
171
180
|
}
|
|
172
181
|
|
|
@@ -210,7 +219,14 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
210
219
|
missionName,
|
|
211
220
|
maxTurns,
|
|
212
221
|
retryLimit,
|
|
213
|
-
{
|
|
222
|
+
{
|
|
223
|
+
domListLimit: domListLimitInfo?.value,
|
|
224
|
+
automaticDomLimit: automaticDomLimitInfo.value,
|
|
225
|
+
storageStatePath: process.env.TESTRONAUT_STORAGE_STATE_PATH,
|
|
226
|
+
debug: debugEnabled,
|
|
227
|
+
resourceGuard,
|
|
228
|
+
humanInput,
|
|
229
|
+
}
|
|
214
230
|
);
|
|
215
231
|
if (!success) {
|
|
216
232
|
console.log(`❌ Aborting after failed goal.`);
|
|
@@ -37,10 +37,22 @@ describe('CLI inspection commands', () => {
|
|
|
37
37
|
const result = run(makeProject(), ['list', '--json']);
|
|
38
38
|
expect(result.status).toBe(0);
|
|
39
39
|
expect(JSON.parse(result.stdout).missions).toEqual([
|
|
40
|
-
{ file: 'login.mission.js', tags: ['authentication', 'smoke'] },
|
|
40
|
+
{ file: 'login.mission.js', tags: ['authentication', 'smoke'], selected: true },
|
|
41
41
|
]);
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
+
it('applies tag filters to list output', () => {
|
|
45
|
+
const result = run(makeProject(), ['list', '--tag=checkout', '--json']);
|
|
46
|
+
expect(result.status).toBe(0);
|
|
47
|
+
expect(JSON.parse(result.stdout).missions[0].selected).toBe(false);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('does not discard a following inline flag', () => {
|
|
51
|
+
const result = run(makeProject(), ['--model=gpt-5.6', '--turns=30', '--dry-run', '--json']);
|
|
52
|
+
expect(result.status).toBe(0);
|
|
53
|
+
expect(JSON.parse(result.stdout).missions[0].file).toBe('missions/login.mission.js');
|
|
54
|
+
});
|
|
55
|
+
|
|
44
56
|
it('reports effective config without exposing the session token', () => {
|
|
45
57
|
const result = run(makeProject(), ['config', '--json']);
|
|
46
58
|
expect(result.status).toBe(0);
|
|
@@ -74,4 +86,17 @@ describe('CLI inspection commands', () => {
|
|
|
74
86
|
expect(result.status).toBe(0);
|
|
75
87
|
expect(JSON.parse(result.stdout).missions[0].missionName).toBe('https://example.test');
|
|
76
88
|
});
|
|
89
|
+
|
|
90
|
+
it('records shared-auth session mode in the report', () => {
|
|
91
|
+
const cwd = makeProject();
|
|
92
|
+
fs.writeFileSync(path.join(cwd, 'missions', 'login.mission.js'), [
|
|
93
|
+
'export const tags = ["authentication"];',
|
|
94
|
+
'export async function executeMission() {',
|
|
95
|
+
' return { missionName: "login", submissionType: "mission", status: "passed", steps: [] };',
|
|
96
|
+
'}',
|
|
97
|
+
].join('\n'));
|
|
98
|
+
const result = run(cwd, ['--session=shared-auth', '--json', '--quiet']);
|
|
99
|
+
expect(result.status).toBe(0);
|
|
100
|
+
expect(JSON.parse(result.stdout).session).toEqual({ mode: 'shared-auth' });
|
|
101
|
+
});
|
|
77
102
|
});
|
|
@@ -15,13 +15,19 @@ describe('model helpers', () => {
|
|
|
15
15
|
'gpt-5.6-luna',
|
|
16
16
|
]));
|
|
17
17
|
expect(openAIModels()).not.toContain('gpt-5.3');
|
|
18
|
-
expect(geminiModels()).
|
|
18
|
+
expect(geminiModels()).toEqual(expect.arrayContaining([
|
|
19
|
+
'gemini-3.8-flash',
|
|
20
|
+
'gemini-3.5-flash-lite',
|
|
21
|
+
'gemini-3.1-pro-preview',
|
|
22
|
+
'gemini-2.5-flash',
|
|
23
|
+
]));
|
|
19
24
|
expect(anthropicModels()).toContain('claude-sonnet-5');
|
|
20
25
|
});
|
|
21
26
|
|
|
22
27
|
it('recognizes known models by provider', () => {
|
|
23
28
|
expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
|
|
24
29
|
expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
|
|
30
|
+
expect(isKnownModel('gemini', 'gemini-3.8-flash')).toBe(true);
|
|
25
31
|
expect(isKnownModel('anthropic', 'claude-sonnet-5')).toBe(true);
|
|
26
32
|
});
|
|
27
33
|
|
|
@@ -6,6 +6,7 @@ vi.mock('../../tools/chromeBrowser.js', () => {
|
|
|
6
6
|
function ChromeBrowser() {}
|
|
7
7
|
ChromeBrowser.prototype.start = async function () {};
|
|
8
8
|
ChromeBrowser.prototype.close = async function () {};
|
|
9
|
+
ChromeBrowser.prototype.probe = async function () { return { passed: true }; };
|
|
9
10
|
return { ChromeBrowser };
|
|
10
11
|
});
|
|
11
12
|
|
|
@@ -17,6 +18,7 @@ vi.mock('../../core/turnLoop.js', () => ({
|
|
|
17
18
|
import { ChromeBrowser } from '../../tools/chromeBrowser.js';
|
|
18
19
|
import { turnLoop } from '../../core/turnLoop.js';
|
|
19
20
|
import { runAgent } from '../../core/agent.js';
|
|
21
|
+
import { resetLaunchProtocols } from '../../core/launchProtocol.js';
|
|
20
22
|
|
|
21
23
|
let startSpy, closeSpy;
|
|
22
24
|
|
|
@@ -26,6 +28,8 @@ describe('core/agent.runAgent', () => {
|
|
|
26
28
|
startSpy = vi.spyOn(ChromeBrowser.prototype, 'start').mockResolvedValue();
|
|
27
29
|
closeSpy = vi.spyOn(ChromeBrowser.prototype, 'close').mockResolvedValue();
|
|
28
30
|
vi.clearAllMocks();
|
|
31
|
+
resetLaunchProtocols();
|
|
32
|
+
delete process.env.TESTRONAUT_SESSION_MODE;
|
|
29
33
|
});
|
|
30
34
|
|
|
31
35
|
afterEach(() => {
|
|
@@ -104,6 +108,63 @@ describe('core/agent.runAgent', () => {
|
|
|
104
108
|
expect(res2[0].steps[0].result).toBe('SUCCESS: step2');
|
|
105
109
|
});
|
|
106
110
|
|
|
111
|
+
it('attaches telemetry recorded during each goal to that result', async () => {
|
|
112
|
+
turnLoop.mockImplementationOnce(async (browser, messages, maxTurns, ct, rc, cs, ctx) => {
|
|
113
|
+
ctx.groundControl.telemetry.push({ kind: 'note', text: 'durable observation' });
|
|
114
|
+
return { success: true };
|
|
115
|
+
});
|
|
116
|
+
const [result] = await runAgent([{ goal: 'Observe' }], 'Telemetry', 3);
|
|
117
|
+
expect(result.telemetry).toEqual([{ kind: 'note', text: 'durable observation' }]);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('reuses a completed launch protocol after a passing shared-auth probe', async () => {
|
|
121
|
+
process.env.TESTRONAUT_SESSION_MODE = 'shared-auth';
|
|
122
|
+
const probeSpy = vi.spyOn(ChromeBrowser.prototype, 'probe').mockResolvedValue({
|
|
123
|
+
passed: true,
|
|
124
|
+
url: 'https://example.test/dashboard',
|
|
125
|
+
});
|
|
126
|
+
const goal = {
|
|
127
|
+
goal: 'Log in',
|
|
128
|
+
submissionType: 'premission',
|
|
129
|
+
launchProtocol: { id: 'authenticated:user', probe: { selector: '#dashboard' } },
|
|
130
|
+
};
|
|
131
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
132
|
+
await runAgent([goal], 'First', 3);
|
|
133
|
+
turnLoop.mockResolvedValueOnce({ success: true });
|
|
134
|
+
const [reused] = await runAgent([
|
|
135
|
+
goal,
|
|
136
|
+
{ goal: 'Use the current dashboard', submissionType: 'mission' },
|
|
137
|
+
], 'Second', 3);
|
|
138
|
+
|
|
139
|
+
expect(turnLoop).toHaveBeenCalledTimes(2);
|
|
140
|
+
expect(probeSpy).toHaveBeenCalledWith({ selector: '#dashboard' });
|
|
141
|
+
expect(reused.launchProtocol).toEqual({ id: 'authenticated:user', status: 'reused', probe: 'passed' });
|
|
142
|
+
expect(reused.steps[0].tokensUsed).toBe(0);
|
|
143
|
+
const mainMessages = turnLoop.mock.calls[1][1];
|
|
144
|
+
expect(mainMessages[0].content).toContain('https://example.test/dashboard');
|
|
145
|
+
expect(mainMessages[0].content).toContain('"loggedIn": true');
|
|
146
|
+
probeSpy.mockRestore();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('falls back to the original preMission when a reuse probe fails', async () => {
|
|
150
|
+
process.env.TESTRONAUT_SESSION_MODE = 'shared-auth';
|
|
151
|
+
const goal = {
|
|
152
|
+
goal: 'Log in',
|
|
153
|
+
submissionType: 'premission',
|
|
154
|
+
launchProtocol: { id: 'authenticated:user', probe: { selector: '#dashboard' } },
|
|
155
|
+
};
|
|
156
|
+
turnLoop.mockResolvedValue({ success: true });
|
|
157
|
+
await runAgent([goal], 'First', 3);
|
|
158
|
+
const probeSpy = vi.spyOn(ChromeBrowser.prototype, 'probe').mockResolvedValueOnce({ passed: false, reason: 'signed out' });
|
|
159
|
+
const [executed] = await runAgent([goal], 'Second', 3);
|
|
160
|
+
|
|
161
|
+
expect(turnLoop).toHaveBeenCalledTimes(2);
|
|
162
|
+
expect(executed.launchProtocol).toEqual({
|
|
163
|
+
id: 'authenticated:user', status: 'executed', probe: 'failed', reason: 'signed out',
|
|
164
|
+
});
|
|
165
|
+
probeSpy.mockRestore();
|
|
166
|
+
});
|
|
167
|
+
|
|
107
168
|
it('includes missionName and submission metadata in results', async () => {
|
|
108
169
|
turnLoop.mockResolvedValueOnce({ success: true });
|
|
109
170
|
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
resolveTurnLimits,
|
|
14
14
|
enforceTurnBudget,
|
|
15
15
|
getDomListLimit,
|
|
16
|
+
getAutomaticDomLimit,
|
|
16
17
|
getResourceGuardConfig,
|
|
17
18
|
getHumanInputConfig,
|
|
18
19
|
} from '../../core/config.js';
|
|
@@ -21,6 +22,15 @@ describe('core/config', () => {
|
|
|
21
22
|
beforeEach(() => {
|
|
22
23
|
vi.resetAllMocks();
|
|
23
24
|
delete process.env.STRICT_LIMITS;
|
|
25
|
+
delete process.env.TESTRONAUT_AUTO_DOM_LIMIT;
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
describe('getAutomaticDomLimit', () => {
|
|
29
|
+
it('uses config and clamps unsafe values', () => {
|
|
30
|
+
expect(getAutomaticDomLimit({ dom: { automaticLimit: 9000 } })).toMatchObject({ value: 9000, source: 'config' });
|
|
31
|
+
process.env.TESTRONAUT_AUTO_DOM_LIMIT = '250000';
|
|
32
|
+
expect(getAutomaticDomLimit({})).toMatchObject({ value: 100000, source: 'env', clamped: true });
|
|
33
|
+
});
|
|
24
34
|
});
|
|
25
35
|
|
|
26
36
|
describe('loadConfig', () => {
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, it, beforeEach } from 'vitest';
|
|
2
|
+
import {
|
|
3
|
+
launchProtocol,
|
|
4
|
+
unwrapLaunchProtocol,
|
|
5
|
+
hasCompletedLaunchProtocol,
|
|
6
|
+
markLaunchProtocolCompleted,
|
|
7
|
+
resetLaunchProtocols,
|
|
8
|
+
} from '../../core/launchProtocol.js';
|
|
9
|
+
|
|
10
|
+
describe('launchProtocol', () => {
|
|
11
|
+
beforeEach(() => resetLaunchProtocols());
|
|
12
|
+
|
|
13
|
+
it('wraps ordinary mission text without changing string coercion', () => {
|
|
14
|
+
const wrapped = launchProtocol('Log in', {
|
|
15
|
+
id: 'authenticated:user',
|
|
16
|
+
probe: { url: 'https://example.test', selector: '#dashboard' },
|
|
17
|
+
});
|
|
18
|
+
expect(String(wrapped)).toBe('Log in');
|
|
19
|
+
expect(unwrapLaunchProtocol(wrapped)).toEqual({
|
|
20
|
+
goal: 'Log in',
|
|
21
|
+
protocol: {
|
|
22
|
+
id: 'authenticated:user',
|
|
23
|
+
probe: { url: 'https://example.test', selector: '#dashboard' },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('tracks completed protocol ids for one process run', () => {
|
|
29
|
+
expect(hasCompletedLaunchProtocol('auth')).toBe(false);
|
|
30
|
+
markLaunchProtocolCompleted('auth');
|
|
31
|
+
expect(hasCompletedLaunchProtocol('auth')).toBe(true);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('rejects incomplete metadata', () => {
|
|
35
|
+
expect(() => launchProtocol('x', { probe: { selector: '#ready' } })).toThrow(/id/);
|
|
36
|
+
expect(() => launchProtocol('x', { id: 'ready', probe: {} })).toThrow(/selector or text/);
|
|
37
|
+
});
|
|
38
|
+
});
|
|
@@ -205,6 +205,27 @@ describe('turnLoop', () => {
|
|
|
205
205
|
expect(step1.result).toBe('✅ Passed');
|
|
206
206
|
expect(shared.chromeToolSpies.click_text).toHaveBeenCalledTimes(1);
|
|
207
207
|
expect(shared.chromeToolSpies.get_dom).toHaveBeenCalled();
|
|
208
|
+
expect(shared.chromeToolSpies.get_dom).toHaveBeenCalledWith(
|
|
209
|
+
browser,
|
|
210
|
+
expect.objectContaining({ limit: 15000, exclude: true }),
|
|
211
|
+
expect.any(Object)
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
const secondRequestMessages = shared.chatMock.mock.calls[1][0].messages;
|
|
215
|
+
const clickCallIndex = secondRequestMessages.findIndex(message =>
|
|
216
|
+
message.role === 'assistant' && message.tool_calls?.[0]?.function?.name === 'click_text'
|
|
217
|
+
);
|
|
218
|
+
expect(secondRequestMessages.slice(clickCallIndex, clickCallIndex + 4).map(message => ({
|
|
219
|
+
role: message.role,
|
|
220
|
+
tool: message.tool_calls?.[0]?.function?.name ?? message.name,
|
|
221
|
+
}))).toEqual([
|
|
222
|
+
{ role: 'assistant', tool: 'click_text' },
|
|
223
|
+
{ role: 'tool', tool: 'click_text' },
|
|
224
|
+
{ role: 'assistant', tool: 'get_dom' },
|
|
225
|
+
{ role: 'tool', tool: 'get_dom' },
|
|
226
|
+
]);
|
|
227
|
+
expect(JSON.parse(secondRequestMessages[clickCallIndex + 2].tool_calls[0].function.arguments))
|
|
228
|
+
.toEqual({ limit: 15000, exclude: true });
|
|
208
229
|
|
|
209
230
|
const step2 = res.steps[1];
|
|
210
231
|
expect(step2.result).toMatch(/Success/);
|
|
@@ -146,14 +146,17 @@ describe('GeminiProvider', () => {
|
|
|
146
146
|
|
|
147
147
|
// 2) Assistant tool call → model role with functionCall
|
|
148
148
|
expect(c[1].role).toBe('model');
|
|
149
|
-
expect(c[1].parts[0]
|
|
149
|
+
expect(c[1].parts[0]).toEqual({
|
|
150
|
+
functionCall: { name: 'search', args: { q: 'x' } },
|
|
151
|
+
thoughtSignature: 'skip_thought_signature_validator',
|
|
152
|
+
});
|
|
150
153
|
|
|
151
|
-
// 3) Tool result →
|
|
154
|
+
// 3) Tool result → structured Gemini function response
|
|
152
155
|
expect(c[2].role).toBe('user');
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
expect(c[2].parts[0].functionResponse).toEqual({
|
|
157
|
+
name: 'search',
|
|
158
|
+
response: { results: [1, 2, 3] },
|
|
159
|
+
});
|
|
157
160
|
|
|
158
161
|
// 4) Final assistant text → model role with text
|
|
159
162
|
expect(c[3].role).toBe('model');
|
|
@@ -168,7 +171,10 @@ describe('GeminiProvider', () => {
|
|
|
168
171
|
{
|
|
169
172
|
content: {
|
|
170
173
|
parts: [
|
|
171
|
-
{
|
|
174
|
+
{
|
|
175
|
+
functionCall: { name: 'open', args: { url: 'https://example.com' } },
|
|
176
|
+
thoughtSignature: 'encrypted-signature',
|
|
177
|
+
},
|
|
172
178
|
{ text: 'Ok!' },
|
|
173
179
|
],
|
|
174
180
|
},
|
|
@@ -189,11 +195,87 @@ describe('GeminiProvider', () => {
|
|
|
189
195
|
expect(Array.isArray(message.tool_calls)).toBe(true);
|
|
190
196
|
expect(message.tool_calls[0].function.name).toBe('open');
|
|
191
197
|
expect(JSON.parse(message.tool_calls[0].function.arguments)).toEqual({ url: 'https://example.com' });
|
|
198
|
+
expect(message.provider_metadata.gemini.parts[0].thoughtSignature).toBe('encrypted-signature');
|
|
192
199
|
|
|
193
200
|
expect(usage.total_tokens).toBe(999);
|
|
194
201
|
expect(usage.providerRaw).toEqual({ totalTokenCount: 999, other: 'meta' });
|
|
195
202
|
});
|
|
196
203
|
|
|
204
|
+
it('replays Gemini response parts so thought signatures survive tool loops', async () => {
|
|
205
|
+
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
206
|
+
const signedPart = {
|
|
207
|
+
functionCall: { name: 'click', args: { selector: '#submit' } },
|
|
208
|
+
thoughtSignature: 'opaque-signature',
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
await prov.chat({
|
|
212
|
+
model: 'gemini-3.8-flash',
|
|
213
|
+
messages: [
|
|
214
|
+
{ role: 'user', content: 'Submit the form' },
|
|
215
|
+
{
|
|
216
|
+
role: 'assistant',
|
|
217
|
+
content: '',
|
|
218
|
+
tool_calls: [{ type: 'function', function: { name: 'click', arguments: '{"selector":"#submit"}' } }],
|
|
219
|
+
provider_metadata: { gemini: { parts: [signedPart] } },
|
|
220
|
+
},
|
|
221
|
+
{ role: 'tool', name: 'click', content: 'Clicked' },
|
|
222
|
+
],
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
expect(shared.lastContents[1]).toEqual({ role: 'model', parts: [signedPart] });
|
|
226
|
+
expect(shared.lastContents[2].parts[0].functionResponse).toEqual({
|
|
227
|
+
name: 'click',
|
|
228
|
+
response: { result: 'Clicked' },
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it('signs deterministic client-injected function calls for Gemini 3 history', async () => {
|
|
233
|
+
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
234
|
+
|
|
235
|
+
await prov.chat({
|
|
236
|
+
model: 'gemini-3.8-flash',
|
|
237
|
+
messages: [
|
|
238
|
+
{ role: 'user', content: 'Open the dashboard' },
|
|
239
|
+
{
|
|
240
|
+
role: 'assistant',
|
|
241
|
+
content: '',
|
|
242
|
+
tool_calls: [{
|
|
243
|
+
id: 'synthetic-dom',
|
|
244
|
+
type: 'function',
|
|
245
|
+
function: { name: 'get_dom', arguments: '{"selector":"main"}' },
|
|
246
|
+
}],
|
|
247
|
+
},
|
|
248
|
+
{ role: 'tool', tool_call_id: 'synthetic-dom', name: 'get_dom', content: '<main>Dashboard</main>' },
|
|
249
|
+
],
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
expect(shared.lastContents[1].parts[0]).toEqual({
|
|
253
|
+
functionCall: { name: 'get_dom', args: { selector: 'main' } },
|
|
254
|
+
thoughtSignature: 'skip_thought_signature_validator',
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it('adds the synthetic signature only to the first parallel function call', async () => {
|
|
259
|
+
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
260
|
+
|
|
261
|
+
await prov.chat({
|
|
262
|
+
model: 'gemini-3.8-flash',
|
|
263
|
+
messages: [
|
|
264
|
+
{ role: 'user', content: 'Inspect both regions' },
|
|
265
|
+
{
|
|
266
|
+
role: 'assistant',
|
|
267
|
+
tool_calls: [
|
|
268
|
+
{ type: 'function', function: { name: 'get_dom', arguments: '{"selector":"main"}' } },
|
|
269
|
+
{ type: 'function', function: { name: 'get_dom', arguments: '{"selector":"nav"}' } },
|
|
270
|
+
],
|
|
271
|
+
},
|
|
272
|
+
],
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
expect(shared.lastContents[1].parts[0].thoughtSignature).toBe('skip_thought_signature_validator');
|
|
276
|
+
expect(shared.lastContents[1].parts[1].thoughtSignature).toBeUndefined();
|
|
277
|
+
});
|
|
278
|
+
|
|
197
279
|
it('supports image parts (encoded to inlineData)', async () => {
|
|
198
280
|
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
199
281
|
|
|
@@ -11,15 +11,16 @@ vi.mock('../../core/config.js', () => ({
|
|
|
11
11
|
enforceTurnBudget: vi.fn(),
|
|
12
12
|
getRetryLimit: vi.fn(),
|
|
13
13
|
getDomListLimit: vi.fn(),
|
|
14
|
+
getAutomaticDomLimit: vi.fn(),
|
|
14
15
|
getResourceGuardConfig: vi.fn(),
|
|
15
16
|
getHumanInputConfig: vi.fn(),
|
|
16
17
|
}));
|
|
17
18
|
|
|
18
19
|
import { runAgent } from '../../core/agent.js';
|
|
19
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../../core/config.js';
|
|
20
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getAutomaticDomLimit, getResourceGuardConfig, getHumanInputConfig } from '../../core/config.js';
|
|
20
21
|
|
|
21
22
|
// Adjust the import path if your file lives elsewhere
|
|
22
|
-
import { runMissions, __test__ as testronautInternals } from '../../runner/testronaut.js';
|
|
23
|
+
import { runMissions, launchProtocol, __test__ as testronautInternals } from '../../runner/testronaut.js';
|
|
23
24
|
|
|
24
25
|
describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
25
26
|
const OLD_ENV = { ...process.env };
|
|
@@ -29,6 +30,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
29
30
|
// Default retry limit unless overridden in a test
|
|
30
31
|
getRetryLimit.mockReturnValue({ value: 2, source: 'default', clamped: false });
|
|
31
32
|
getDomListLimit.mockReturnValue({ value: 3, mode: 'number', source: 'default', clamped: false });
|
|
33
|
+
getAutomaticDomLimit.mockReturnValue({ value: 15000, source: 'default', clamped: false });
|
|
32
34
|
getResourceGuardConfig.mockReturnValue({ enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] });
|
|
33
35
|
getHumanInputConfig.mockReturnValue({ enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false });
|
|
34
36
|
});
|
|
@@ -58,13 +60,38 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
58
60
|
expect.stringContaining('Clamping to 200')
|
|
59
61
|
);
|
|
60
62
|
expect(runAgent).toHaveBeenCalledWith(
|
|
61
|
-
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } }
|
|
63
|
+
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, automaticDomLimit: 15000, storageStatePath: undefined, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } }
|
|
62
64
|
);
|
|
63
65
|
|
|
64
66
|
warn.mockRestore();
|
|
65
67
|
log.mockRestore();
|
|
66
68
|
});
|
|
67
69
|
|
|
70
|
+
it('unwraps launch protocols while preserving preMission authoring', async () => {
|
|
71
|
+
loadConfig.mockResolvedValue({ maxTurns: 20 });
|
|
72
|
+
enforceTurnBudget.mockReturnValue({
|
|
73
|
+
effectiveMax: 20,
|
|
74
|
+
limits: { softMaxTurns: 50, hardMaxTurns: 200, hardMinTurns: 5 },
|
|
75
|
+
notes: [],
|
|
76
|
+
});
|
|
77
|
+
runAgent.mockResolvedValue([{ steps: [{ result: 'SUCCESS' }], status: 'passed' }]);
|
|
78
|
+
const protocol = launchProtocol('Log in', {
|
|
79
|
+
id: 'authenticated:user',
|
|
80
|
+
probe: { url: 'https://example.test', selector: '#dashboard' },
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
await runMissions({ preMission: protocol, mission: 'Test dashboard' }, 'Protocol mission');
|
|
84
|
+
const goals = runAgent.mock.calls[0][0];
|
|
85
|
+
expect(goals[0]).toMatchObject({
|
|
86
|
+
goal: 'Log in',
|
|
87
|
+
submissionType: 'premission',
|
|
88
|
+
launchProtocol: {
|
|
89
|
+
id: 'authenticated:user',
|
|
90
|
+
probe: { url: 'https://example.test', selector: '#dashboard' },
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
68
95
|
it('does not warn when no notes are returned', async () => {
|
|
69
96
|
loadConfig.mockResolvedValue({ maxTurns: 20 });
|
|
70
97
|
enforceTurnBudget.mockReturnValue({
|
|
@@ -85,7 +112,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
85
112
|
await runMissions({ mission: 'No warnings' }, 'Clean');
|
|
86
113
|
|
|
87
114
|
expect(warn).not.toHaveBeenCalled();
|
|
88
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
115
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, automaticDomLimit: 15000, storageStatePath: undefined, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
89
116
|
|
|
90
117
|
warn.mockRestore();
|
|
91
118
|
log.mockRestore();
|
|
@@ -213,7 +240,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
213
240
|
expect(goals[1].submissionName).toMatch(/^My Mission/);
|
|
214
241
|
|
|
215
242
|
// Effective max turns and retry limit passed through
|
|
216
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
243
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, automaticDomLimit: 15000, storageStatePath: undefined, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
217
244
|
|
|
218
245
|
log.mockRestore();
|
|
219
246
|
});
|
|
@@ -252,7 +279,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
252
279
|
|
|
253
280
|
await runMissions({ mission: 'Debug' }, 'Debug Mission');
|
|
254
281
|
|
|
255
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
282
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, automaticDomLimit: 15000, storageStatePath: undefined, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
256
283
|
|
|
257
284
|
log.mockRestore();
|
|
258
285
|
});
|