testronaut 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -4
- package/TODO.md +9 -0
- package/bin/cli.js +267 -37
- package/bin/init.js +35 -5
- package/bin/initHelpers.js +16 -5
- package/core/missionDiscovery.js +3 -3
- package/core/missionLoader.js +16 -0
- package/core/turnLoop.js +18 -15
- package/llm/anthropic/anthropicProvider.js +119 -0
- package/llm/anthropic/models.js +7 -0
- package/llm/llmFactory.js +6 -1
- package/package.json +3 -1
- package/runner/testronaut.js +4 -1
- package/tests/binTests/cli.helpers.test.js +75 -2
- package/tests/binTests/cli.integration.test.js +77 -0
- package/tests/binTests/init.test.js +24 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +3 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +4 -0
- package/tests/binTests/initHelpersTests/models.test.js +3 -1
- package/tests/coreTests/missionDiscovery.test.js +2 -2
- package/tests/coreTests/missionLoader.test.js +52 -0
- package/tests/coreTests/turnLoop.test.js +40 -0
- package/tests/llmTests/anthropicProvider.test.js +41 -0
- package/tests/llmTests/llmFactory.test.js +17 -1
- package/tests/toolsTests/generateHtmlReport.test.js +18 -0
- package/tools/chromeBrowser.js +14 -4
- package/tools/generateHtmlReport.js +22 -7
- package/tools/mfaCode.js +4 -1
- package/tools/tokenControl.js +7 -1
|
@@ -77,6 +77,9 @@ describe('initializeTestronautProject', () => {
|
|
|
77
77
|
expect(cfg.initialized).toBe(true);
|
|
78
78
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
79
79
|
expect(typeof cfg.maxTurns).toBe('number');
|
|
80
|
+
expect(cfg.tags).toEqual([]);
|
|
81
|
+
expect(cfg.tagMatch).toBe('any');
|
|
82
|
+
expect(cfg.addTags).toEqual([]);
|
|
80
83
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
81
84
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
82
85
|
expect(cfg.humanInput?.enabled).toBe(true);
|
|
@@ -95,6 +98,9 @@ describe('initializeTestronautProject', () => {
|
|
|
95
98
|
// Welcome mission (name depends on your implementation; we check presence in missions/)
|
|
96
99
|
const files = fs.readdirSync(path.join(temp, 'missions'));
|
|
97
100
|
expect(files.some(f => /welcome/i.test(f))).toBe(true);
|
|
101
|
+
const welcome = fs.readFileSync(path.join(temp, 'missions', 'welcome.mission.js'), 'utf8');
|
|
102
|
+
expect(welcome).toContain('export const tags = [];');
|
|
103
|
+
expect(welcome).toContain('tags\n },');
|
|
98
104
|
|
|
99
105
|
// Exactly two prompts were asked
|
|
100
106
|
expect(shared.calls).toBe(2);
|
|
@@ -112,6 +118,9 @@ describe('initializeTestronautProject', () => {
|
|
|
112
118
|
const cfg = readJson(path.join(temp, 'testronaut-config.json'));
|
|
113
119
|
expect(cfg.provider).toBe('gemini');
|
|
114
120
|
expect(cfg.model).toBe('gemini-2.5-flash');
|
|
121
|
+
expect(cfg.tags).toEqual([]);
|
|
122
|
+
expect(cfg.tagMatch).toBe('any');
|
|
123
|
+
expect(cfg.addTags).toEqual([]);
|
|
115
124
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
116
125
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
117
126
|
expect(cfg.humanInput?.enabled).toBe(true);
|
|
@@ -129,6 +138,21 @@ describe('initializeTestronautProject', () => {
|
|
|
129
138
|
expect(shared.calls).toBe(2);
|
|
130
139
|
});
|
|
131
140
|
|
|
141
|
+
it('scaffolds a new Anthropic project', async () => {
|
|
142
|
+
const temp = makeTempProject();
|
|
143
|
+
process.chdir(temp);
|
|
144
|
+
shared.answers.push({ llmProvider: 'anthropic' });
|
|
145
|
+
shared.answers.push({ anthropicModel: 'claude-sonnet-5' });
|
|
146
|
+
|
|
147
|
+
await initializeTestronautProject();
|
|
148
|
+
|
|
149
|
+
const cfg = readJson(path.join(temp, 'testronaut-config.json'));
|
|
150
|
+
expect(cfg.provider).toBe('anthropic');
|
|
151
|
+
expect(cfg.model).toBe('claude-sonnet-5');
|
|
152
|
+
expect(fs.readFileSync(path.join(temp, '.env'), 'utf8')).toMatch(/ANTHROPIC_API_KEY=sk-ant-/);
|
|
153
|
+
expect(shared.calls).toBe(2);
|
|
154
|
+
});
|
|
155
|
+
|
|
132
156
|
it('is idempotent: when initialized, it skips prompts and does not overwrite .env', async () => {
|
|
133
157
|
const temp = makeTempProject();
|
|
134
158
|
process.chdir(temp);
|
|
@@ -8,6 +8,9 @@ describe('defaultConfig', () => {
|
|
|
8
8
|
expect(cfg.outputDir).toBe('missions/mission_reports');
|
|
9
9
|
expect(cfg.projectName).toBe('my-app');
|
|
10
10
|
expect(cfg.maxTurns).toBe(20);
|
|
11
|
+
expect(cfg.tags).toEqual([]);
|
|
12
|
+
expect(cfg.tagMatch).toBe('any');
|
|
13
|
+
expect(cfg.addTags).toEqual([]);
|
|
11
14
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
12
15
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
13
16
|
expect(cfg.resourceGuard?.hrefIncludes).toContain('/document/');
|
|
@@ -11,4 +11,8 @@ describe('makeEnvTemplate', () => {
|
|
|
11
11
|
const s = makeEnvTemplate('gemini');
|
|
12
12
|
expect(s).toMatch(/GEMINI_API_KEY=AIza\.\.\./);
|
|
13
13
|
});
|
|
14
|
+
|
|
15
|
+
it('returns Anthropic template', () => {
|
|
16
|
+
expect(makeEnvTemplate('anthropic')).toMatch(/ANTHROPIC_API_KEY=sk-ant-/);
|
|
17
|
+
});
|
|
14
18
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
|
|
2
|
+
import { openAIModels, geminiModels, anthropicModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
|
|
3
3
|
|
|
4
4
|
describe('model helpers', () => {
|
|
5
5
|
it('lists known models', () => {
|
|
@@ -16,11 +16,13 @@ describe('model helpers', () => {
|
|
|
16
16
|
]));
|
|
17
17
|
expect(openAIModels()).not.toContain('gpt-5.3');
|
|
18
18
|
expect(geminiModels()).toContain('gemini-2.5-flash');
|
|
19
|
+
expect(anthropicModels()).toContain('claude-sonnet-5');
|
|
19
20
|
});
|
|
20
21
|
|
|
21
22
|
it('recognizes known models by provider', () => {
|
|
22
23
|
expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
|
|
23
24
|
expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
|
|
25
|
+
expect(isKnownModel('anthropic', 'claude-sonnet-5')).toBe(true);
|
|
24
26
|
});
|
|
25
27
|
|
|
26
28
|
it('picks initial index from list or fallback', () => {
|
|
@@ -31,11 +31,11 @@ describe('discoverMissionFiles', () => {
|
|
|
31
31
|
fs.writeFileSync(path.join(missionsDir, 'beta.txt'), '// ignore');
|
|
32
32
|
fs.mkdirSync(path.join(missionsDir, 'nested'), { recursive: true });
|
|
33
33
|
fs.writeFileSync(path.join(missionsDir, 'nested', 'gamma.mission.js'), '// nested');
|
|
34
|
-
fs.writeFileSync(path.join(missionsDir, 'delta.mission.ts'), '// ts
|
|
34
|
+
fs.writeFileSync(path.join(missionsDir, 'delta.mission.ts'), '// ts');
|
|
35
35
|
|
|
36
36
|
const res = await discoverMissionFiles({ cwd: tmp });
|
|
37
37
|
expect(res.root).toBe(path.resolve(tmp, 'missions'));
|
|
38
|
-
expect(res.files).toEqual(['alpha.mission.js']);
|
|
38
|
+
expect(res.files).toEqual(['alpha.mission.js', 'delta.mission.ts']);
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it('respects missions.root with include patterns', async () => {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { loadMissionModule } from '../../core/missionLoader.js';
|
|
6
|
+
|
|
7
|
+
const tempDirs = [];
|
|
8
|
+
|
|
9
|
+
function makeProject(packageJson) {
|
|
10
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-loader-'));
|
|
11
|
+
tempDirs.push(dir);
|
|
12
|
+
if (packageJson) {
|
|
13
|
+
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(packageJson));
|
|
14
|
+
}
|
|
15
|
+
return dir;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
describe('loadMissionModule', () => {
|
|
23
|
+
it.each([
|
|
24
|
+
['without a package type', undefined],
|
|
25
|
+
['inside a CommonJS project', { type: 'commonjs' }],
|
|
26
|
+
])('loads ESM-style .mission.js %s', async (_label, packageJson) => {
|
|
27
|
+
const dir = makeProject(packageJson);
|
|
28
|
+
const missionPath = path.join(dir, 'sample.mission.js');
|
|
29
|
+
fs.writeFileSync(missionPath, 'export const tags = ["smoke"]; export async function executeMission() { return "ok"; }');
|
|
30
|
+
|
|
31
|
+
const mission = await loadMissionModule(missionPath);
|
|
32
|
+
|
|
33
|
+
expect(mission.tags).toEqual(['smoke']);
|
|
34
|
+
await expect(mission.executeMission()).resolves.toBe('ok');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('loads a typed .mission.ts file with relative imports', async () => {
|
|
38
|
+
const dir = makeProject({ type: 'commonjs' });
|
|
39
|
+
fs.writeFileSync(path.join(dir, 'goal.ts'), 'export const goal: string = "TypeScript works";');
|
|
40
|
+
const missionPath = path.join(dir, 'sample.mission.ts');
|
|
41
|
+
fs.writeFileSync(missionPath, [
|
|
42
|
+
'import { goal } from "./goal.ts";',
|
|
43
|
+
'export const tags: string[] = ["typescript"];',
|
|
44
|
+
'export async function executeMission(): Promise<string> { return goal; }',
|
|
45
|
+
].join('\n'));
|
|
46
|
+
|
|
47
|
+
const mission = await loadMissionModule(missionPath);
|
|
48
|
+
|
|
49
|
+
expect(mission.tags).toEqual(['typescript']);
|
|
50
|
+
await expect(mission.executeMission()).resolves.toBe('TypeScript works');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -78,6 +78,7 @@ vi.mock('../../tools/toolSchema.js', () => ({
|
|
|
78
78
|
{ type: 'function', function: { name: 'screenshot', description: 'take a screenshot', parameters: {} } },
|
|
79
79
|
{ type: 'function', function: { name: 'list_local_files', description: 'list files', parameters: {} } },
|
|
80
80
|
{ type: 'function', function: { name: 'get_mfa_code', description: 'get mfa', parameters: {} } },
|
|
81
|
+
{ type: 'function', function: { name: 'set_ground_control_state', description: 'set state', parameters: {} } },
|
|
81
82
|
],
|
|
82
83
|
}));
|
|
83
84
|
|
|
@@ -126,6 +127,7 @@ describe('turnLoop', () => {
|
|
|
126
127
|
let browser;
|
|
127
128
|
|
|
128
129
|
beforeEach(() => {
|
|
130
|
+
delete process.env.TESTRONAUT_SCREENSHOTS;
|
|
129
131
|
browser = {};
|
|
130
132
|
// reset spies and chat mock
|
|
131
133
|
shared.chromeToolSpies.fill.mockClear();
|
|
@@ -136,6 +138,19 @@ describe('turnLoop', () => {
|
|
|
136
138
|
if (shared.chatMock) shared.chatMock.mockReset();
|
|
137
139
|
});
|
|
138
140
|
|
|
141
|
+
it('does not expose the screenshot tool when screenshots are disabled', async () => {
|
|
142
|
+
process.env.TESTRONAUT_SCREENSHOTS = '0';
|
|
143
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
144
|
+
message: { role: 'assistant', content: 'FINAL: done' },
|
|
145
|
+
usage: { total_tokens: 2 },
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
await turnLoop(browser, baseMessages(), 1, 0, 0, {}, { steps: [], missionName: 'demo' });
|
|
149
|
+
|
|
150
|
+
const tools = shared.chatMock.mock.calls[0][0].tools;
|
|
151
|
+
expect(tools.map(tool => tool.function.name)).not.toContain('screenshot');
|
|
152
|
+
});
|
|
153
|
+
|
|
139
154
|
it('handles a final response in a single turn (no tools)', async () => {
|
|
140
155
|
shared.chatMock.mockResolvedValueOnce({
|
|
141
156
|
message: { role: 'assistant', content: 'FINAL: success' },
|
|
@@ -193,6 +208,31 @@ describe('turnLoop', () => {
|
|
|
193
208
|
expect(step2.result).toMatch(/Success/);
|
|
194
209
|
});
|
|
195
210
|
|
|
211
|
+
it('handles Ground Control state locally without dispatching it as a browser tool', async () => {
|
|
212
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
213
|
+
message: {
|
|
214
|
+
role: 'assistant', content: '', tool_calls: [{
|
|
215
|
+
id: 'tool_gc', type: 'function', function: {
|
|
216
|
+
name: 'set_ground_control_state',
|
|
217
|
+
arguments: JSON.stringify({ app: { baseUrl: 'https://example.com' }, session: { loggedIn: false } }),
|
|
218
|
+
},
|
|
219
|
+
}],
|
|
220
|
+
},
|
|
221
|
+
usage: { total_tokens: 4 },
|
|
222
|
+
});
|
|
223
|
+
shared.chatMock.mockResolvedValueOnce({
|
|
224
|
+
message: { role: 'assistant', content: 'FINAL: done' }, usage: { total_tokens: 2 },
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
const ctx = { steps: [], missionName: 'demo' };
|
|
228
|
+
const res = await turnLoop(browser, baseMessages(), 2, 0, 0, {}, ctx);
|
|
229
|
+
|
|
230
|
+
expect(res.success).toBe(true);
|
|
231
|
+
expect(ctx.groundControl.app.baseUrl).toBe('https://example.com');
|
|
232
|
+
expect(ctx.groundControl.session.loggedIn).toBe(false);
|
|
233
|
+
expect(res.steps[0].events).toContain('[tool ] ← set_ground_control_state result: ✅ Success');
|
|
234
|
+
});
|
|
235
|
+
|
|
196
236
|
it('retries a turn on tool error before marking issues', async () => {
|
|
197
237
|
// First turn: tool call → click_text will throw
|
|
198
238
|
shared.chatMock.mockResolvedValueOnce({
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const { create } = vi.hoisted(() => ({ create: vi.fn() }));
|
|
4
|
+
vi.mock('@anthropic-ai/sdk', () => ({
|
|
5
|
+
default: class Anthropic { constructor(options) { this.options = options; this.messages = { create }; } },
|
|
6
|
+
}));
|
|
7
|
+
|
|
8
|
+
import { AnthropicProvider, fromAnthropicMessage, toAnthropicRequest } from '../../llm/anthropic/anthropicProvider.js';
|
|
9
|
+
|
|
10
|
+
describe('AnthropicProvider', () => {
|
|
11
|
+
it('requires an API key', () => {
|
|
12
|
+
expect(() => new AnthropicProvider()).toThrow(/ANTHROPIC_API_KEY/);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it('converts system, images, tool calls, and tool results', () => {
|
|
16
|
+
const request = toAnthropicRequest([
|
|
17
|
+
{ role: 'system', content: 'Be precise' },
|
|
18
|
+
{ role: 'user', content: [{ type: 'text', text: 'Inspect' }, { type: 'image', mimeType: 'image/png', data: Buffer.from('png') }] },
|
|
19
|
+
{ role: 'assistant', content: '', tool_calls: [{ id: 'call_1', function: { name: 'click', arguments: '{"selector":"#go"}' } }] },
|
|
20
|
+
{ role: 'tool', tool_call_id: 'call_1', content: 'clicked' },
|
|
21
|
+
]);
|
|
22
|
+
expect(request.system).toEqual([{ type: 'text', text: 'Be precise' }]);
|
|
23
|
+
expect(request.messages[0].content[1].source).toMatchObject({ type: 'base64', media_type: 'image/png' });
|
|
24
|
+
expect(request.messages[1].content[0]).toMatchObject({ type: 'tool_use', id: 'call_1', name: 'click' });
|
|
25
|
+
expect(request.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'call_1' });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('normalizes Claude output and usage', async () => {
|
|
29
|
+
create.mockResolvedValueOnce({
|
|
30
|
+
content: [{ type: 'text', text: 'Working' }, { type: 'tool_use', id: 'toolu_1', name: 'navigate', input: { url: 'https://example.com' } }],
|
|
31
|
+
usage: { input_tokens: 12, output_tokens: 8 },
|
|
32
|
+
});
|
|
33
|
+
const result = await new AnthropicProvider({ apiKey: 'sk-ant-test' }).chat({
|
|
34
|
+
model: 'claude-sonnet-5', messages: [{ role: 'user', content: 'Go' }],
|
|
35
|
+
tools: [{ type: 'function', function: { name: 'navigate', description: 'Navigate', parameters: { type: 'object' } } }],
|
|
36
|
+
});
|
|
37
|
+
expect(create).toHaveBeenCalledWith(expect.objectContaining({ model: 'claude-sonnet-5', max_tokens: 8192 }));
|
|
38
|
+
expect(result.message).toEqual(fromAnthropicMessage({ content: [{ type: 'text', text: 'Working' }, { type: 'tool_use', id: 'toolu_1', name: 'navigate', input: { url: 'https://example.com' } }] }));
|
|
39
|
+
expect(result.usage.total_tokens).toBe(20);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
@@ -6,6 +6,7 @@ const { shared } = vi.hoisted(() => ({
|
|
|
6
6
|
shared: {
|
|
7
7
|
openAICtor: vi.fn(),
|
|
8
8
|
geminiCtor: vi.fn(),
|
|
9
|
+
anthropicCtor: vi.fn(),
|
|
9
10
|
},
|
|
10
11
|
}));
|
|
11
12
|
|
|
@@ -24,6 +25,12 @@ vi.mock('../../llm/gemini/geminiProvider.js', () => ({
|
|
|
24
25
|
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
25
26
|
},
|
|
26
27
|
}));
|
|
28
|
+
vi.mock('../../llm/anthropic/anthropicProvider.js', () => ({
|
|
29
|
+
AnthropicProvider: function (opts) {
|
|
30
|
+
shared.anthropicCtor(opts);
|
|
31
|
+
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
32
|
+
},
|
|
33
|
+
}));
|
|
27
34
|
|
|
28
35
|
// Import SUT after mocks
|
|
29
36
|
import { getLLM } from '../../llm/llmFactory.js';
|
|
@@ -35,6 +42,7 @@ describe('getLLM', () => {
|
|
|
35
42
|
process.env = { ...ORIGINAL_ENV };
|
|
36
43
|
shared.openAICtor.mockClear();
|
|
37
44
|
shared.geminiCtor.mockClear();
|
|
45
|
+
shared.anthropicCtor.mockClear();
|
|
38
46
|
});
|
|
39
47
|
|
|
40
48
|
afterEach(() => {
|
|
@@ -72,7 +80,15 @@ describe('getLLM', () => {
|
|
|
72
80
|
expect(typeof llm.chat).toBe('function');
|
|
73
81
|
});
|
|
74
82
|
|
|
83
|
+
it('creates Anthropic provider and accepts claude as an alias', () => {
|
|
84
|
+
process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
|
|
85
|
+
getLLM('claude');
|
|
86
|
+
expect(shared.anthropicCtor).toHaveBeenCalledWith(
|
|
87
|
+
expect.objectContaining({ apiKey: 'sk-ant-test' })
|
|
88
|
+
);
|
|
89
|
+
});
|
|
90
|
+
|
|
75
91
|
it('throws on unsupported provider', () => {
|
|
76
|
-
expect(() => getLLM('
|
|
92
|
+
expect(() => getLLM('mistral')).toThrow(/Unsupported LLM provider/i);
|
|
77
93
|
});
|
|
78
94
|
});
|
|
@@ -21,11 +21,14 @@ describe('generateHtmlReport', () => {
|
|
|
21
21
|
it('writes an HTML report containing mission and retry metadata', () => {
|
|
22
22
|
const report = {
|
|
23
23
|
runId: 'run_test',
|
|
24
|
+
llm: { provider: 'anthropic', model: 'claude-sonnet-5' },
|
|
25
|
+
cli: { version: '1.7.0' },
|
|
24
26
|
missions: [
|
|
25
27
|
{
|
|
26
28
|
missionName: 'Mission A',
|
|
27
29
|
submissionType: 'mission',
|
|
28
30
|
submissionName: 'Main',
|
|
31
|
+
file: 'missions/login.mission.ts',
|
|
29
32
|
status: 'passed',
|
|
30
33
|
tags: ['smoke'],
|
|
31
34
|
steps: [
|
|
@@ -57,11 +60,16 @@ describe('generateHtmlReport', () => {
|
|
|
57
60
|
expect(html).toContain('⚠️ Turn Issues');
|
|
58
61
|
expect(html).toContain('data-tag="smoke"');
|
|
59
62
|
expect(html).toContain('data-tags="smoke"');
|
|
63
|
+
expect(html).toContain('CLI version:</strong> 1.7.0');
|
|
64
|
+
expect(html).toContain('missions/login.mission.ts');
|
|
60
65
|
expect(html).toContain('data-tag="untagged"');
|
|
61
66
|
expect(html).toContain('id="match-count"');
|
|
62
67
|
expect(html).toContain('No missions match these tags');
|
|
63
68
|
expect(html).toContain('Keep missions that do not match visible');
|
|
64
69
|
expect(html).not.toContain('<div class="pill">LLM:');
|
|
70
|
+
expect(html).not.toContain('<strong>LLM:</strong>');
|
|
71
|
+
expect(html).toContain('title="Anthropic"');
|
|
72
|
+
expect(html).toContain('<strong>claude-sonnet-5</strong>');
|
|
65
73
|
});
|
|
66
74
|
|
|
67
75
|
it('escapes unsafe text in output', () => {
|
|
@@ -95,4 +103,14 @@ describe('generateHtmlReport', () => {
|
|
|
95
103
|
expect(html).toContain('<script>alert(1)</script>');
|
|
96
104
|
expect(html).toContain('<b>bold</b>');
|
|
97
105
|
});
|
|
106
|
+
|
|
107
|
+
it('renders the OpenAI logo and model without a textual LLM label', () => {
|
|
108
|
+
const outPath = path.join(tmpDir, 'openai.html');
|
|
109
|
+
generateHtmlReport({ runId: 'openai_run', llm: { provider: 'openai', model: 'gpt-5.6' } }, outPath);
|
|
110
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
111
|
+
|
|
112
|
+
expect(html).toContain('title="OpenAI"');
|
|
113
|
+
expect(html).toContain('<strong>gpt-5.6</strong>');
|
|
114
|
+
expect(html).not.toContain('<strong>LLM:</strong>');
|
|
115
|
+
});
|
|
98
116
|
});
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -26,7 +26,7 @@ import { requestHumanInput } from './humanInput.js';
|
|
|
26
26
|
import { getMfaCode } from './mfaCode.js';
|
|
27
27
|
|
|
28
28
|
const FILES_DIR = path.join('missions', 'files');
|
|
29
|
-
const REPORTS_DIR = path.join('missions', 'mission_reports');
|
|
29
|
+
const REPORTS_DIR = process.env.TESTRONAUT_OUTPUT_DIR || path.join('missions', 'mission_reports');
|
|
30
30
|
const FILES_LOG = path.join(REPORTS_DIR, 'files.jsonl');
|
|
31
31
|
|
|
32
32
|
function ensureDir(p) {
|
|
@@ -752,7 +752,11 @@ export class ChromeBrowser {
|
|
|
752
752
|
}
|
|
753
753
|
if (!chosen) {
|
|
754
754
|
await this._debugClickableSnapshot({ scope }).catch(()=>{});
|
|
755
|
-
|
|
755
|
+
if (process.env.TESTRONAUT_SCREENSHOTS !== '0') {
|
|
756
|
+
const failurePath = path.join(REPORTS_DIR, 'screenshots', process.env.TESTRONAUT_RUN_ID || '', 'click_fail.png');
|
|
757
|
+
ensureDir(path.dirname(failurePath));
|
|
758
|
+
await this.page.screenshot({ path: failurePath }).catch(()=>{});
|
|
759
|
+
}
|
|
756
760
|
throw new Error('No clickable locator became ready');
|
|
757
761
|
}
|
|
758
762
|
// do the click (standard → hover+click → JS click)
|
|
@@ -1183,13 +1187,19 @@ export class ChromeBrowser {
|
|
|
1183
1187
|
}
|
|
1184
1188
|
|
|
1185
1189
|
async screenshot({ name = 'screenshot' }) {
|
|
1190
|
+
if (process.env.TESTRONAUT_SCREENSHOTS === '0') {
|
|
1191
|
+
throw new Error('Screenshots are disabled for this run (--no-screenshots).');
|
|
1192
|
+
}
|
|
1186
1193
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
1187
|
-
const
|
|
1194
|
+
const screenshotDir = path.join(REPORTS_DIR, 'screenshots', process.env.TESTRONAUT_RUN_ID || '');
|
|
1195
|
+
const screenshotPath = path.join(screenshotDir, `${name}_${timestamp}.png`);
|
|
1196
|
+
ensureDir(screenshotDir);
|
|
1188
1197
|
|
|
1189
1198
|
await this.page.screenshot({ path: screenshotPath });
|
|
1190
1199
|
|
|
1191
1200
|
// ✅ Return relative path for HTML report
|
|
1192
|
-
|
|
1201
|
+
const reportRelativePath = path.relative(REPORTS_DIR, screenshotPath).split(path.sep).join('/');
|
|
1202
|
+
return `Screenshot saved at: ./${reportRelativePath}`;
|
|
1193
1203
|
}
|
|
1194
1204
|
|
|
1195
1205
|
async upload_file({ selector, fileName, useChooser = false, timeoutMs = 15000 }) {
|
|
@@ -27,7 +27,7 @@ import { normalizeTags } from '../core/tags.js';
|
|
|
27
27
|
* @returns {string} absolute path to the written HTML file
|
|
28
28
|
*/
|
|
29
29
|
export function generateHtmlReport(report, outputPath) {
|
|
30
|
-
const { runId, startTime, endTime, missions = [], summary = {}, llm = {} } = report;
|
|
30
|
+
const { runId, startTime, endTime, missions = [], summary = {}, llm = {}, cli = {} } = report;
|
|
31
31
|
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
32
32
|
const durationSec =
|
|
33
33
|
(startTime && endTime)
|
|
@@ -42,6 +42,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
42
42
|
const badge = (status) =>
|
|
43
43
|
status === 'passed' ? '✅ Passed' :
|
|
44
44
|
status === 'failed' ? '❌ Failed' : (status || '—');
|
|
45
|
+
const providerKey = String(llm.provider ?? '').trim().toLowerCase() === 'claude'
|
|
46
|
+
? 'anthropic'
|
|
47
|
+
: String(llm.provider ?? '').trim().toLowerCase();
|
|
48
|
+
const provider = {
|
|
49
|
+
openai: { name: 'OpenAI', logo: '<svg viewBox="0 0 24 24" aria-hidden="true"><path fill="currentColor" d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"/></svg>' },
|
|
50
|
+
anthropic: { name: 'Anthropic', logo: '<img src="data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8dGl0bGU+Q2xhdWRlPC90aXRsZT4KICA8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHJ4PSI1IiBmaWxsPSIjRDk3NzU3Ii8+CiAgPHBhdGggZmlsbD0iI2ZmZiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMyAzKSBzY2FsZSguNzUpIiBkPSJNNC43MDkgMTUuOTU1bDQuNzItMi42NDcuMDgtLjIzLS4wOC0uMTI4SDkuMmwtLjc5LS4wNDgtMi42OTgtLjA3My0yLjMzOS0uMDk3LTIuMjY2LS4xMjItLjU3MS0uMTIxTDAgMTEuNzg0bC4wNTUtLjM1Mi40OC0uMzIxLjY4Ni4wNiAxLjUyLjEwMyAyLjI3OC4xNTggMS42NTIuMDk3IDIuNDQ5LjI1NWguMzg5bC4wNTUtLjE1Ny0uMTM0LS4wOTgtLjEwMy0uMDk3LTIuMzU4LTEuNTk2LTIuNTUyLTEuNjg4LTEuMzM2LS45NzItLjcyNC0uNDkxLS4zNjQtLjQ2Mi0uMTU4LTEuMDA4LjY1Ni0uNzIyLjg4MS4wNi4yMjUuMDYxLjg5My42ODYgMS45MDggMS40NzYgMi40OTEgMS44MzMuMzY1LjMwNC4xNDUtLjEwMy4wMTktLjA3My0uMTY0LS4yNzQtMS4zNTUtMi40NDYtMS40NDYtMi40OS0uNjQ0LTEuMDMyLS4xNy0uNjE5YTIuOTcgMi45NyAwIDAxLS4xMDQtLjcyOUw2LjI4My4xMzQgNi42OTYgMGwuOTk2LjEzNC40Mi4zNjQuNjIgMS40MTQgMS4wMDIgMi4yMjkgMS41NTUgMy4wMy40NTYuODk4LjI0My44MzIuMDkxLjI1NWguMTU4VjkuMDFsLjEyOC0xLjcwNi4yMzctMi4wOTUuMjMtMi42OTUuMDgtLjc2LjM3Ni0uOTEuNzQ3LS40OTIuNTg0LjI4LjQ4LjY4NS0uMDY3LjQ0NC0uMjg2IDEuODUxLS41NTkgMi45MDMtLjM2NCAxLjk0MmguMjEybC4yNDMtLjI0Mi45ODUtMS4zMDYgMS42NTItMi4wNjQuNzMtLjgyLjg1LS45MDQuNTQ3LS40MzFoMS4wMzNsLjc2IDEuMTI5LS4zNCAxLjE2Ni0xLjA2NCAxLjM0Ny0uODgxIDEuMTQyLTEuMjY0IDEuNy0uNzkgMS4zNi4wNzMuMTEuMTg4LS4wMiAyLjg1Ni0uNjA2IDEuNTQzLS4yOCAxLjg0MS0uMzE1LjgzMy4zODguMDkxLjM5NS0uMzI4LjgwNy0xLjk2OS40ODYtMi4zMDkuNDYyLTMuNDM5LjgxMy0uMDQyLjAzLjA0OS4wNjEgMS41NDkuMTQ2LjY2Mi4wMzZoMS42MjJsMy4wMi4yMjUuNzkuNTIyLjQ3NC42MzgtLjA3OS40ODUtMS4yMTUuNjItMS42NC0uMzg5LTMuODI5LS45MS0xLjMxMi0uMzI5aC0uMTgydi4xMWwxLjA5MyAxLjA2OCAyLjAwNiAxLjgxIDIuNTA5IDIuMzMuMTI3LjU3OC0uMzIyLjQ1NS0uMzQtLjA0OS0yLjIwNS0xLjY1Ny0uODUxLS43NDctMS45MjYtMS42MmgtLjEyOHYuMTdsLjQ0NC42NDkgMi4zNDUgMy41MjEuMTIyIDEuMDgtLjE3LjM1My0uNjA4LjIxMy0uNjY4LS4xMjItMS4zNzQtMS45MjUtMS40MTUtMi4xNjctMS4xNDMtMS45NDMtLjE0LjA4LS42NzQgNy4yNTQtLjMxNi4zNy0uNzI5LjI4LS42MDctLjQ2MS0uMzIyLS43NDcuMzIyLTEuNDc2LjM4OS0xLjkyNC4zMTUtMS41My4yODYtMS45LjE3LS42MzItLjAxMi0uMDQyLS4xNC4wMTgtMS40MzQgMS45NjctMi4xOCAyLjk0NS0xLjcyNiAxLjg0NS0uNDE0LjE2NC0uNzE3LS4zNy4wNjctLjY2Mi40MDEtLjU4OSAyLjM4OC0zLjAzNiAxLjQ0LTEuODgyLjkzLTEuMDg2LS4wMDYtLjE1OGgtLjA1NUw0LjEzMiAxOC41NmwtMS4xMy4xNDYtLjQ4Ny0uNDU2LjA2MS0uNzQ2LjIzMS0uMjQzIDEuOTA4LTEuMzEyLS4wMDYuMDA2eiIvPgo8L3N2Zz4K" alt="" />' },
|
|
51
|
+
}[providerKey] ?? { name: llm.provider || 'Unknown provider', logo: '' };
|
|
45
52
|
const tagPalette = [
|
|
46
53
|
['#60a5fa', 'rgba(96,165,250,.18)'], ['#a78bfa', 'rgba(167,139,250,.18)'],
|
|
47
54
|
['#34d399', 'rgba(52,211,153,.18)'], ['#f472b6', 'rgba(244,114,182,.18)'],
|
|
@@ -137,10 +144,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
137
144
|
const missionGroupBlock = (missionName, subs) => {
|
|
138
145
|
const status = groupStatus(subs);
|
|
139
146
|
const totalSteps = subs.reduce((n, s) => n + (Array.isArray(s.steps) ? s.steps.length : 0), 0);
|
|
140
|
-
const
|
|
141
|
-
const
|
|
142
|
-
const
|
|
147
|
+
const startTimes = subs.map(s => s.startTime || 0).filter(Boolean);
|
|
148
|
+
const endTimes = subs.map(s => s.endTime || 0).filter(Boolean);
|
|
149
|
+
const firstStart = startTimes.length ? Math.min(...startTimes) : null;
|
|
150
|
+
const lastEnd = endTimes.length ? Math.max(...endTimes) : null;
|
|
151
|
+
const groupDur = (firstStart != null && lastEnd != null) ? ((lastEnd - firstStart) / 1000).toFixed(2) : '—';
|
|
143
152
|
const tags = normalizeTags(subs.flatMap(s => s.submissionType === 'mission' ? (s.tags ?? []) : []));
|
|
153
|
+
const sourceFiles = [...new Set(subs.map(s => s.file).filter(Boolean))];
|
|
144
154
|
|
|
145
155
|
// pre → mission → post
|
|
146
156
|
const order = { premission: 0, mission: 1, postmission: 2 };
|
|
@@ -155,7 +165,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
155
165
|
<span class="name">${esc(missionName)}</span>
|
|
156
166
|
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
157
167
|
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
158
|
-
<span class="meta"
|
|
168
|
+
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} • ` : ''}submissions: ${subs.length} • steps: ${totalSteps} • duration: ${groupDur}s</span>
|
|
159
169
|
<span class="toolbar">
|
|
160
170
|
<button class="btn-mini toggle" data-scope="mission" aria-label="Expand">▼</button>
|
|
161
171
|
</span>
|
|
@@ -236,6 +246,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
236
246
|
.header{ max-width:960px; margin:0 auto 16px; padding:16px 18px; }
|
|
237
247
|
h1{ margin:0 0 6px; font-size:22px; font-weight:800; color:#fff; }
|
|
238
248
|
.run-meta{ color:var(--text-muted); display:grid; gap:4px; }
|
|
249
|
+
.provider-meta{ display:inline-flex; align-items:center; gap:7px; }
|
|
250
|
+
.provider-logo{ width:20px; height:20px; color:var(--text); flex:none; }
|
|
251
|
+
.provider-logo img,.provider-logo svg{ display:block; width:20px; height:20px; border:0; border-radius:0; margin:0; background:transparent; }
|
|
239
252
|
|
|
240
253
|
/* ===== Chips ===== */
|
|
241
254
|
.summary{ display:flex; flex-wrap:wrap; gap:12px; margin:16px auto 20px; max-width:960px; }
|
|
@@ -346,10 +359,12 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
346
359
|
<div class="header glass">
|
|
347
360
|
<h1>🧑🚀 <span class="gradient-text">Testronaut Report</span></h1>
|
|
348
361
|
<div class="run-meta">
|
|
362
|
+
<div class="provider-meta"><span class="provider-logo" title="${esc(provider.name)}" aria-label="${esc(provider.name)}">${provider.logo}</span><strong>${esc(llm.model ?? '—')}</strong></div>
|
|
349
363
|
<div><strong>Run ID:</strong> ${esc(runId ?? '—')}</div>
|
|
350
364
|
<div><strong>Start:</strong> ${esc(startTime ?? '—')}</div>
|
|
351
|
-
<div><strong>End:</strong> ${esc(endTime ?? '—')}
|
|
352
|
-
<div><strong>
|
|
365
|
+
<div><strong>End:</strong> ${esc(endTime ?? '—')}</div>
|
|
366
|
+
<div><strong>Duration:</strong> ${durationSec}s</div>
|
|
367
|
+
<div><strong>CLI version:</strong> ${esc(cli.version ?? '—')}</div>
|
|
353
368
|
</div>
|
|
354
369
|
</div>
|
|
355
370
|
|
package/tools/mfaCode.js
CHANGED
|
@@ -9,7 +9,10 @@ export const DEV_API_BASE = 'https://staging.api.testronaut.app';
|
|
|
9
9
|
|
|
10
10
|
const MIN_REMAINING_SECONDS = 5;
|
|
11
11
|
const MAX_REMAINING_SECONDS = 25;
|
|
12
|
-
const API_DEBUG_LOG_RELATIVE_PATH =
|
|
12
|
+
const API_DEBUG_LOG_RELATIVE_PATH = path.join(
|
|
13
|
+
process.env.TESTRONAUT_OUTPUT_DIR || 'missions/mission_reports',
|
|
14
|
+
'api-debug.log'
|
|
15
|
+
);
|
|
13
16
|
const REDACTED = '<redacted>';
|
|
14
17
|
|
|
15
18
|
const SENSITIVE_LOG_KEYS = new Set([
|
package/tools/tokenControl.js
CHANGED
|
@@ -63,6 +63,10 @@ const DEFAULT_LIMITS = [
|
|
|
63
63
|
{ test: /^gemini-2\.5-flash-8b(-|$)/i, tpm: 300000 },
|
|
64
64
|
{ test: /^gemini-2\.5-flash(-|$)/i, tpm: 300000 },
|
|
65
65
|
|
|
66
|
+
// Anthropic limits vary by usage tier; this conservative fallback is only for local pacing.
|
|
67
|
+
{ test: /^claude-(opus|sonnet)-/i, tpm: 80000 },
|
|
68
|
+
{ test: /^claude-haiku-/i, tpm: 100000 },
|
|
69
|
+
|
|
66
70
|
// Ultimate fallback for anything else
|
|
67
71
|
{ test: /.*/, tpm: 150000 },
|
|
68
72
|
];
|
|
@@ -76,7 +80,7 @@ const warnedModels = new Set();
|
|
|
76
80
|
/* ---------------- Tokenizer helpers ----------------
|
|
77
81
|
* We prefer tiktoken's per-model encoding when available.
|
|
78
82
|
* For unknown models, we pick a close base:
|
|
79
|
-
* - o200k_base for modern, long-context families (OpenAI
|
|
83
|
+
* - o200k_base for modern, long-context families (OpenAI, Gemini, Claude, O*)
|
|
80
84
|
* - cl100k_base as a broad fallback
|
|
81
85
|
*/
|
|
82
86
|
/**
|
|
@@ -94,10 +98,12 @@ function getTokenizer(model) {
|
|
|
94
98
|
|
|
95
99
|
// Treat Gemini 2.5 like modern long-context models
|
|
96
100
|
const isGemini25 = /^gemini-2\.5/.test(m);
|
|
101
|
+
const isClaude = /^claude-/.test(m);
|
|
97
102
|
|
|
98
103
|
// OpenAI modern families also map well to o200k_base
|
|
99
104
|
const useO200k =
|
|
100
105
|
isGemini25 ||
|
|
106
|
+
isClaude ||
|
|
101
107
|
m.startsWith('gpt-5') ||
|
|
102
108
|
m.startsWith('gpt-4o') ||
|
|
103
109
|
m.startsWith('gpt-4.1') ||
|