testronaut 1.6.2 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/bin/cli.js +2 -2
- package/bin/init.js +30 -3
- package/bin/initHelpers.js +16 -5
- package/core/turnLoop.js +1 -1
- package/llm/anthropic/anthropicProvider.js +119 -0
- package/llm/anthropic/models.js +7 -0
- package/llm/llmFactory.js +6 -1
- package/package.json +3 -2
- package/tests/binTests/init.test.js +15 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +4 -0
- package/tests/binTests/initHelpersTests/models.test.js +3 -1
- package/tests/llmTests/anthropicProvider.test.js +41 -0
- package/tests/llmTests/llmFactory.test.js +17 -1
- package/tests/toolsTests/generateHtmlReport.test.js +14 -0
- package/tools/generateHtmlReport.js +13 -2
- package/tools/tokenControl.js +7 -1
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@ Join the Discord to connect with other Testronauts, get support, and help shape
|
|
|
18
18
|
|
|
19
19
|
- Write tests in plain English — no brittle selectors
|
|
20
20
|
- Runs real browser sessions via **Playwright**
|
|
21
|
-
- Works with **multiple LLM providers** (OpenAI, Google Gemini, and
|
|
21
|
+
- Works with **multiple LLM providers** (OpenAI, Google Gemini, and Anthropic Claude)
|
|
22
22
|
- Modular tool and DOM-reasoning system
|
|
23
23
|
- Dynamic rate-limit and token-tracking logic
|
|
24
24
|
- Generates JSON + HTML reports automatically
|
|
@@ -120,6 +120,9 @@ OPENAI_API_KEY=sk-...
|
|
|
120
120
|
# Or for Gemini
|
|
121
121
|
GEMINI_API_KEY=AIza...
|
|
122
122
|
|
|
123
|
+
# Or for Anthropic Claude
|
|
124
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
125
|
+
|
|
123
126
|
URL=https://example.com/login
|
|
124
127
|
USERNAME=example@example.com
|
|
125
128
|
PASSWORD=********
|
|
@@ -146,8 +149,9 @@ Current supported providers:
|
|
|
146
149
|
|-----------|----------------|
|
|
147
150
|
| **OpenAI** | gpt-5.6, gpt-5.6-terra, gpt-5.6-luna, gpt-5.5, and legacy GPT/o-series models |
|
|
148
151
|
| **Google Gemini** | gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-8b |
|
|
152
|
+
| **Anthropic Claude** | claude-sonnet-5, claude-opus-5, claude-haiku-4-5 |
|
|
149
153
|
|
|
150
|
-
|
|
154
|
+
The canonical Claude provider ID is `anthropic`; `claude` is also accepted as a CLI alias.
|
|
151
155
|
|
|
152
156
|
---
|
|
153
157
|
|
package/bin/cli.js
CHANGED
|
@@ -172,7 +172,7 @@ function parseProviderArgs(argsList) {
|
|
|
172
172
|
provider = provider.trim();
|
|
173
173
|
}
|
|
174
174
|
|
|
175
|
-
const supportedProviders = new Set(['openai', 'gemini']);
|
|
175
|
+
const supportedProviders = new Set(['openai', 'gemini', 'anthropic', 'claude']);
|
|
176
176
|
const isValid = (v) => !!v && supportedProviders.has(String(v).toLowerCase());
|
|
177
177
|
if (!isValid(provider)) {
|
|
178
178
|
invalid = true;
|
|
@@ -728,7 +728,7 @@ Options:
|
|
|
728
728
|
--init Scaffold project folders and a welcome mission
|
|
729
729
|
--turns=<n> Override max turns for this run (e.g., --turns=30)
|
|
730
730
|
--debug[=<bool>] Enable verbose debug logs (or set TESTRONAUT_DEBUG=1)
|
|
731
|
-
--provider=<id> Override LLM provider
|
|
731
|
+
--provider=<id> Override LLM provider (openai, gemini, or anthropic)
|
|
732
732
|
-o, --options key=value Set run options, such as mfa=github-test-mfa
|
|
733
733
|
--dev Use the staging API base URL
|
|
734
734
|
--vercel-bypass=<secret> Send Vercel protection bypass header for protected deployments
|
package/bin/init.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* writing a provider/model-aware config, and scaffolding a .env file.
|
|
7
7
|
*
|
|
8
8
|
* Responsibilities:
|
|
9
|
-
* 1) Ask the user for an LLM provider (OpenAI or
|
|
9
|
+
* 1) Ask the user for an LLM provider (OpenAI, Gemini, or Anthropic Claude).
|
|
10
10
|
* 2) Ask for a provider-specific model (keeps prior choice when re-run).
|
|
11
11
|
* 3) Write `testronaut-config.json` and an initial `.env` placeholder.
|
|
12
12
|
* 4) Ensure folder structure and create a welcome mission.
|
|
@@ -33,9 +33,11 @@ import {
|
|
|
33
33
|
defaultConfig,
|
|
34
34
|
makeEnvTemplate,
|
|
35
35
|
geminiModels,
|
|
36
|
+
anthropicModels,
|
|
36
37
|
pickInitialIndex
|
|
37
38
|
} from './initHelpers.js';
|
|
38
39
|
import { DEFAULT_OPENAI_MODEL, OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
40
|
+
import { ANTHROPIC_MODELS, DEFAULT_ANTHROPIC_MODEL } from '../llm/anthropic/models.js';
|
|
39
41
|
|
|
40
42
|
export async function initializeTestronautProject() {
|
|
41
43
|
// ─────────────────────────────────────────────
|
|
@@ -69,10 +71,10 @@ export async function initializeTestronautProject() {
|
|
|
69
71
|
choices: [
|
|
70
72
|
{ title: 'OpenAI (GPT family)', value: 'openai' },
|
|
71
73
|
{ title: 'Google Gemini', value: 'gemini' },
|
|
72
|
-
|
|
74
|
+
{ title: 'Anthropic Claude', value: 'anthropic' },
|
|
73
75
|
],
|
|
74
76
|
// If user re-runs init before writing config, preselect known value.
|
|
75
|
-
initial: ['openai', 'gemini'].indexOf(config.provider ?? 'openai')
|
|
77
|
+
initial: Math.max(0, ['openai', 'gemini', 'anthropic'].indexOf(config.provider ?? 'openai'))
|
|
76
78
|
});
|
|
77
79
|
|
|
78
80
|
// Persist provider explicitly to config.
|
|
@@ -128,6 +130,19 @@ export async function initializeTestronautProject() {
|
|
|
128
130
|
});
|
|
129
131
|
|
|
130
132
|
config.model = geminiModel;
|
|
133
|
+
} else if (llmProvider === 'anthropic') {
|
|
134
|
+
const models = anthropicModels();
|
|
135
|
+
const { anthropicModel } = await prompts({
|
|
136
|
+
type: 'select',
|
|
137
|
+
name: 'anthropicModel',
|
|
138
|
+
message: 'Select an Anthropic Claude model:',
|
|
139
|
+
choices: ANTHROPIC_MODELS.map(model => ({
|
|
140
|
+
title: `${model.label} (${model.description})`,
|
|
141
|
+
value: model.id,
|
|
142
|
+
})),
|
|
143
|
+
initial: pickInitialIndex(models, config.model, DEFAULT_ANTHROPIC_MODEL),
|
|
144
|
+
});
|
|
145
|
+
config.model = anthropicModel;
|
|
131
146
|
}
|
|
132
147
|
|
|
133
148
|
// ─────────────────────────────────────────────
|
|
@@ -183,6 +198,18 @@ export async function initializeTestronautProject() {
|
|
|
183
198
|
|
|
184
199
|
📌 If you plan to use tools/function-calling or images, make sure your account has access to those features.
|
|
185
200
|
|
|
201
|
+
✅ Ready to write missions in \`missions/\`. Try:
|
|
202
|
+
|
|
203
|
+
testronaut welcome.mission.js
|
|
204
|
+
`);
|
|
205
|
+
} else if (config.provider === 'anthropic') {
|
|
206
|
+
console.log(`
|
|
207
|
+
🔧 Setup Complete!
|
|
208
|
+
|
|
209
|
+
👉 Ensure your .env file contains your Anthropic key:
|
|
210
|
+
|
|
211
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
212
|
+
|
|
186
213
|
✅ Ready to write missions in \`missions/\`. Try:
|
|
187
214
|
|
|
188
215
|
testronaut welcome.mission.js
|
package/bin/initHelpers.js
CHANGED
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
29
|
+
import { ANTHROPIC_MODELS } from '../llm/anthropic/models.js';
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Ensures backward compatibility for older configs where
|
|
@@ -76,7 +77,7 @@ export function defaultConfig(rootBasename) {
|
|
|
76
77
|
* Returns a provider-specific .env template string.
|
|
77
78
|
* This scaffolds a safe placeholder API key for the user to fill in manually.
|
|
78
79
|
*
|
|
79
|
-
* @param {'openai'|'gemini'} provider - The chosen LLM provider
|
|
80
|
+
* @param {'openai'|'gemini'|'anthropic'} provider - The chosen LLM provider
|
|
80
81
|
* @returns {string} A multiline template for the .env file
|
|
81
82
|
*/
|
|
82
83
|
export function makeEnvTemplate(provider) {
|
|
@@ -88,6 +89,11 @@ OPENAI_API_KEY=sk-...
|
|
|
88
89
|
if (provider === 'gemini') {
|
|
89
90
|
return `# Add your Google Gemini API key below
|
|
90
91
|
GEMINI_API_KEY=AIza...
|
|
92
|
+
`;
|
|
93
|
+
}
|
|
94
|
+
if (provider === 'anthropic') {
|
|
95
|
+
return `# Add your Anthropic API key below
|
|
96
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
91
97
|
`;
|
|
92
98
|
}
|
|
93
99
|
return '';
|
|
@@ -115,17 +121,22 @@ export function geminiModels() {
|
|
|
115
121
|
];
|
|
116
122
|
}
|
|
117
123
|
|
|
124
|
+
export function anthropicModels() {
|
|
125
|
+
return ANTHROPIC_MODELS.map(({ id }) => id);
|
|
126
|
+
}
|
|
127
|
+
|
|
118
128
|
/**
|
|
119
129
|
* Determines whether a given model name belongs to a known provider.
|
|
120
130
|
*
|
|
121
|
-
* @param {'openai'|'gemini'} provider - The LLM provider
|
|
131
|
+
* @param {'openai'|'gemini'|'anthropic'|'claude'} provider - The LLM provider
|
|
122
132
|
* @param {string} model - The model name to check
|
|
123
133
|
* @returns {boolean} True if the model is recognized for that provider
|
|
124
134
|
*/
|
|
125
135
|
export function isKnownModel(provider, model) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
136
|
+
if (provider === 'openai') return openAIModels().includes(model);
|
|
137
|
+
if (provider === 'gemini') return geminiModels().includes(model);
|
|
138
|
+
if (provider === 'anthropic' || provider === 'claude') return anthropicModels().includes(model);
|
|
139
|
+
return false;
|
|
129
140
|
}
|
|
130
141
|
|
|
131
142
|
/**
|
package/core/turnLoop.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*
|
|
25
25
|
* Notes:
|
|
26
26
|
* - Provider-agnostic: routes all LLM calls through the `llmFactory` adapter layer.
|
|
27
|
-
* -
|
|
27
|
+
* - All providers normalize responses to an OpenAI-like message format.
|
|
28
28
|
* - Token control is model-sensitive but provider-neutral.
|
|
29
29
|
*/
|
|
30
30
|
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
|
|
3
|
+
function textBlocks(content) {
|
|
4
|
+
const parts = Array.isArray(content) ? content : [{ type: 'text', text: content ?? '' }];
|
|
5
|
+
return parts.map(part => {
|
|
6
|
+
if (typeof part === 'string') return { type: 'text', text: part };
|
|
7
|
+
if (part?.type === 'image') {
|
|
8
|
+
return {
|
|
9
|
+
type: 'image',
|
|
10
|
+
source: {
|
|
11
|
+
type: 'base64',
|
|
12
|
+
media_type: part.mimeType,
|
|
13
|
+
data: Buffer.isBuffer(part.data) ? part.data.toString('base64') : String(part.data ?? ''),
|
|
14
|
+
},
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
return { type: 'text', text: part?.text ?? '' };
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function appendMessage(messages, role, blocks) {
|
|
22
|
+
if (!blocks.length) return;
|
|
23
|
+
const previous = messages.at(-1);
|
|
24
|
+
if (previous?.role === role) {
|
|
25
|
+
const existing = Array.isArray(previous.content)
|
|
26
|
+
? previous.content
|
|
27
|
+
: [{ type: 'text', text: previous.content }];
|
|
28
|
+
previous.content = [...existing, ...blocks];
|
|
29
|
+
} else {
|
|
30
|
+
messages.push({ role, content: blocks });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function toAnthropicRequest(messages = []) {
|
|
35
|
+
const system = [];
|
|
36
|
+
const converted = [];
|
|
37
|
+
|
|
38
|
+
for (const message of messages) {
|
|
39
|
+
if (message.role === 'system') {
|
|
40
|
+
system.push(...textBlocks(message.content).filter(block => block.type === 'text'));
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (message.role === 'tool') {
|
|
45
|
+
appendMessage(converted, 'user', [{
|
|
46
|
+
type: 'tool_result',
|
|
47
|
+
tool_use_id: message.tool_call_id,
|
|
48
|
+
content: String(message.content ?? ''),
|
|
49
|
+
}]);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const blocks = textBlocks(message.content).filter(block => block.type !== 'text' || block.text);
|
|
54
|
+
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
|
|
55
|
+
for (const call of message.tool_calls) {
|
|
56
|
+
let input = {};
|
|
57
|
+
try { input = JSON.parse(call.function?.arguments ?? '{}'); } catch { /* keep safe empty input */ }
|
|
58
|
+
blocks.push({
|
|
59
|
+
type: 'tool_use',
|
|
60
|
+
id: call.id,
|
|
61
|
+
name: call.function?.name,
|
|
62
|
+
input,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
appendMessage(converted, message.role === 'assistant' ? 'assistant' : 'user', blocks);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return { system, messages: converted };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function fromAnthropicMessage(response) {
|
|
73
|
+
const text = [];
|
|
74
|
+
const tool_calls = [];
|
|
75
|
+
for (const block of response?.content ?? []) {
|
|
76
|
+
if (block.type === 'text') text.push(block.text);
|
|
77
|
+
if (block.type === 'tool_use') {
|
|
78
|
+
tool_calls.push({
|
|
79
|
+
id: block.id,
|
|
80
|
+
type: 'function',
|
|
81
|
+
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
role: 'assistant',
|
|
87
|
+
content: text.join(''),
|
|
88
|
+
tool_calls: tool_calls.length ? tool_calls : undefined,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export class AnthropicProvider {
|
|
93
|
+
constructor({ apiKey, maxTokens = 8192 } = {}) {
|
|
94
|
+
if (!apiKey) throw new Error('ANTHROPIC_API_KEY is required for Anthropic provider');
|
|
95
|
+
this.client = new Anthropic({ apiKey });
|
|
96
|
+
this.maxTokens = maxTokens;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async chat({ model, messages, tools }) {
|
|
100
|
+
const converted = toAnthropicRequest(messages);
|
|
101
|
+
const response = await this.client.messages.create({
|
|
102
|
+
model,
|
|
103
|
+
max_tokens: this.maxTokens,
|
|
104
|
+
system: converted.system.length ? converted.system : undefined,
|
|
105
|
+
messages: converted.messages,
|
|
106
|
+
tools: tools?.length ? tools.map(tool => ({
|
|
107
|
+
name: tool.function?.name ?? tool.name,
|
|
108
|
+
description: tool.function?.description ?? tool.description ?? '',
|
|
109
|
+
input_schema: tool.function?.parameters ?? tool.parameters ?? { type: 'object', properties: {} },
|
|
110
|
+
})) : undefined,
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const usage = {
|
|
114
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
115
|
+
providerRaw: response.usage,
|
|
116
|
+
};
|
|
117
|
+
return { message: fromAnthropicMessage(response), usage };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export const DEFAULT_ANTHROPIC_MODEL = 'claude-sonnet-5';
|
|
2
|
+
|
|
3
|
+
export const ANTHROPIC_MODELS = [
|
|
4
|
+
{ id: 'claude-sonnet-5', label: 'Claude Sonnet 5', description: 'best balance of capability and speed' },
|
|
5
|
+
{ id: 'claude-opus-5', label: 'Claude Opus 5', description: 'highest capability' },
|
|
6
|
+
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', description: 'fastest and most cost-efficient' },
|
|
7
|
+
];
|
package/llm/llmFactory.js
CHANGED
|
@@ -26,12 +26,13 @@
|
|
|
26
26
|
|
|
27
27
|
import { OpenAIProvider } from './openAI/openaiProvider.js';
|
|
28
28
|
import { GeminiProvider } from './gemini/geminiProvider.js';
|
|
29
|
+
import { AnthropicProvider } from './anthropic/anthropicProvider.js';
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Return an adapter implementing:
|
|
32
33
|
* chat({ model, messages, tools }) -> Promise<{ message, usage }>
|
|
33
34
|
*
|
|
34
|
-
* @param {string} providerName - 'openai' | 'gemini'
|
|
35
|
+
* @param {string} providerName - 'openai' | 'gemini' | 'anthropic' | 'claude'
|
|
35
36
|
* @param {object} [opts] - Optional overrides (e.g., { apiKey, baseURL, ... })
|
|
36
37
|
* @returns {{ chat: Function }} provider adapter
|
|
37
38
|
*/
|
|
@@ -44,6 +45,10 @@ export function getLLM(providerName, opts = {}) {
|
|
|
44
45
|
case 'gemini':
|
|
45
46
|
return new GeminiProvider({ apiKey: process.env.GEMINI_API_KEY, ...opts });
|
|
46
47
|
|
|
48
|
+
case 'anthropic':
|
|
49
|
+
case 'claude':
|
|
50
|
+
return new AnthropicProvider({ apiKey: process.env.ANTHROPIC_API_KEY, ...opts });
|
|
51
|
+
|
|
47
52
|
default:
|
|
48
53
|
throw new Error(`Unsupported LLM provider: ${providerName}`);
|
|
49
54
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.1",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"testronaut": "bin/cli.js"
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"url": "https://github.com/mission-testronaut/testronaut-cli"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
+
"@anthropic-ai/sdk": "^0.124.0",
|
|
27
28
|
"@dqbd/tiktoken": "^1.0.21",
|
|
28
29
|
"@google/generative-ai": "^0.24.1",
|
|
29
30
|
"cheerio": "^1.1.0",
|
|
@@ -46,6 +47,6 @@
|
|
|
46
47
|
"devDependencies": {
|
|
47
48
|
"@types/node": "^25.9.3",
|
|
48
49
|
"periapsis": "^2.0.1",
|
|
49
|
-
"vitest": "^4.1.
|
|
50
|
+
"vitest": "^4.1.11"
|
|
50
51
|
}
|
|
51
52
|
}
|
|
@@ -138,6 +138,21 @@ describe('initializeTestronautProject', () => {
|
|
|
138
138
|
expect(shared.calls).toBe(2);
|
|
139
139
|
});
|
|
140
140
|
|
|
141
|
+
it('scaffolds a new Anthropic project', async () => {
|
|
142
|
+
const temp = makeTempProject();
|
|
143
|
+
process.chdir(temp);
|
|
144
|
+
shared.answers.push({ llmProvider: 'anthropic' });
|
|
145
|
+
shared.answers.push({ anthropicModel: 'claude-sonnet-5' });
|
|
146
|
+
|
|
147
|
+
await initializeTestronautProject();
|
|
148
|
+
|
|
149
|
+
const cfg = readJson(path.join(temp, 'testronaut-config.json'));
|
|
150
|
+
expect(cfg.provider).toBe('anthropic');
|
|
151
|
+
expect(cfg.model).toBe('claude-sonnet-5');
|
|
152
|
+
expect(fs.readFileSync(path.join(temp, '.env'), 'utf8')).toMatch(/ANTHROPIC_API_KEY=sk-ant-/);
|
|
153
|
+
expect(shared.calls).toBe(2);
|
|
154
|
+
});
|
|
155
|
+
|
|
141
156
|
it('is idempotent: when initialized, it skips prompts and does not overwrite .env', async () => {
|
|
142
157
|
const temp = makeTempProject();
|
|
143
158
|
process.chdir(temp);
|
|
@@ -11,4 +11,8 @@ describe('makeEnvTemplate', () => {
|
|
|
11
11
|
const s = makeEnvTemplate('gemini');
|
|
12
12
|
expect(s).toMatch(/GEMINI_API_KEY=AIza\.\.\./);
|
|
13
13
|
});
|
|
14
|
+
|
|
15
|
+
it('returns Anthropic template', () => {
|
|
16
|
+
expect(makeEnvTemplate('anthropic')).toMatch(/ANTHROPIC_API_KEY=sk-ant-/);
|
|
17
|
+
});
|
|
14
18
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
|
|
2
|
+
import { openAIModels, geminiModels, anthropicModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
|
|
3
3
|
|
|
4
4
|
describe('model helpers', () => {
|
|
5
5
|
it('lists known models', () => {
|
|
@@ -16,11 +16,13 @@ describe('model helpers', () => {
|
|
|
16
16
|
]));
|
|
17
17
|
expect(openAIModels()).not.toContain('gpt-5.3');
|
|
18
18
|
expect(geminiModels()).toContain('gemini-2.5-flash');
|
|
19
|
+
expect(anthropicModels()).toContain('claude-sonnet-5');
|
|
19
20
|
});
|
|
20
21
|
|
|
21
22
|
it('recognizes known models by provider', () => {
|
|
22
23
|
expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
|
|
23
24
|
expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
|
|
25
|
+
expect(isKnownModel('anthropic', 'claude-sonnet-5')).toBe(true);
|
|
24
26
|
});
|
|
25
27
|
|
|
26
28
|
it('picks initial index from list or fallback', () => {
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const { create } = vi.hoisted(() => ({ create: vi.fn() }));
|
|
4
|
+
vi.mock('@anthropic-ai/sdk', () => ({
|
|
5
|
+
default: class Anthropic { constructor(options) { this.options = options; this.messages = { create }; } },
|
|
6
|
+
}));
|
|
7
|
+
|
|
8
|
+
import { AnthropicProvider, fromAnthropicMessage, toAnthropicRequest } from '../../llm/anthropic/anthropicProvider.js';
|
|
9
|
+
|
|
10
|
+
describe('AnthropicProvider', () => {
|
|
11
|
+
it('requires an API key', () => {
|
|
12
|
+
expect(() => new AnthropicProvider()).toThrow(/ANTHROPIC_API_KEY/);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it('converts system, images, tool calls, and tool results', () => {
|
|
16
|
+
const request = toAnthropicRequest([
|
|
17
|
+
{ role: 'system', content: 'Be precise' },
|
|
18
|
+
{ role: 'user', content: [{ type: 'text', text: 'Inspect' }, { type: 'image', mimeType: 'image/png', data: Buffer.from('png') }] },
|
|
19
|
+
{ role: 'assistant', content: '', tool_calls: [{ id: 'call_1', function: { name: 'click', arguments: '{"selector":"#go"}' } }] },
|
|
20
|
+
{ role: 'tool', tool_call_id: 'call_1', content: 'clicked' },
|
|
21
|
+
]);
|
|
22
|
+
expect(request.system).toEqual([{ type: 'text', text: 'Be precise' }]);
|
|
23
|
+
expect(request.messages[0].content[1].source).toMatchObject({ type: 'base64', media_type: 'image/png' });
|
|
24
|
+
expect(request.messages[1].content[0]).toMatchObject({ type: 'tool_use', id: 'call_1', name: 'click' });
|
|
25
|
+
expect(request.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'call_1' });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('normalizes Claude output and usage', async () => {
|
|
29
|
+
create.mockResolvedValueOnce({
|
|
30
|
+
content: [{ type: 'text', text: 'Working' }, { type: 'tool_use', id: 'toolu_1', name: 'navigate', input: { url: 'https://example.com' } }],
|
|
31
|
+
usage: { input_tokens: 12, output_tokens: 8 },
|
|
32
|
+
});
|
|
33
|
+
const result = await new AnthropicProvider({ apiKey: 'sk-ant-test' }).chat({
|
|
34
|
+
model: 'claude-sonnet-5', messages: [{ role: 'user', content: 'Go' }],
|
|
35
|
+
tools: [{ type: 'function', function: { name: 'navigate', description: 'Navigate', parameters: { type: 'object' } } }],
|
|
36
|
+
});
|
|
37
|
+
expect(create).toHaveBeenCalledWith(expect.objectContaining({ model: 'claude-sonnet-5', max_tokens: 8192 }));
|
|
38
|
+
expect(result.message).toEqual(fromAnthropicMessage({ content: [{ type: 'text', text: 'Working' }, { type: 'tool_use', id: 'toolu_1', name: 'navigate', input: { url: 'https://example.com' } }] }));
|
|
39
|
+
expect(result.usage.total_tokens).toBe(20);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
@@ -6,6 +6,7 @@ const { shared } = vi.hoisted(() => ({
|
|
|
6
6
|
shared: {
|
|
7
7
|
openAICtor: vi.fn(),
|
|
8
8
|
geminiCtor: vi.fn(),
|
|
9
|
+
anthropicCtor: vi.fn(),
|
|
9
10
|
},
|
|
10
11
|
}));
|
|
11
12
|
|
|
@@ -24,6 +25,12 @@ vi.mock('../../llm/gemini/geminiProvider.js', () => ({
|
|
|
24
25
|
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
25
26
|
},
|
|
26
27
|
}));
|
|
28
|
+
vi.mock('../../llm/anthropic/anthropicProvider.js', () => ({
|
|
29
|
+
AnthropicProvider: function (opts) {
|
|
30
|
+
shared.anthropicCtor(opts);
|
|
31
|
+
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
32
|
+
},
|
|
33
|
+
}));
|
|
27
34
|
|
|
28
35
|
// Import SUT after mocks
|
|
29
36
|
import { getLLM } from '../../llm/llmFactory.js';
|
|
@@ -35,6 +42,7 @@ describe('getLLM', () => {
|
|
|
35
42
|
process.env = { ...ORIGINAL_ENV };
|
|
36
43
|
shared.openAICtor.mockClear();
|
|
37
44
|
shared.geminiCtor.mockClear();
|
|
45
|
+
shared.anthropicCtor.mockClear();
|
|
38
46
|
});
|
|
39
47
|
|
|
40
48
|
afterEach(() => {
|
|
@@ -72,7 +80,15 @@ describe('getLLM', () => {
|
|
|
72
80
|
expect(typeof llm.chat).toBe('function');
|
|
73
81
|
});
|
|
74
82
|
|
|
83
|
+
it('creates Anthropic provider and accepts claude as an alias', () => {
|
|
84
|
+
process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
|
|
85
|
+
getLLM('claude');
|
|
86
|
+
expect(shared.anthropicCtor).toHaveBeenCalledWith(
|
|
87
|
+
expect.objectContaining({ apiKey: 'sk-ant-test' })
|
|
88
|
+
);
|
|
89
|
+
});
|
|
90
|
+
|
|
75
91
|
it('throws on unsupported provider', () => {
|
|
76
|
-
expect(() => getLLM('
|
|
92
|
+
expect(() => getLLM('mistral')).toThrow(/Unsupported LLM provider/i);
|
|
77
93
|
});
|
|
78
94
|
});
|
|
@@ -21,6 +21,7 @@ describe('generateHtmlReport', () => {
|
|
|
21
21
|
it('writes an HTML report containing mission and retry metadata', () => {
|
|
22
22
|
const report = {
|
|
23
23
|
runId: 'run_test',
|
|
24
|
+
llm: { provider: 'anthropic', model: 'claude-sonnet-5' },
|
|
24
25
|
cli: { version: '1.7.0' },
|
|
25
26
|
missions: [
|
|
26
27
|
{
|
|
@@ -66,6 +67,9 @@ describe('generateHtmlReport', () => {
|
|
|
66
67
|
expect(html).toContain('No missions match these tags');
|
|
67
68
|
expect(html).toContain('Keep missions that do not match visible');
|
|
68
69
|
expect(html).not.toContain('<div class="pill">LLM:');
|
|
70
|
+
expect(html).not.toContain('<strong>LLM:</strong>');
|
|
71
|
+
expect(html).toContain('title="Anthropic"');
|
|
72
|
+
expect(html).toContain('<strong>claude-sonnet-5</strong>');
|
|
69
73
|
});
|
|
70
74
|
|
|
71
75
|
it('escapes unsafe text in output', () => {
|
|
@@ -99,4 +103,14 @@ describe('generateHtmlReport', () => {
|
|
|
99
103
|
expect(html).toContain('<script>alert(1)</script>');
|
|
100
104
|
expect(html).toContain('<b>bold</b>');
|
|
101
105
|
});
|
|
106
|
+
|
|
107
|
+
it('renders the OpenAI logo and model without a textual LLM label', () => {
|
|
108
|
+
const outPath = path.join(tmpDir, 'openai.html');
|
|
109
|
+
generateHtmlReport({ runId: 'openai_run', llm: { provider: 'openai', model: 'gpt-5.6' } }, outPath);
|
|
110
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
111
|
+
|
|
112
|
+
expect(html).toContain('title="OpenAI"');
|
|
113
|
+
expect(html).toContain('<strong>gpt-5.6</strong>');
|
|
114
|
+
expect(html).not.toContain('<strong>LLM:</strong>');
|
|
115
|
+
});
|
|
102
116
|
});
|
|
@@ -42,6 +42,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
42
42
|
const badge = (status) =>
|
|
43
43
|
status === 'passed' ? '✅ Passed' :
|
|
44
44
|
status === 'failed' ? '❌ Failed' : (status || '—');
|
|
45
|
+
const providerKey = String(llm.provider ?? '').trim().toLowerCase() === 'claude'
|
|
46
|
+
? 'anthropic'
|
|
47
|
+
: String(llm.provider ?? '').trim().toLowerCase();
|
|
48
|
+
const provider = {
|
|
49
|
+
openai: { name: 'OpenAI', logo: '<svg viewBox="0 0 24 24" aria-hidden="true"><path fill="currentColor" d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"/></svg>' },
|
|
50
|
+
anthropic: { name: 'Anthropic', logo: '<img src="data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8dGl0bGU+Q2xhdWRlPC90aXRsZT4KICA8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHJ4PSI1IiBmaWxsPSIjRDk3NzU3Ii8+CiAgPHBhdGggZmlsbD0iI2ZmZiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMyAzKSBzY2FsZSguNzUpIiBkPSJNNC43MDkgMTUuOTU1bDQuNzItMi42NDcuMDgtLjIzLS4wOC0uMTI4SDkuMmwtLjc5LS4wNDgtMi42OTgtLjA3My0yLjMzOS0uMDk3LTIuMjY2LS4xMjItLjU3MS0uMTIxTDAgMTEuNzg0bC4wNTUtLjM1Mi40OC0uMzIxLjY4Ni4wNiAxLjUyLjEwMyAyLjI3OC4xNTggMS42NTIuMDk3IDIuNDQ5LjI1NWguMzg5bC4wNTUtLjE1Ny0uMTM0LS4wOTgtLjEwMy0uMDk3LTIuMzU4LTEuNTk2LTIuNTUyLTEuNjg4LTEuMzM2LS45NzItLjcyNC0uNDkxLS4zNjQtLjQ2Mi0uMTU4LTEuMDA4LjY1Ni0uNzIyLjg4MS4wNi4yMjUuMDYxLjg5My42ODYgMS45MDggMS40NzYgMi40OTEgMS44MzMuMzY1LjMwNC4xNDUtLjEwMy4wMTktLjA3My0uMTY0LS4yNzQtMS4zNTUtMi40NDYtMS40NDYtMi40OS0uNjQ0LTEuMDMyLS4xNy0uNjE5YTIuOTcgMi45NyAwIDAxLS4xMDQtLjcyOUw2LjI4My4xMzQgNi42OTYgMGwuOTk2LjEzNC40Mi4zNjQuNjIgMS40MTQgMS4wMDIgMi4yMjkgMS41NTUgMy4wMy40NTYuODk4LjI0My44MzIuMDkxLjI1NWguMTU4VjkuMDFsLjEyOC0xLjcwNi4yMzctMi4wOTUuMjMtMi42OTUuMDgtLjc2LjM3Ni0uOTEuNzQ3LS40OTIuNTg0LjI4LjQ4LjY4NS0uMDY3LjQ0NC0uMjg2IDEuODUxLS41NTkgMi45MDMtLjM2NCAxLjk0MmguMjEybC4yNDMtLjI0Mi45ODUtMS4zMDYgMS42NTItMi4wNjQuNzMtLjgyLjg1LS45MDQuNTQ3LS40MzFoMS4wMzNsLjc2IDEuMTI5LS4zNCAxLjE2Ni0xLjA2NCAxLjM0Ny0uODgxIDEuMTQyLTEuMjY0IDEuNy0uNzkgMS4zNi4wNzMuMTEuMTg4LS4wMiAyLjg1Ni0uNjA2IDEuNTQzLS4yOCAxLjg0MS0uMzE1LjgzMy4zODguMDkxLjM5NS0uMzI4LjgwNy0xLjk2OS40ODYtMi4zMDkuNDYyLTMuNDM5LjgxMy0uMDQyLjAzLjA0OS4wNjEgMS41NDkuMTQ2LjY2Mi4wMzZoMS42MjJsMy4wMi4yMjUuNzkuNTIyLjQ3NC42MzgtLjA3OS40ODUtMS4yMTUuNjItMS42NC0uMzg5LTMuODI5LS45MS0xLjMxMi0uMzI5aC0uMTgydi4xMWwxLjA5MyAxLjA2OCAyLjAwNiAxLjgxIDIuNTA5IDIuMzMuMTI3LjU3OC0uMzIyLjQ1NS0uMzQtLjA0OS0yLjIwNS0xLjY1Ny0uODUxLS43NDctMS45MjYtMS42MmgtLjEyOHYuMTdsLjQ0NC42NDkgMi4zNDUgMy41MjEuMTIyIDEuMDgtLjE3LjM1My0uNjA4LjIxMy0uNjY4LS4xMjItMS4zNzQtMS45MjUtMS40MTUtMi4xNjctMS4xNDMtMS45NDMtLjE0LjA4LS42NzQgNy4yNTQtLjMxNi4zNy0uNzI5LjI4LS42MDctLjQ2MS0uMzIyLS43NDcuMzIyLTEuNDc2LjM4OS0xLjkyNC4zMTUtMS41My4yODYtMS45LjE3LS42MzItLjAxMi0uMDQyLS4xNC4wMTgtMS40MzQgMS45NjctMi4xOCAyLjk0NS0xLjcyNiAxLjg0NS0uNDE0LjE2NC0uNzE3LS4zNy4wNjctLjY2Mi40MDEtLjU4OSAyLjM4OC0zLjAzNiAxLjQ0LTEuODgyLjkzLTEuMDg2LS4wMDYtLjE1OGgtLjA1NUw0LjEzMiAxOC41NmwtMS4xMy4xNDYtLjQ4Ny0uNDU2LjA2MS0uNzQ2LjIzMS0uMjQzIDEuOTA4LTEuMzEyLS4wMDYuMDA2eiIvPgo8L3N2Zz4K" alt="" />' },
|
|
51
|
+
}[providerKey] ?? { name: llm.provider || 'Unknown provider', logo: '' };
|
|
45
52
|
const tagPalette = [
|
|
46
53
|
['#60a5fa', 'rgba(96,165,250,.18)'], ['#a78bfa', 'rgba(167,139,250,.18)'],
|
|
47
54
|
['#34d399', 'rgba(52,211,153,.18)'], ['#f472b6', 'rgba(244,114,182,.18)'],
|
|
@@ -239,6 +246,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
239
246
|
.header{ max-width:960px; margin:0 auto 16px; padding:16px 18px; }
|
|
240
247
|
h1{ margin:0 0 6px; font-size:22px; font-weight:800; color:#fff; }
|
|
241
248
|
.run-meta{ color:var(--text-muted); display:grid; gap:4px; }
|
|
249
|
+
.provider-meta{ display:inline-flex; align-items:center; gap:7px; }
|
|
250
|
+
.provider-logo{ width:20px; height:20px; color:var(--text); flex:none; }
|
|
251
|
+
.provider-logo img,.provider-logo svg{ display:block; width:20px; height:20px; border:0; border-radius:0; margin:0; background:transparent; }
|
|
242
252
|
|
|
243
253
|
/* ===== Chips ===== */
|
|
244
254
|
.summary{ display:flex; flex-wrap:wrap; gap:12px; margin:16px auto 20px; max-width:960px; }
|
|
@@ -349,10 +359,11 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
349
359
|
<div class="header glass">
|
|
350
360
|
<h1>🧑🚀 <span class="gradient-text">Testronaut Report</span></h1>
|
|
351
361
|
<div class="run-meta">
|
|
362
|
+
<div class="provider-meta"><span class="provider-logo" title="${esc(provider.name)}" aria-label="${esc(provider.name)}">${provider.logo}</span><strong>${esc(llm.model ?? '—')}</strong></div>
|
|
352
363
|
<div><strong>Run ID:</strong> ${esc(runId ?? '—')}</div>
|
|
353
364
|
<div><strong>Start:</strong> ${esc(startTime ?? '—')}</div>
|
|
354
|
-
<div><strong>End:</strong> ${esc(endTime ?? '—')}
|
|
355
|
-
<div><strong>
|
|
365
|
+
<div><strong>End:</strong> ${esc(endTime ?? '—')}</div>
|
|
366
|
+
<div><strong>Duration:</strong> ${durationSec}s</div>
|
|
356
367
|
<div><strong>CLI version:</strong> ${esc(cli.version ?? '—')}</div>
|
|
357
368
|
</div>
|
|
358
369
|
</div>
|
package/tools/tokenControl.js
CHANGED
|
@@ -63,6 +63,10 @@ const DEFAULT_LIMITS = [
|
|
|
63
63
|
{ test: /^gemini-2\.5-flash-8b(-|$)/i, tpm: 300000 },
|
|
64
64
|
{ test: /^gemini-2\.5-flash(-|$)/i, tpm: 300000 },
|
|
65
65
|
|
|
66
|
+
// Anthropic limits vary by usage tier; this conservative fallback is only for local pacing.
|
|
67
|
+
{ test: /^claude-(opus|sonnet)-/i, tpm: 80000 },
|
|
68
|
+
{ test: /^claude-haiku-/i, tpm: 100000 },
|
|
69
|
+
|
|
66
70
|
// Ultimate fallback for anything else
|
|
67
71
|
{ test: /.*/, tpm: 150000 },
|
|
68
72
|
];
|
|
@@ -76,7 +80,7 @@ const warnedModels = new Set();
|
|
|
76
80
|
/* ---------------- Tokenizer helpers ----------------
|
|
77
81
|
* We prefer tiktoken's per-model encoding when available.
|
|
78
82
|
* For unknown models, we pick a close base:
|
|
79
|
-
* - o200k_base for modern, long-context families (OpenAI
|
|
83
|
+
* - o200k_base for modern, long-context families (OpenAI, Gemini, Claude, O*)
|
|
80
84
|
* - cl100k_base as a broad fallback
|
|
81
85
|
*/
|
|
82
86
|
/**
|
|
@@ -94,10 +98,12 @@ function getTokenizer(model) {
|
|
|
94
98
|
|
|
95
99
|
// Treat Gemini 2.5 like modern long-context models
|
|
96
100
|
const isGemini25 = /^gemini-2\.5/.test(m);
|
|
101
|
+
const isClaude = /^claude-/.test(m);
|
|
97
102
|
|
|
98
103
|
// OpenAI modern families also map well to o200k_base
|
|
99
104
|
const useO200k =
|
|
100
105
|
isGemini25 ||
|
|
106
|
+
isClaude ||
|
|
101
107
|
m.startsWith('gpt-5') ||
|
|
102
108
|
m.startsWith('gpt-4o') ||
|
|
103
109
|
m.startsWith('gpt-4.1') ||
|