testronaut 1.6.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -4
- package/TODO.md +9 -0
- package/bin/cli.js +267 -37
- package/bin/init.js +35 -5
- package/bin/initHelpers.js +16 -5
- package/core/missionDiscovery.js +3 -3
- package/core/missionLoader.js +16 -0
- package/core/turnLoop.js +18 -15
- package/llm/anthropic/anthropicProvider.js +119 -0
- package/llm/anthropic/models.js +7 -0
- package/llm/llmFactory.js +6 -1
- package/package.json +3 -1
- package/runner/testronaut.js +4 -1
- package/tests/binTests/cli.helpers.test.js +75 -2
- package/tests/binTests/cli.integration.test.js +77 -0
- package/tests/binTests/init.test.js +24 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +3 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +4 -0
- package/tests/binTests/initHelpersTests/models.test.js +3 -1
- package/tests/coreTests/missionDiscovery.test.js +2 -2
- package/tests/coreTests/missionLoader.test.js +52 -0
- package/tests/coreTests/turnLoop.test.js +40 -0
- package/tests/llmTests/anthropicProvider.test.js +41 -0
- package/tests/llmTests/llmFactory.test.js +17 -1
- package/tests/toolsTests/generateHtmlReport.test.js +18 -0
- package/tools/chromeBrowser.js +14 -4
- package/tools/generateHtmlReport.js +22 -7
- package/tools/mfaCode.js +4 -1
- package/tools/tokenControl.js +7 -1
package/bin/init.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* writing a provider/model-aware config, and scaffolding a .env file.
|
|
7
7
|
*
|
|
8
8
|
* Responsibilities:
|
|
9
|
-
* 1) Ask the user for an LLM provider (OpenAI or
|
|
9
|
+
* 1) Ask the user for an LLM provider (OpenAI, Gemini, or Anthropic Claude).
|
|
10
10
|
* 2) Ask for a provider-specific model (keeps prior choice when re-run).
|
|
11
11
|
* 3) Write `testronaut-config.json` and an initial `.env` placeholder.
|
|
12
12
|
* 4) Ensure folder structure and create a welcome mission.
|
|
@@ -33,9 +33,11 @@ import {
|
|
|
33
33
|
defaultConfig,
|
|
34
34
|
makeEnvTemplate,
|
|
35
35
|
geminiModels,
|
|
36
|
+
anthropicModels,
|
|
36
37
|
pickInitialIndex
|
|
37
38
|
} from './initHelpers.js';
|
|
38
39
|
import { DEFAULT_OPENAI_MODEL, OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
40
|
+
import { ANTHROPIC_MODELS, DEFAULT_ANTHROPIC_MODEL } from '../llm/anthropic/models.js';
|
|
39
41
|
|
|
40
42
|
export async function initializeTestronautProject() {
|
|
41
43
|
// ─────────────────────────────────────────────
|
|
@@ -43,7 +45,6 @@ export async function initializeTestronautProject() {
|
|
|
43
45
|
// ─────────────────────────────────────────────
|
|
44
46
|
const root = process.cwd();
|
|
45
47
|
const missionsDir = path.join(root, 'missions');
|
|
46
|
-
const reportsDir = path.join(missionsDir, 'mission_reports');
|
|
47
48
|
const configPath = path.join(root, 'testronaut-config.json');
|
|
48
49
|
const envPath = path.join(root, '.env');
|
|
49
50
|
|
|
@@ -70,10 +71,10 @@ export async function initializeTestronautProject() {
|
|
|
70
71
|
choices: [
|
|
71
72
|
{ title: 'OpenAI (GPT family)', value: 'openai' },
|
|
72
73
|
{ title: 'Google Gemini', value: 'gemini' },
|
|
73
|
-
|
|
74
|
+
{ title: 'Anthropic Claude', value: 'anthropic' },
|
|
74
75
|
],
|
|
75
76
|
// If user re-runs init before writing config, preselect known value.
|
|
76
|
-
initial: ['openai', 'gemini'].indexOf(config.provider ?? 'openai')
|
|
77
|
+
initial: Math.max(0, ['openai', 'gemini', 'anthropic'].indexOf(config.provider ?? 'openai'))
|
|
77
78
|
});
|
|
78
79
|
|
|
79
80
|
// Persist provider explicitly to config.
|
|
@@ -129,6 +130,19 @@ export async function initializeTestronautProject() {
|
|
|
129
130
|
});
|
|
130
131
|
|
|
131
132
|
config.model = geminiModel;
|
|
133
|
+
} else if (llmProvider === 'anthropic') {
|
|
134
|
+
const models = anthropicModels();
|
|
135
|
+
const { anthropicModel } = await prompts({
|
|
136
|
+
type: 'select',
|
|
137
|
+
name: 'anthropicModel',
|
|
138
|
+
message: 'Select an Anthropic Claude model:',
|
|
139
|
+
choices: ANTHROPIC_MODELS.map(model => ({
|
|
140
|
+
title: `${model.label} (${model.description})`,
|
|
141
|
+
value: model.id,
|
|
142
|
+
})),
|
|
143
|
+
initial: pickInitialIndex(models, config.model, DEFAULT_ANTHROPIC_MODEL),
|
|
144
|
+
});
|
|
145
|
+
config.model = anthropicModel;
|
|
132
146
|
}
|
|
133
147
|
|
|
134
148
|
// ─────────────────────────────────────────────
|
|
@@ -139,6 +153,9 @@ export async function initializeTestronautProject() {
|
|
|
139
153
|
config.outputDir = config.outputDir || defaults.outputDir;
|
|
140
154
|
config.projectName = config.projectName || defaults.projectName;
|
|
141
155
|
config.maxTurns = config.maxTurns ?? defaults.maxTurns;
|
|
156
|
+
config.tags = config.tags ?? defaults.tags;
|
|
157
|
+
config.tagMatch = config.tagMatch ?? defaults.tagMatch;
|
|
158
|
+
config.addTags = config.addTags ?? defaults.addTags;
|
|
142
159
|
config.dom = { ...defaults.dom, ...(config.dom || {}) };
|
|
143
160
|
config.resourceGuard = { ...defaults.resourceGuard, ...(config.resourceGuard || {}) };
|
|
144
161
|
config.humanInput = { ...defaults.humanInput, ...(config.humanInput || {}) };
|
|
@@ -181,6 +198,18 @@ export async function initializeTestronautProject() {
|
|
|
181
198
|
|
|
182
199
|
📌 If you plan to use tools/function-calling or images, make sure your account has access to those features.
|
|
183
200
|
|
|
201
|
+
✅ Ready to write missions in \`missions/\`. Try:
|
|
202
|
+
|
|
203
|
+
testronaut welcome.mission.js
|
|
204
|
+
`);
|
|
205
|
+
} else if (config.provider === 'anthropic') {
|
|
206
|
+
console.log(`
|
|
207
|
+
🔧 Setup Complete!
|
|
208
|
+
|
|
209
|
+
👉 Ensure your .env file contains your Anthropic key:
|
|
210
|
+
|
|
211
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
212
|
+
|
|
184
213
|
✅ Ready to write missions in \`missions/\`. Try:
|
|
185
214
|
|
|
186
215
|
testronaut welcome.mission.js
|
|
@@ -197,12 +226,13 @@ export async function initializeTestronautProject() {
|
|
|
197
226
|
// ─────────────────────────────────────────────
|
|
198
227
|
// STEP 5: Ensure folder structure exists (safe to call every run)
|
|
199
228
|
// ─────────────────────────────────────────────
|
|
229
|
+
const reportsDir = path.resolve(root, config.outputDir || 'missions/mission_reports');
|
|
200
230
|
if (!fs.existsSync(missionsDir)) {
|
|
201
231
|
fs.mkdirSync(missionsDir);
|
|
202
232
|
console.log('📁 Created missions/');
|
|
203
233
|
}
|
|
204
234
|
if (!fs.existsSync(reportsDir)) {
|
|
205
235
|
fs.mkdirSync(reportsDir);
|
|
206
|
-
console.log(
|
|
236
|
+
console.log(`📁 Created ${path.relative(root, reportsDir) || reportsDir}/`);
|
|
207
237
|
}
|
|
208
238
|
}
|
package/bin/initHelpers.js
CHANGED
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
29
|
+
import { ANTHROPIC_MODELS } from '../llm/anthropic/models.js';
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Ensures backward compatibility for older configs where
|
|
@@ -76,7 +77,7 @@ export function defaultConfig(rootBasename) {
|
|
|
76
77
|
* Returns a provider-specific .env template string.
|
|
77
78
|
* This scaffolds a safe placeholder API key for the user to fill in manually.
|
|
78
79
|
*
|
|
79
|
-
* @param {'openai'|'gemini'} provider - The chosen LLM provider
|
|
80
|
+
* @param {'openai'|'gemini'|'anthropic'} provider - The chosen LLM provider
|
|
80
81
|
* @returns {string} A multiline template for the .env file
|
|
81
82
|
*/
|
|
82
83
|
export function makeEnvTemplate(provider) {
|
|
@@ -88,6 +89,11 @@ OPENAI_API_KEY=sk-...
|
|
|
88
89
|
if (provider === 'gemini') {
|
|
89
90
|
return `# Add your Google Gemini API key below
|
|
90
91
|
GEMINI_API_KEY=AIza...
|
|
92
|
+
`;
|
|
93
|
+
}
|
|
94
|
+
if (provider === 'anthropic') {
|
|
95
|
+
return `# Add your Anthropic API key below
|
|
96
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
91
97
|
`;
|
|
92
98
|
}
|
|
93
99
|
return '';
|
|
@@ -115,17 +121,22 @@ export function geminiModels() {
|
|
|
115
121
|
];
|
|
116
122
|
}
|
|
117
123
|
|
|
124
|
+
export function anthropicModels() {
|
|
125
|
+
return ANTHROPIC_MODELS.map(({ id }) => id);
|
|
126
|
+
}
|
|
127
|
+
|
|
118
128
|
/**
|
|
119
129
|
* Determines whether a given model name belongs to a known provider.
|
|
120
130
|
*
|
|
121
|
-
* @param {'openai'|'gemini'} provider - The LLM provider
|
|
131
|
+
* @param {'openai'|'gemini'|'anthropic'|'claude'} provider - The LLM provider
|
|
122
132
|
* @param {string} model - The model name to check
|
|
123
133
|
* @returns {boolean} True if the model is recognized for that provider
|
|
124
134
|
*/
|
|
125
135
|
export function isKnownModel(provider, model) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
136
|
+
if (provider === 'openai') return openAIModels().includes(model);
|
|
137
|
+
if (provider === 'gemini') return geminiModels().includes(model);
|
|
138
|
+
if (provider === 'anthropic' || provider === 'claude') return anthropicModels().includes(model);
|
|
139
|
+
return false;
|
|
129
140
|
}
|
|
130
141
|
|
|
131
142
|
/**
|
package/core/missionDiscovery.js
CHANGED
|
@@ -9,7 +9,7 @@ const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
|
|
|
9
9
|
/**
|
|
10
10
|
* Discover mission files based on testronaut-config.json.
|
|
11
11
|
* - When no missions block exists, preserve legacy behavior:
|
|
12
|
-
* read the ./missions folder (non-recursive) for
|
|
12
|
+
* read the ./missions folder (non-recursive) for JavaScript/TypeScript missions.
|
|
13
13
|
* - When missions is defined, apply root/include/exclude globs relative to root.
|
|
14
14
|
*
|
|
15
15
|
* @param {object} opts
|
|
@@ -21,13 +21,13 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
|
|
|
21
21
|
const cfg = await loadConfig(cwd);
|
|
22
22
|
const missionsCfg = cfg?.missions;
|
|
23
23
|
|
|
24
|
-
// Legacy behavior
|
|
24
|
+
// Legacy behavior remains non-recursive, now with first-class TypeScript support.
|
|
25
25
|
if (!missionsCfg) {
|
|
26
26
|
if (!fs.existsSync(defaultRoot)) {
|
|
27
27
|
return { root: defaultRoot, files: [] };
|
|
28
28
|
}
|
|
29
29
|
const files = fs.readdirSync(defaultRoot)
|
|
30
|
-
.filter(f => f.endsWith('.mission.js'))
|
|
30
|
+
.filter(f => f.endsWith('.mission.js') || f.endsWith('.mission.ts'))
|
|
31
31
|
.sort();
|
|
32
32
|
return { root: defaultRoot, files };
|
|
33
33
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { createJiti } from 'jiti';
|
|
2
|
+
|
|
3
|
+
// Mission files are user-authored configuration/code, so Testronaut owns their
|
|
4
|
+
// module loading semantics instead of inheriting the host project's package type.
|
|
5
|
+
const jiti = createJiti(import.meta.url, {
|
|
6
|
+
interopDefault: true,
|
|
7
|
+
sourceMaps: true,
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Load an ESM-style JavaScript or TypeScript mission regardless of whether the
|
|
12
|
+
* consuming project is ESM, CommonJS, or has no package.json `type` field.
|
|
13
|
+
*/
|
|
14
|
+
export async function loadMissionModule(modulePath) {
|
|
15
|
+
return jiti.import(modulePath);
|
|
16
|
+
}
|
package/core/turnLoop.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*
|
|
25
25
|
* Notes:
|
|
26
26
|
* - Provider-agnostic: routes all LLM calls through the `llmFactory` adapter layer.
|
|
27
|
-
* -
|
|
27
|
+
* - All providers normalize responses to an OpenAI-like message format.
|
|
28
28
|
* - Token control is model-sensitive but provider-neutral.
|
|
29
29
|
*/
|
|
30
30
|
|
|
@@ -156,6 +156,8 @@ const FIRE_AND_FORGET_TOOLS = new Set([
|
|
|
156
156
|
'switch_to_page',
|
|
157
157
|
'close_current_page',
|
|
158
158
|
'request_human_input',
|
|
159
|
+
'set_ground_control_state',
|
|
160
|
+
'record_mission_telemetry',
|
|
159
161
|
]);
|
|
160
162
|
|
|
161
163
|
/**
|
|
@@ -252,9 +254,10 @@ export const turnLoop = async (
|
|
|
252
254
|
const maxAttempts = retryLimitClamped + 1; // includes initial attempt
|
|
253
255
|
ctx.groundControl = groundControl;
|
|
254
256
|
const humanInput = ctx.humanInput || { enabled: true, timeoutSeconds: 60 };
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
257
|
+
const disabledTools = new Set();
|
|
258
|
+
if (humanInput.enabled === false) disabledTools.add('request_human_input');
|
|
259
|
+
if (process.env.TESTRONAUT_SCREENSHOTS === '0') disabledTools.add('screenshot');
|
|
260
|
+
const activeToolsSchema = toolsSchema.filter(t => !disabledTools.has(t?.function?.name));
|
|
258
261
|
let agentMemory = { lastMenuExpanded: false, humanInput };
|
|
259
262
|
ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
260
263
|
let turnRetries = 0;
|
|
@@ -433,7 +436,17 @@ export const turnLoop = async (
|
|
|
433
436
|
let result;
|
|
434
437
|
let errorMessage = null;
|
|
435
438
|
try {
|
|
436
|
-
|
|
439
|
+
if (fnName === 'set_ground_control_state') {
|
|
440
|
+
applyGroundControlUpdate(groundControl, args);
|
|
441
|
+
result = { ok: true, groundControl };
|
|
442
|
+
} else if (fnName === 'record_mission_telemetry') {
|
|
443
|
+
const recorded = recordGroundTelemetry(groundControl, args, { turn });
|
|
444
|
+
result = { ok: true, recorded };
|
|
445
|
+
} else {
|
|
446
|
+
const toolHandler = CHROME_TOOL_MAP[fnName];
|
|
447
|
+
if (typeof toolHandler !== 'function') throw new Error(`Unknown tool: ${fnName}`);
|
|
448
|
+
result = await toolHandler(browser, args, agentMemory);
|
|
449
|
+
}
|
|
437
450
|
if (typeof result !== 'string') result = JSON.stringify(result ?? '');
|
|
438
451
|
} catch (e) {
|
|
439
452
|
errorMessage = `ERROR: ${e.message}`;
|
|
@@ -649,16 +662,6 @@ export const turnLoop = async (
|
|
|
649
662
|
}
|
|
650
663
|
}
|
|
651
664
|
|
|
652
|
-
if (fnName === 'set_ground_control_state') {
|
|
653
|
-
applyGroundControlUpdate(groundControl, args);
|
|
654
|
-
result = JSON.stringify({ ok: true, groundControl });
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
if (fnName === 'record_mission_telemetry') {
|
|
658
|
-
const recorded = recordGroundTelemetry(groundControl, args, { turn });
|
|
659
|
-
result = JSON.stringify({ ok: true, recorded });
|
|
660
|
-
}
|
|
661
|
-
|
|
662
665
|
// Decide what to send back to the LLM for this tool.
|
|
663
666
|
// - For "fire-and-forget" tools, send a tiny stub (OK/error) to avoid
|
|
664
667
|
// bloating context with large payloads (file JSON, etc.).
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
|
|
3
|
+
function textBlocks(content) {
|
|
4
|
+
const parts = Array.isArray(content) ? content : [{ type: 'text', text: content ?? '' }];
|
|
5
|
+
return parts.map(part => {
|
|
6
|
+
if (typeof part === 'string') return { type: 'text', text: part };
|
|
7
|
+
if (part?.type === 'image') {
|
|
8
|
+
return {
|
|
9
|
+
type: 'image',
|
|
10
|
+
source: {
|
|
11
|
+
type: 'base64',
|
|
12
|
+
media_type: part.mimeType,
|
|
13
|
+
data: Buffer.isBuffer(part.data) ? part.data.toString('base64') : String(part.data ?? ''),
|
|
14
|
+
},
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
return { type: 'text', text: part?.text ?? '' };
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function appendMessage(messages, role, blocks) {
|
|
22
|
+
if (!blocks.length) return;
|
|
23
|
+
const previous = messages.at(-1);
|
|
24
|
+
if (previous?.role === role) {
|
|
25
|
+
const existing = Array.isArray(previous.content)
|
|
26
|
+
? previous.content
|
|
27
|
+
: [{ type: 'text', text: previous.content }];
|
|
28
|
+
previous.content = [...existing, ...blocks];
|
|
29
|
+
} else {
|
|
30
|
+
messages.push({ role, content: blocks });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function toAnthropicRequest(messages = []) {
|
|
35
|
+
const system = [];
|
|
36
|
+
const converted = [];
|
|
37
|
+
|
|
38
|
+
for (const message of messages) {
|
|
39
|
+
if (message.role === 'system') {
|
|
40
|
+
system.push(...textBlocks(message.content).filter(block => block.type === 'text'));
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (message.role === 'tool') {
|
|
45
|
+
appendMessage(converted, 'user', [{
|
|
46
|
+
type: 'tool_result',
|
|
47
|
+
tool_use_id: message.tool_call_id,
|
|
48
|
+
content: String(message.content ?? ''),
|
|
49
|
+
}]);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const blocks = textBlocks(message.content).filter(block => block.type !== 'text' || block.text);
|
|
54
|
+
if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
|
|
55
|
+
for (const call of message.tool_calls) {
|
|
56
|
+
let input = {};
|
|
57
|
+
try { input = JSON.parse(call.function?.arguments ?? '{}'); } catch { /* keep safe empty input */ }
|
|
58
|
+
blocks.push({
|
|
59
|
+
type: 'tool_use',
|
|
60
|
+
id: call.id,
|
|
61
|
+
name: call.function?.name,
|
|
62
|
+
input,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
appendMessage(converted, message.role === 'assistant' ? 'assistant' : 'user', blocks);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return { system, messages: converted };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function fromAnthropicMessage(response) {
|
|
73
|
+
const text = [];
|
|
74
|
+
const tool_calls = [];
|
|
75
|
+
for (const block of response?.content ?? []) {
|
|
76
|
+
if (block.type === 'text') text.push(block.text);
|
|
77
|
+
if (block.type === 'tool_use') {
|
|
78
|
+
tool_calls.push({
|
|
79
|
+
id: block.id,
|
|
80
|
+
type: 'function',
|
|
81
|
+
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
role: 'assistant',
|
|
87
|
+
content: text.join(''),
|
|
88
|
+
tool_calls: tool_calls.length ? tool_calls : undefined,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export class AnthropicProvider {
|
|
93
|
+
constructor({ apiKey, maxTokens = 8192 } = {}) {
|
|
94
|
+
if (!apiKey) throw new Error('ANTHROPIC_API_KEY is required for Anthropic provider');
|
|
95
|
+
this.client = new Anthropic({ apiKey });
|
|
96
|
+
this.maxTokens = maxTokens;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async chat({ model, messages, tools }) {
|
|
100
|
+
const converted = toAnthropicRequest(messages);
|
|
101
|
+
const response = await this.client.messages.create({
|
|
102
|
+
model,
|
|
103
|
+
max_tokens: this.maxTokens,
|
|
104
|
+
system: converted.system.length ? converted.system : undefined,
|
|
105
|
+
messages: converted.messages,
|
|
106
|
+
tools: tools?.length ? tools.map(tool => ({
|
|
107
|
+
name: tool.function?.name ?? tool.name,
|
|
108
|
+
description: tool.function?.description ?? tool.description ?? '',
|
|
109
|
+
input_schema: tool.function?.parameters ?? tool.parameters ?? { type: 'object', properties: {} },
|
|
110
|
+
})) : undefined,
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const usage = {
|
|
114
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
115
|
+
providerRaw: response.usage,
|
|
116
|
+
};
|
|
117
|
+
return { message: fromAnthropicMessage(response), usage };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export const DEFAULT_ANTHROPIC_MODEL = 'claude-sonnet-5';
|
|
2
|
+
|
|
3
|
+
export const ANTHROPIC_MODELS = [
|
|
4
|
+
{ id: 'claude-sonnet-5', label: 'Claude Sonnet 5', description: 'best balance of capability and speed' },
|
|
5
|
+
{ id: 'claude-opus-5', label: 'Claude Opus 5', description: 'highest capability' },
|
|
6
|
+
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', description: 'fastest and most cost-efficient' },
|
|
7
|
+
];
|
package/llm/llmFactory.js
CHANGED
|
@@ -26,12 +26,13 @@
|
|
|
26
26
|
|
|
27
27
|
import { OpenAIProvider } from './openAI/openaiProvider.js';
|
|
28
28
|
import { GeminiProvider } from './gemini/geminiProvider.js';
|
|
29
|
+
import { AnthropicProvider } from './anthropic/anthropicProvider.js';
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Return an adapter implementing:
|
|
32
33
|
* chat({ model, messages, tools }) -> Promise<{ message, usage }>
|
|
33
34
|
*
|
|
34
|
-
* @param {string} providerName - 'openai' | 'gemini'
|
|
35
|
+
* @param {string} providerName - 'openai' | 'gemini' | 'anthropic' | 'claude'
|
|
35
36
|
* @param {object} [opts] - Optional overrides (e.g., { apiKey, baseURL, ... })
|
|
36
37
|
* @returns {{ chat: Function }} provider adapter
|
|
37
38
|
*/
|
|
@@ -44,6 +45,10 @@ export function getLLM(providerName, opts = {}) {
|
|
|
44
45
|
case 'gemini':
|
|
45
46
|
return new GeminiProvider({ apiKey: process.env.GEMINI_API_KEY, ...opts });
|
|
46
47
|
|
|
48
|
+
case 'anthropic':
|
|
49
|
+
case 'claude':
|
|
50
|
+
return new AnthropicProvider({ apiKey: process.env.ANTHROPIC_API_KEY, ...opts });
|
|
51
|
+
|
|
47
52
|
default:
|
|
48
53
|
throw new Error(`Unsupported LLM provider: ${providerName}`);
|
|
49
54
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"testronaut": "bin/cli.js"
|
|
@@ -24,12 +24,14 @@
|
|
|
24
24
|
"url": "https://github.com/mission-testronaut/testronaut-cli"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
+
"@anthropic-ai/sdk": "^0.124.0",
|
|
27
28
|
"@dqbd/tiktoken": "^1.0.21",
|
|
28
29
|
"@google/generative-ai": "^0.24.1",
|
|
29
30
|
"cheerio": "^1.1.0",
|
|
30
31
|
"dotenv": "^17.2.0",
|
|
31
32
|
"fast-glob": "^3.3.3",
|
|
32
33
|
"inquirer": "^12.9.1",
|
|
34
|
+
"jiti": "^2.7.0",
|
|
33
35
|
"node-fetch": "^3.3.2",
|
|
34
36
|
"openai": "^6.39.0",
|
|
35
37
|
"playwright": "^1.54.1",
|
package/runner/testronaut.js
CHANGED
|
@@ -23,9 +23,11 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
// Mission modules commonly interpolate project .env values at module load time.
|
|
27
|
+
// Keep env loading eager even though the browser/agent import below is lazy.
|
|
28
|
+
import 'dotenv/config';
|
|
26
29
|
import fs from 'fs';
|
|
27
30
|
import path from 'path';
|
|
28
|
-
import { runAgent } from '../core/agent.js';
|
|
29
31
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
32
|
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
33
|
import { normalizeTags } from '../core/tags.js';
|
|
@@ -202,6 +204,7 @@ export async function runMissions({ preMission, mission, postMission, tags = []
|
|
|
202
204
|
);
|
|
203
205
|
|
|
204
206
|
// 3) Execute
|
|
207
|
+
const { runAgent } = await import('../core/agent.js');
|
|
205
208
|
const success = await runAgent(
|
|
206
209
|
goals,
|
|
207
210
|
missionName,
|
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
1
|
+
import { afterEach, describe, it, expect } from 'vitest';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import fs from 'fs';
|
|
4
|
+
import os from 'os';
|
|
4
5
|
|
|
5
6
|
// Import helper exports from cli.js
|
|
6
7
|
import { __test__ } from '../../bin/cli.js';
|
|
7
8
|
|
|
9
|
+
const tempDirs = [];
|
|
10
|
+
const makeTempDir = (prefix) => {
|
|
11
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
12
|
+
tempDirs.push(dir);
|
|
13
|
+
return dir;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
18
|
+
});
|
|
19
|
+
|
|
8
20
|
describe('cli helpers', () => {
|
|
9
21
|
it('guessMimeType returns expected types', () => {
|
|
10
22
|
const { guessMimeType } = __test__;
|
|
@@ -22,7 +34,7 @@ describe('cli helpers', () => {
|
|
|
22
34
|
|
|
23
35
|
it('findLatestReportPair finds latest html/json', () => {
|
|
24
36
|
const { findLatestReportPair } = __test__;
|
|
25
|
-
const tmp =
|
|
37
|
+
const tmp = makeTempDir('testronaut-report-');
|
|
26
38
|
const r1 = path.join(tmp, 'run_100.html');
|
|
27
39
|
const r2 = path.join(tmp, 'run_200.html');
|
|
28
40
|
fs.writeFileSync(r1, 'a');
|
|
@@ -34,6 +46,67 @@ describe('cli helpers', () => {
|
|
|
34
46
|
expect(latest?.jsonFile).toBe('run_200.json');
|
|
35
47
|
});
|
|
36
48
|
|
|
49
|
+
it('resolves bare mission names from the missions root and direct paths from cwd', () => {
|
|
50
|
+
const { resolveMissionPath } = __test__;
|
|
51
|
+
const cwd = '/tmp/project';
|
|
52
|
+
const missionsRoot = '/tmp/project/missions';
|
|
53
|
+
expect(resolveMissionPath('login.mission.js', { cwd, missionsRoot }))
|
|
54
|
+
.toBe('/tmp/project/missions/login.mission.js');
|
|
55
|
+
expect(resolveMissionPath('missions/login.mission.js', { cwd, missionsRoot }))
|
|
56
|
+
.toBe('/tmp/project/missions/login.mission.js');
|
|
57
|
+
expect(resolveMissionPath('../shared/login.mission.js', { cwd, missionsRoot }))
|
|
58
|
+
.toBe('/tmp/shared/login.mission.js');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('collects only unique screenshots referenced by a report', () => {
|
|
62
|
+
const { collectReportScreenshotNames, collectReportScreenshotPaths } = __test__;
|
|
63
|
+
const report = { missions: [
|
|
64
|
+
{ steps: [{ screenshotPath: './screenshots/one.png' }, { screenshotPath: './screenshots/two.jpg' }] },
|
|
65
|
+
{ steps: [{ screenshotPath: './screenshots/one.png' }, { events: [] }] },
|
|
66
|
+
] };
|
|
67
|
+
expect(collectReportScreenshotNames(report)).toEqual(['one.png', 'two.jpg']);
|
|
68
|
+
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/one.png', 'screenshots/two.jpg']);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('preserves run-specific screenshot paths', () => {
|
|
72
|
+
const { collectReportScreenshotPaths } = __test__;
|
|
73
|
+
const report = { missions: [{ steps: [
|
|
74
|
+
{ screenshotPath: './screenshots/run_123/one.png' },
|
|
75
|
+
] }] };
|
|
76
|
+
expect(collectReportScreenshotPaths(report)).toEqual(['screenshots/run_123/one.png']);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('resolves configured report directories and suggests close mission names', () => {
|
|
80
|
+
const { resolveReportDir, closestMissionMatch } = __test__;
|
|
81
|
+
expect(resolveReportDir({ outputDir: 'artifacts/reports' }, '/tmp/project'))
|
|
82
|
+
.toBe('/tmp/project/artifacts/reports');
|
|
83
|
+
expect(closestMissionMatch('logn.mission.js', ['checkout.mission.js', 'login.mission.js']))
|
|
84
|
+
.toBe('login.mission.js');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('resolves selected reports by run id, filename, and direct path', () => {
|
|
88
|
+
const { resolveReportFile } = __test__;
|
|
89
|
+
const tmp = makeTempDir('testronaut-selected-report-');
|
|
90
|
+
const reportDir = path.join(tmp, 'reports');
|
|
91
|
+
fs.mkdirSync(reportDir);
|
|
92
|
+
const report = path.join(reportDir, 'run_123.json');
|
|
93
|
+
fs.writeFileSync(report, '{}');
|
|
94
|
+
|
|
95
|
+
expect(resolveReportFile('run_123', reportDir, tmp)).toBe(report);
|
|
96
|
+
expect(resolveReportFile('run_123.json', reportDir, tmp)).toBe(report);
|
|
97
|
+
expect(resolveReportFile(report, reportDir, tmp)).toBe(report);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('builds a redacted effective config with source information', () => {
|
|
101
|
+
const { buildEffectiveConfig } = __test__;
|
|
102
|
+
const effective = buildEffectiveConfig({
|
|
103
|
+
provider: 'openai', model: 'gpt-5.6', outputDir: 'artifacts', sessionToken: 'secret',
|
|
104
|
+
}, '/tmp/project');
|
|
105
|
+
expect(effective.outputDir).toEqual({ value: '/tmp/project/artifacts', source: 'config' });
|
|
106
|
+
expect(effective.authenticated).toBe(true);
|
|
107
|
+
expect(JSON.stringify(effective)).not.toContain('secret');
|
|
108
|
+
});
|
|
109
|
+
|
|
37
110
|
it('parses booleans in a tolerant way', () => {
|
|
38
111
|
const { parseBool } = __test__;
|
|
39
112
|
expect(parseBool('true')).toBe(true);
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
|
|
8
|
+
const cliPath = fileURLToPath(new URL('../../bin/cli.js', import.meta.url));
|
|
9
|
+
const tempDirs = [];
|
|
10
|
+
|
|
11
|
+
function makeProject() {
|
|
12
|
+
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-cli-integration-'));
|
|
13
|
+
tempDirs.push(cwd);
|
|
14
|
+
fs.mkdirSync(path.join(cwd, 'missions'));
|
|
15
|
+
fs.writeFileSync(path.join(cwd, 'missions', 'login.mission.js'), [
|
|
16
|
+
'export const tags = ["authentication", "smoke"];',
|
|
17
|
+
'export function executeMission() { throw new Error("must not execute"); }',
|
|
18
|
+
].join('\n'));
|
|
19
|
+
fs.writeFileSync(path.join(cwd, 'testronaut-config.json'), JSON.stringify({
|
|
20
|
+
provider: 'openai', model: 'gpt-5.6', outputDir: 'artifacts/reports', sessionToken: 'private',
|
|
21
|
+
}));
|
|
22
|
+
return cwd;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function run(cwd, args) {
|
|
26
|
+
const result = spawnSync(process.execPath, [cliPath, ...args], { cwd, encoding: 'utf8' });
|
|
27
|
+
if (result.error) throw result.error;
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
while (tempDirs.length) fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe('CLI inspection commands', () => {
|
|
36
|
+
it('lists mission tags without executing the mission', () => {
|
|
37
|
+
const result = run(makeProject(), ['list', '--json']);
|
|
38
|
+
expect(result.status).toBe(0);
|
|
39
|
+
expect(JSON.parse(result.stdout).missions).toEqual([
|
|
40
|
+
{ file: 'login.mission.js', tags: ['authentication', 'smoke'] },
|
|
41
|
+
]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('reports effective config without exposing the session token', () => {
|
|
45
|
+
const result = run(makeProject(), ['config', '--json']);
|
|
46
|
+
expect(result.status).toBe(0);
|
|
47
|
+
const output = JSON.parse(result.stdout);
|
|
48
|
+
expect(output.outputDir.value).toMatch(/artifacts[/\\]reports$/);
|
|
49
|
+
expect(output.authenticated).toBe(true);
|
|
50
|
+
expect(result.stdout).not.toContain('private');
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('suggests a close mission name and exits nonzero during dry-run', () => {
|
|
54
|
+
const result = run(makeProject(), ['logn.mission.js', '--dry-run', '--json']);
|
|
55
|
+
expect(result.status).toBe(1);
|
|
56
|
+
expect(JSON.parse(result.stdout).missions[0]).toMatchObject({
|
|
57
|
+
status: 'missing', suggestion: 'login.mission.js',
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('loads project .env values before evaluating mission modules', () => {
|
|
62
|
+
const cwd = makeProject();
|
|
63
|
+
fs.writeFileSync(path.join(cwd, '.env'), 'MISSION_DESTINATION=https://example.test\n');
|
|
64
|
+
fs.writeFileSync(path.join(cwd, 'missions', 'login.mission.js'), [
|
|
65
|
+
'const destination = process.env.MISSION_DESTINATION;',
|
|
66
|
+
'export const tags = ["authentication"];',
|
|
67
|
+
'export async function executeMission() {',
|
|
68
|
+
' return { missionName: destination, status: "passed", steps: [] };',
|
|
69
|
+
'}',
|
|
70
|
+
].join('\n'));
|
|
71
|
+
|
|
72
|
+
const result = run(cwd, ['--tags', 'authentication', '--json', '--quiet']);
|
|
73
|
+
|
|
74
|
+
expect(result.status).toBe(0);
|
|
75
|
+
expect(JSON.parse(result.stdout).missions[0].missionName).toBe('https://example.test');
|
|
76
|
+
});
|
|
77
|
+
});
|