testronaut 1.4.1 β 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/cli.js +114 -0
- package/bin/createWelcomeMission.js +5 -2
- package/bin/init.js +21 -21
- package/bin/initHelpers.js +6 -12
- package/core/tags.js +50 -0
- package/core/turnLoop.js +3 -1
- package/llm/modelResolver.js +4 -3
- package/llm/openAI/models.js +32 -0
- package/llm/openAI/openaiProvider.js +13 -2
- package/package.json +1 -1
- package/runner/testronaut.js +11 -1
- package/tests/binTests/cli.helpers.test.js +43 -0
- package/tests/binTests/initHelpersTests/models.test.js +11 -0
- package/tests/coreTests/tags.test.js +31 -0
- package/tests/coreTests/turnLoop.test.js +1 -0
- package/tests/llmTests/modelResolver.test.js +4 -4
- package/tests/llmTests/openAIProvider.test.js +18 -0
- package/tests/runnerTests/testronaut.test.js +35 -0
- package/tests/toolsTests/generateHtmlReport.test.js +8 -0
- package/tests/toolsTests/tokenControl.test.js +19 -0
- package/todo.md +17 -0
- package/tools/generateHtmlReport.js +93 -2
- package/tools/tokenControl.js +27 -1
package/README.md
CHANGED
|
@@ -134,7 +134,7 @@ Current supported providers:
|
|
|
134
134
|
|
|
135
135
|
| Provider | Example Models |
|
|
136
136
|
|-----------|----------------|
|
|
137
|
-
| **OpenAI** | gpt-
|
|
137
|
+
| **OpenAI** | gpt-5.6, gpt-5.6-terra, gpt-5.6-luna, gpt-5.5, and legacy GPT/o-series models |
|
|
138
138
|
| **Google Gemini** | gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-8b |
|
|
139
139
|
|
|
140
140
|
More providers coming soon (Anthropic, Mistral, etc.).
|
package/bin/cli.js
CHANGED
|
@@ -51,6 +51,8 @@ const exec = promisify(execCmd);
|
|
|
51
51
|
import url from 'url';
|
|
52
52
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
53
53
|
import { discoverMissionFiles } from '../core/missionDiscovery.js';
|
|
54
|
+
import { loadConfig } from '../core/config.js';
|
|
55
|
+
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../core/tags.js';
|
|
54
56
|
|
|
55
57
|
// Keep PW browsers inside the project to avoid global cache skew
|
|
56
58
|
process.env.PLAYWRIGHT_BROWSERS_PATH = process.env.PLAYWRIGHT_BROWSERS_PATH || '0';
|
|
@@ -311,6 +313,61 @@ export const __test__ = {
|
|
|
311
313
|
isDirectInvocation,
|
|
312
314
|
};
|
|
313
315
|
|
|
316
|
+
function parseTagArgs(argsList) {
|
|
317
|
+
const nextArgs = [];
|
|
318
|
+
const tagValues = [];
|
|
319
|
+
const addTagValues = [];
|
|
320
|
+
let tagMatchValue;
|
|
321
|
+
let tagsPresent = false;
|
|
322
|
+
let addTagsPresent = false;
|
|
323
|
+
let tagMatchPresent = false;
|
|
324
|
+
let tagsMissingValue = false;
|
|
325
|
+
let addTagsMissingValue = false;
|
|
326
|
+
|
|
327
|
+
const matchFlag = (raw, names) => names.some(name => raw === name || raw.startsWith(`${name}=`));
|
|
328
|
+
const readValue = (raw, index) => {
|
|
329
|
+
if (raw.includes('=')) return { value: raw.slice(raw.indexOf('=') + 1), consumed: 1 };
|
|
330
|
+
const candidate = argsList[index + 1];
|
|
331
|
+
return candidate && !candidate.startsWith('-')
|
|
332
|
+
? { value: candidate, consumed: 2 }
|
|
333
|
+
: { value: undefined, consumed: 1 };
|
|
334
|
+
};
|
|
335
|
+
|
|
336
|
+
for (let index = 0; index < argsList.length;) {
|
|
337
|
+
const raw = argsList[index];
|
|
338
|
+
if (matchFlag(raw, ['--tag', '--tags'])) {
|
|
339
|
+
const { value, consumed } = readValue(raw, index);
|
|
340
|
+
tagsPresent = true;
|
|
341
|
+
if (value !== undefined) tagValues.push(value);
|
|
342
|
+
else tagsMissingValue = true;
|
|
343
|
+
index += consumed;
|
|
344
|
+
} else if (matchFlag(raw, ['--add-tag', '--add-tags', '--add_tag', '--add_tags'])) {
|
|
345
|
+
const { value, consumed } = readValue(raw, index);
|
|
346
|
+
addTagsPresent = true;
|
|
347
|
+
if (value !== undefined) addTagValues.push(value);
|
|
348
|
+
else addTagsMissingValue = true;
|
|
349
|
+
index += consumed;
|
|
350
|
+
} else if (matchFlag(raw, ['--tag-match', '--tag_match'])) {
|
|
351
|
+
const { value, consumed } = readValue(raw, index);
|
|
352
|
+
tagMatchPresent = true;
|
|
353
|
+
tagMatchValue = value;
|
|
354
|
+
index += consumed;
|
|
355
|
+
} else {
|
|
356
|
+
nextArgs.push(raw);
|
|
357
|
+
index += 1;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
return {
|
|
362
|
+
args: nextArgs,
|
|
363
|
+
tags: { present: tagsPresent, value: tagValues.length ? tagValues.join(',') : undefined, missingValue: tagsMissingValue },
|
|
364
|
+
addTags: { present: addTagsPresent, value: addTagValues.length ? addTagValues.join(',') : undefined, missingValue: addTagsMissingValue },
|
|
365
|
+
tagMatch: { present: tagMatchPresent, value: tagMatchValue },
|
|
366
|
+
hasUnquotedCommaSpace: [...tagValues, ...addTagValues].some(value => value.trimEnd().endsWith(',')),
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
__test__.parseTagArgs = parseTagArgs;
|
|
370
|
+
|
|
314
371
|
// Look for --model=<id> or --model <id>
|
|
315
372
|
let modelOverride;
|
|
316
373
|
const modelFlagIndex = args.findIndex(a => a === '--model' || a.startsWith('--model='));
|
|
@@ -356,6 +413,26 @@ if (cliMfaName) {
|
|
|
356
413
|
console.log(`π MFA nickname override: ${process.env.TESTRONAUT_MFA_NAME}`);
|
|
357
414
|
}
|
|
358
415
|
|
|
416
|
+
const tagArgs = parseTagArgs(args);
|
|
417
|
+
args = tagArgs.args;
|
|
418
|
+
let cliTags;
|
|
419
|
+
let cliAddTags = [];
|
|
420
|
+
let cliTagMatch;
|
|
421
|
+
try {
|
|
422
|
+
if (tagArgs.hasUnquotedCommaSpace) {
|
|
423
|
+
throw new Error('A tag list ends with a comma. Quote comma-and-space lists (for example --tags "authentication, smoke") or repeat --tag for each tag.');
|
|
424
|
+
}
|
|
425
|
+
if (tagArgs.tags.missingValue || (tagArgs.tags.present && !tagArgs.tags.value)) throw new Error('--tag/--tags requires a tag value.');
|
|
426
|
+
if (tagArgs.addTags.missingValue || (tagArgs.addTags.present && !tagArgs.addTags.value)) throw new Error('--add-tag/--add-tags requires a tag value.');
|
|
427
|
+
if (tagArgs.tagMatch.present && !tagArgs.tagMatch.value) throw new Error('--tag-match requires "any" or "all".');
|
|
428
|
+
if (tagArgs.tags.present) cliTags = normalizeTags(tagArgs.tags.value, { allowUntagged: true });
|
|
429
|
+
if (tagArgs.addTags.present) cliAddTags = normalizeTags(tagArgs.addTags.value);
|
|
430
|
+
if (tagArgs.tagMatch.present) cliTagMatch = normalizeTagMatch(tagArgs.tagMatch.value);
|
|
431
|
+
} catch (error) {
|
|
432
|
+
console.error(`β ${error.message}`);
|
|
433
|
+
process.exit(1);
|
|
434
|
+
}
|
|
435
|
+
|
|
359
436
|
// Look for --debug / --debug=<bool> / --no-debug
|
|
360
437
|
// Normalize CLI boolean strings into true/false/null for optional flags.
|
|
361
438
|
const parseBool = (v) => {
|
|
@@ -545,6 +622,11 @@ Options:
|
|
|
545
622
|
--human-input-timeout=<s> Override human input wait timeout in seconds (default: 60)
|
|
546
623
|
--help Show this help message
|
|
547
624
|
--retry_limit=<n> Override agent turn retry limits (minimum 1, maximum 10)
|
|
625
|
+
--tag=<tag> Run missions matching a tag; repeat for multiple tags
|
|
626
|
+
--tags=<tag,...> Compact comma-list form (OR/any by default)
|
|
627
|
+
--tag-match=<any|all> Match any or all requested tags
|
|
628
|
+
--add-tag=<tag> Add one report tag; repeat for multiple tags
|
|
629
|
+
--add-tags=<tag,...> Compact comma-list form for report tags
|
|
548
630
|
|
|
549
631
|
Examples:
|
|
550
632
|
${cliName}
|
|
@@ -552,6 +634,8 @@ Examples:
|
|
|
552
634
|
${cliName} upload
|
|
553
635
|
${cliName} serve
|
|
554
636
|
${cliName} --init
|
|
637
|
+
${cliName} --tag authentication --tag smoke
|
|
638
|
+
${cliName} --add-tag staging --add-tag full-test-run
|
|
555
639
|
`;
|
|
556
640
|
|
|
557
641
|
async function main() {
|
|
@@ -619,6 +703,21 @@ if (args.includes('serve') || args.includes('view')) {
|
|
|
619
703
|
}
|
|
620
704
|
|
|
621
705
|
const { root: missionsRoot, files: discoveredMissions } = await discoverMissionFiles({ cwd: process.cwd() });
|
|
706
|
+
const tagConfig = await loadConfig(process.cwd());
|
|
707
|
+
const explicitFiles = args.length > 0;
|
|
708
|
+
let requestedTags = [];
|
|
709
|
+
let tagMatch = 'any';
|
|
710
|
+
try {
|
|
711
|
+
requestedTags = explicitFiles ? [] : (cliTags ?? normalizeTags(tagConfig?.tags, { allowUntagged: true }));
|
|
712
|
+
tagMatch = explicitFiles ? 'any' : normalizeTagMatch(cliTagMatch ?? tagConfig?.tagMatch);
|
|
713
|
+
process.env.TESTRONAUT_ADD_TAGS = normalizeTags([
|
|
714
|
+
...normalizeTags(tagConfig?.addTags),
|
|
715
|
+
...cliAddTags,
|
|
716
|
+
]).join(',');
|
|
717
|
+
} catch (error) {
|
|
718
|
+
console.error(`β ${error.message}`);
|
|
719
|
+
process.exit(1);
|
|
720
|
+
}
|
|
622
721
|
|
|
623
722
|
if (!fs.existsSync(missionsRoot)) {
|
|
624
723
|
console.error(`β Missions directory not found: ${path.relative(process.cwd(), missionsRoot)}`);
|
|
@@ -630,17 +729,24 @@ const runFile = async (filePath) => {
|
|
|
630
729
|
const modulePath = path.resolve(missionsRoot, filePath);
|
|
631
730
|
const missionsModule = await import(`file://${modulePath}`);
|
|
632
731
|
|
|
732
|
+
if (!explicitFiles) {
|
|
733
|
+
const moduleTags = normalizeTags(missionsModule.tags);
|
|
734
|
+
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return false;
|
|
735
|
+
}
|
|
736
|
+
|
|
633
737
|
if (typeof missionsModule.executeMission === 'function') {
|
|
634
738
|
const result = await missionsModule.executeMission();
|
|
635
739
|
allResults.push({
|
|
636
740
|
file: filePath,
|
|
637
741
|
result
|
|
638
742
|
});
|
|
743
|
+
return true;
|
|
639
744
|
}
|
|
640
745
|
} catch (err) {
|
|
641
746
|
console.error(`β Error running mission: ${filePath}`);
|
|
642
747
|
console.error(err);
|
|
643
748
|
}
|
|
749
|
+
return false;
|
|
644
750
|
};
|
|
645
751
|
|
|
646
752
|
if (args.length > 0) {
|
|
@@ -655,6 +761,11 @@ if (args.length > 0) {
|
|
|
655
761
|
}
|
|
656
762
|
}
|
|
657
763
|
|
|
764
|
+
if (!explicitFiles && requestedTags.length && allResults.length === 0) {
|
|
765
|
+
console.error(`β No missions matched ${tagMatch === 'all' ? 'all' : 'any'} of: ${requestedTags.join(', ')}`);
|
|
766
|
+
process.exit(1);
|
|
767
|
+
}
|
|
768
|
+
|
|
658
769
|
const endTime = new Date();
|
|
659
770
|
|
|
660
771
|
const flatMissions = allResults.flatMap(entry => {
|
|
@@ -696,6 +807,9 @@ const report = {
|
|
|
696
807
|
},
|
|
697
808
|
missions: flatMissions
|
|
698
809
|
};
|
|
810
|
+
report.tags = normalizeTags(flatMissions.flatMap(m =>
|
|
811
|
+
m.submissionType === 'mission' ? (m.tags ?? []) : []
|
|
812
|
+
));
|
|
699
813
|
|
|
700
814
|
const outputDir = './missions/mission_reports';
|
|
701
815
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
@@ -15,7 +15,9 @@ export const createWelcomeMission = async () => {
|
|
|
15
15
|
return;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
const content = `import { runMissions } from 'testronaut';
|
|
19
|
+
|
|
20
|
+
export const tags = [];
|
|
19
21
|
|
|
20
22
|
const welcomeGoal = \`
|
|
21
23
|
Welcome to Testronaut!
|
|
@@ -29,7 +31,8 @@ If you cannot reach this message, report FAILURE.
|
|
|
29
31
|
|
|
30
32
|
export async function executeMission() {
|
|
31
33
|
return await runMissions({
|
|
32
|
-
mission: welcomeGoal
|
|
34
|
+
mission: welcomeGoal,
|
|
35
|
+
tags
|
|
33
36
|
}, 'Welcome Mission!');
|
|
34
37
|
}
|
|
35
38
|
|
package/bin/init.js
CHANGED
|
@@ -32,10 +32,10 @@ import {
|
|
|
32
32
|
migrateLegacyConfig,
|
|
33
33
|
defaultConfig,
|
|
34
34
|
makeEnvTemplate,
|
|
35
|
-
openAIModels,
|
|
36
35
|
geminiModels,
|
|
37
36
|
pickInitialIndex
|
|
38
37
|
} from './initHelpers.js';
|
|
38
|
+
import { DEFAULT_OPENAI_MODEL, OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
39
39
|
|
|
40
40
|
export async function initializeTestronautProject() {
|
|
41
41
|
// βββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -81,36 +81,36 @@ export async function initializeTestronautProject() {
|
|
|
81
81
|
|
|
82
82
|
// 2b) Provider-specific model selection
|
|
83
83
|
if (llmProvider === 'openai') {
|
|
84
|
-
const
|
|
84
|
+
const currentModels = OPENAI_MODELS.filter(model => !model.legacy);
|
|
85
|
+
const legacyModels = OPENAI_MODELS.filter(model => model.legacy);
|
|
86
|
+
const choices = [
|
|
87
|
+
...currentModels.map(model => ({
|
|
88
|
+
title: `${model.label} (${model.description})`,
|
|
89
|
+
value: model.id,
|
|
90
|
+
})),
|
|
91
|
+
{ title: 'ββ Legacy models ββ', disabled: true },
|
|
92
|
+
...legacyModels.map(model => ({
|
|
93
|
+
title: `${model.label} (${model.description})`,
|
|
94
|
+
value: model.id,
|
|
95
|
+
})),
|
|
96
|
+
];
|
|
97
|
+
const selectedIndex = choices.findIndex(choice => choice.value === config.model);
|
|
98
|
+
const defaultIndex = choices.findIndex(choice => choice.value === DEFAULT_OPENAI_MODEL);
|
|
85
99
|
const { openaiModel } = await prompts({
|
|
86
100
|
type: 'select',
|
|
87
101
|
name: 'openaiModel',
|
|
88
102
|
message: 'Select an OpenAI model for agentic workflows (function/tool calling):',
|
|
89
|
-
choices
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
{ title: 'GPT-4o (multimodal, tool use)', value: 'gpt-4o' },
|
|
93
|
-
{ title: 'GPT-4o mini (speed/cost optimized)', value: 'gpt-4o-mini' },
|
|
94
|
-
{ title: 'o3 (reasoning w/ native tool use)', value: 'o3' },
|
|
95
|
-
{ title: 'o4-mini (reasoning, cost-effective)', value: 'o4-mini' },
|
|
96
|
-
// GPT-5 family (availability varies)
|
|
97
|
-
{ title: 'GPT-5 (previous reasoning model)', value: 'gpt-5' },
|
|
98
|
-
{ title: 'GPT-5 mini (faster)', value: 'gpt-5-mini' },
|
|
99
|
-
{ title: 'GPT-5 nano (lightweight)', value: 'gpt-5-nano' },
|
|
100
|
-
{ title: 'GPT-5.1 (latest reasoning model)', value: 'gpt-5.1' },
|
|
101
|
-
],
|
|
102
|
-
// Keep prior selection highlighted if present; fallback to 4.1-mini.
|
|
103
|
-
initial: pickInitialIndex(models, config.model, 'gpt-4.1-mini')
|
|
103
|
+
choices,
|
|
104
|
+
// Keep a prior selection highlighted; otherwise default to GPT-5.6.
|
|
105
|
+
initial: selectedIndex >= 0 ? selectedIndex : defaultIndex,
|
|
104
106
|
});
|
|
105
107
|
|
|
106
108
|
config.model = openaiModel;
|
|
107
109
|
|
|
108
|
-
//
|
|
110
|
+
// API access and rate limits vary by OpenAI account and project.
|
|
109
111
|
if (openaiModel?.startsWith('gpt-5')) {
|
|
110
112
|
console.log(`
|
|
111
|
-
|
|
112
|
-
- Access depends on your OpenAI account/region
|
|
113
|
-
- Some capabilities may be gated or rate-limited
|
|
113
|
+
βΉοΈ GPT-5 availability and rate limits depend on your OpenAI account and project.
|
|
114
114
|
`);
|
|
115
115
|
}
|
|
116
116
|
} else if (llmProvider === 'gemini') {
|
package/bin/initHelpers.js
CHANGED
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
* const models = openAIModels();
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
+
import { OPENAI_MODELS } from '../llm/openAI/models.js';
|
|
29
|
+
|
|
28
30
|
/**
|
|
29
31
|
* Ensures backward compatibility for older configs where
|
|
30
32
|
* provider information may have been implied by the model field.
|
|
@@ -52,6 +54,9 @@ export function defaultConfig(rootBasename) {
|
|
|
52
54
|
outputDir: 'missions/mission_reports',
|
|
53
55
|
projectName: rootBasename,
|
|
54
56
|
maxTurns: 20,
|
|
57
|
+
tags: [],
|
|
58
|
+
tagMatch: 'any',
|
|
59
|
+
addTags: [],
|
|
55
60
|
dom: {
|
|
56
61
|
listItemLimit: 3,
|
|
57
62
|
},
|
|
@@ -94,18 +99,7 @@ GEMINI_API_KEY=AIza...
|
|
|
94
99
|
* @returns {string[]} Array of OpenAI model names
|
|
95
100
|
*/
|
|
96
101
|
export function openAIModels() {
|
|
97
|
-
return
|
|
98
|
-
'gpt-4.1',
|
|
99
|
-
'gpt-4.1-mini',
|
|
100
|
-
'gpt-4o',
|
|
101
|
-
'gpt-4o-mini',
|
|
102
|
-
'o3',
|
|
103
|
-
'o4-mini',
|
|
104
|
-
'gpt-5',
|
|
105
|
-
'gpt-5-mini',
|
|
106
|
-
'gpt-5-nano',
|
|
107
|
-
'gpt-5.1',
|
|
108
|
-
];
|
|
102
|
+
return OPENAI_MODELS.map(({ id }) => id);
|
|
109
103
|
}
|
|
110
104
|
|
|
111
105
|
/**
|
package/core/tags.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export const TAG_PATTERN = /^[a-z0-9_-]+$/;
|
|
2
|
+
export const MAX_TAG_LENGTH = 40;
|
|
3
|
+
export const MAX_TAGS_PER_MISSION = 50;
|
|
4
|
+
export const UNTAGGED_FILTER = 'untagged';
|
|
5
|
+
|
|
6
|
+
export function normalizeTag(value) {
|
|
7
|
+
const tag = String(value ?? '').trim().toLowerCase();
|
|
8
|
+
if (!tag) throw new Error('Tags cannot be empty.');
|
|
9
|
+
if (tag.length > MAX_TAG_LENGTH) {
|
|
10
|
+
throw new Error(`Tag "${tag}" exceeds the ${MAX_TAG_LENGTH}-character limit.`);
|
|
11
|
+
}
|
|
12
|
+
if (!TAG_PATTERN.test(tag)) {
|
|
13
|
+
throw new Error(`Invalid tag "${tag}". Use only letters, numbers, hyphens, and underscores.`);
|
|
14
|
+
}
|
|
15
|
+
return tag;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function normalizeTags(values, { allowUntagged = false } = {}) {
|
|
19
|
+
const input = Array.isArray(values)
|
|
20
|
+
? values
|
|
21
|
+
: typeof values === 'string'
|
|
22
|
+
? (values.trim() ? values.split(',') : [])
|
|
23
|
+
: values == null
|
|
24
|
+
? []
|
|
25
|
+
: [values];
|
|
26
|
+
const tags = [...new Set(input.map(normalizeTag))].sort();
|
|
27
|
+
if (!allowUntagged && tags.includes(UNTAGGED_FILTER)) {
|
|
28
|
+
throw new Error(`"${UNTAGGED_FILTER}" is reserved for filtering and cannot be saved as a tag.`);
|
|
29
|
+
}
|
|
30
|
+
if (tags.length > MAX_TAGS_PER_MISSION) {
|
|
31
|
+
throw new Error(`A mission can have at most ${MAX_TAGS_PER_MISSION} tags.`);
|
|
32
|
+
}
|
|
33
|
+
return tags;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function matchesTagFilter(missionTags, requestedTags, match = 'any') {
|
|
37
|
+
const actual = normalizeTags(missionTags);
|
|
38
|
+
const requested = normalizeTags(requestedTags, { allowUntagged: true });
|
|
39
|
+
if (!requested.length) return true;
|
|
40
|
+
const has = (tag) => tag === UNTAGGED_FILTER ? actual.length === 0 : actual.includes(tag);
|
|
41
|
+
return match === 'all' ? requested.every(has) : requested.some(has);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function normalizeTagMatch(value) {
|
|
45
|
+
const match = String(value ?? 'any').trim().toLowerCase();
|
|
46
|
+
if (match !== 'any' && match !== 'all') {
|
|
47
|
+
throw new Error('tagMatch must be "any" or "all".');
|
|
48
|
+
}
|
|
49
|
+
return match;
|
|
50
|
+
}
|
package/core/turnLoop.js
CHANGED
|
@@ -41,7 +41,8 @@ import {
|
|
|
41
41
|
tokenUseCoolOff,
|
|
42
42
|
recordTokenUsage,
|
|
43
43
|
pruneOldTokenUsage,
|
|
44
|
-
updateLimitsFromHeaders
|
|
44
|
+
updateLimitsFromHeaders,
|
|
45
|
+
warnIfContextNearLimit
|
|
45
46
|
} from '../tools/tokenControl.js';
|
|
46
47
|
import { resolveProviderModel } from '../llm/modelResolver.js';
|
|
47
48
|
import { getLLM } from '../llm/llmFactory.js';
|
|
@@ -349,6 +350,7 @@ export const turnLoop = async (
|
|
|
349
350
|
// βββββββββββββββββββββββββββββββββββββββββββββ
|
|
350
351
|
// STEP 2: Request next reasoning turn from model
|
|
351
352
|
// βββββββββββββββββββββββββββββββββββββββββββββ
|
|
353
|
+
await warnIfContextNearLimit(MODEL_ID, { messages, tools: activeToolsSchema });
|
|
352
354
|
const { message, usage, headers } = await llm.chat({
|
|
353
355
|
model: MODEL_ID,
|
|
354
356
|
messages,
|
package/llm/modelResolver.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* 1) Env: TESTRONAUT_PROVIDER / TESTRONAUT_MODEL
|
|
10
10
|
* 2) Config file (provider + model)
|
|
11
11
|
* 3) Legacy config (model === "openai", no provider)
|
|
12
|
-
* 4) Default: { provider: "openai", model: "gpt-
|
|
12
|
+
* 4) Default: { provider: "openai", model: "gpt-5.6" }
|
|
13
13
|
*
|
|
14
14
|
* Related tests: tests/llmTests/modelResolver.test.js
|
|
15
15
|
* Used by: core/turnLoop.js, llm/llmFactory.js
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import path from 'path';
|
|
19
19
|
import fs from 'fs';
|
|
20
|
+
import { DEFAULT_OPENAI_MODEL } from './openAI/models.js';
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
23
|
* Resolve provider & model, honoring env overrides and safe defaults.
|
|
@@ -50,7 +51,7 @@ export function resolveProviderModel(opts = {}) {
|
|
|
50
51
|
|
|
51
52
|
// Legacy: model === "openai" and no provider
|
|
52
53
|
if (!cfg?.provider && cfg?.model === 'openai') {
|
|
53
|
-
return { provider: 'openai', model: envModel ||
|
|
54
|
+
return { provider: 'openai', model: envModel || DEFAULT_OPENAI_MODEL };
|
|
54
55
|
}
|
|
55
56
|
}
|
|
56
57
|
} catch (e) {
|
|
@@ -60,6 +61,6 @@ export function resolveProviderModel(opts = {}) {
|
|
|
60
61
|
// 3) Defaults
|
|
61
62
|
return {
|
|
62
63
|
provider: envProvider || 'openai',
|
|
63
|
-
model: envModel ||
|
|
64
|
+
model: envModel || DEFAULT_OPENAI_MODEL,
|
|
64
65
|
};
|
|
65
66
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Supported OpenAI model aliases and the capabilities Testronaut relies on.
|
|
3
|
+
* Dated snapshots and specialized Chat, Codex, Pro, and Cyber models are
|
|
4
|
+
* intentionally excluded from the interactive catalog.
|
|
5
|
+
*/
|
|
6
|
+
export const OPENAI_MODELS = [
|
|
7
|
+
{ id: 'gpt-5.6', label: 'GPT-5.6 Sol', description: 'flagship capability', contextWindow: 1050000, chatToolReasoningEffort: 'none' },
|
|
8
|
+
{ id: 'gpt-5.6-terra', label: 'GPT-5.6 Terra', description: 'balanced intelligence and cost', contextWindow: 1050000, chatToolReasoningEffort: 'none' },
|
|
9
|
+
{ id: 'gpt-5.6-luna', label: 'GPT-5.6 Luna', description: 'cost-sensitive, high-volume workloads', contextWindow: 1050000, chatToolReasoningEffort: 'none' },
|
|
10
|
+
{ id: 'gpt-5.5', label: 'GPT-5.5', description: 'previous flagship model', contextWindow: 1050000 },
|
|
11
|
+
|
|
12
|
+
{ id: 'gpt-5.4', label: 'GPT-5.4', description: 'earlier professional model', contextWindow: 1050000, legacy: true },
|
|
13
|
+
{ id: 'gpt-5.4-mini', label: 'GPT-5.4 mini', description: 'earlier balanced model', contextWindow: 400000, legacy: true },
|
|
14
|
+
{ id: 'gpt-5.4-nano', label: 'GPT-5.4 nano', description: 'earlier high-volume model', contextWindow: 400000, legacy: true },
|
|
15
|
+
{ id: 'gpt-5.2', label: 'GPT-5.2', description: 'previous frontier model', contextWindow: 400000, legacy: true },
|
|
16
|
+
{ id: 'gpt-5.1', label: 'GPT-5.1', description: 'legacy reasoning model', legacy: true },
|
|
17
|
+
{ id: 'gpt-5', label: 'GPT-5', description: 'legacy reasoning model', legacy: true },
|
|
18
|
+
{ id: 'gpt-5-mini', label: 'GPT-5 mini', description: 'legacy faster model', legacy: true },
|
|
19
|
+
{ id: 'gpt-5-nano', label: 'GPT-5 nano', description: 'legacy lightweight model', legacy: true },
|
|
20
|
+
{ id: 'gpt-4.1', label: 'GPT-4.1', description: 'legacy general-purpose model', legacy: true },
|
|
21
|
+
{ id: 'gpt-4.1-mini', label: 'GPT-4.1 mini', description: 'legacy faster model', legacy: true },
|
|
22
|
+
{ id: 'gpt-4o', label: 'GPT-4o', description: 'legacy multimodal model', legacy: true },
|
|
23
|
+
{ id: 'gpt-4o-mini', label: 'GPT-4o mini', description: 'legacy cost-optimized model', legacy: true },
|
|
24
|
+
{ id: 'o3', label: 'o3', description: 'legacy reasoning model', legacy: true },
|
|
25
|
+
{ id: 'o4-mini', label: 'o4-mini', description: 'legacy cost-effective reasoning model', legacy: true },
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
export const DEFAULT_OPENAI_MODEL = 'gpt-5.6';
|
|
29
|
+
|
|
30
|
+
export function getOpenAIModel(modelId) {
|
|
31
|
+
return OPENAI_MODELS.find(({ id }) => id === modelId);
|
|
32
|
+
}
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import OpenAI from 'openai';
|
|
23
|
+
import { getOpenAIModel } from './models.js';
|
|
23
24
|
|
|
24
25
|
export class OpenAIProvider {
|
|
25
26
|
constructor({ apiKey } = {}) {
|
|
@@ -33,11 +34,21 @@ export class OpenAIProvider {
|
|
|
33
34
|
* @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}, headers?:any}>}
|
|
34
35
|
*/
|
|
35
36
|
async chat({ model, messages, tools }) {
|
|
36
|
-
const
|
|
37
|
+
const request = {
|
|
37
38
|
model,
|
|
38
39
|
messages,
|
|
39
40
|
tools,
|
|
40
|
-
}
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// GPT-5.6 defaults to medium reasoning, but Chat Completions does not
|
|
44
|
+
// support combining reasoning with function tools. Responses API support
|
|
45
|
+
// is tracked separately; keep the current tool loop compatible meanwhile.
|
|
46
|
+
const chatToolReasoningEffort = getOpenAIModel(model)?.chatToolReasoningEffort;
|
|
47
|
+
if (tools?.length && chatToolReasoningEffort) {
|
|
48
|
+
request.reasoning_effort = chatToolReasoningEffort;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const res = await this.client.chat.completions.create(request);
|
|
41
52
|
|
|
42
53
|
// OpenAI already returns an OpenAI-like message and usage structure.
|
|
43
54
|
const message = res.choices?.[0]?.message ?? { role: 'assistant', content: '' };
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -28,6 +28,7 @@ import path from 'path';
|
|
|
28
28
|
import { runAgent } from '../core/agent.js';
|
|
29
29
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
30
|
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
|
+
import { normalizeTags } from '../core/tags.js';
|
|
31
32
|
|
|
32
33
|
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
33
34
|
const isDebugEnabled = () => {
|
|
@@ -44,7 +45,7 @@ const formatListLimit = (v) => v === Infinity ? 'all' : v;
|
|
|
44
45
|
* @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
|
|
45
46
|
* @param {string} missionName
|
|
46
47
|
*/
|
|
47
|
-
export async function runMissions({ preMission, mission, postMission }, missionName) {
|
|
48
|
+
export async function runMissions({ preMission, mission, postMission, tags = [] }, missionName) {
|
|
48
49
|
// 1) Config and turn budget (with guardrails)
|
|
49
50
|
const cfg = await loadConfig();
|
|
50
51
|
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
@@ -55,6 +56,11 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
55
56
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
56
57
|
const humanInput = getHumanInputConfig(cfg);
|
|
57
58
|
const debugEnabled = isDebugEnabled();
|
|
59
|
+
const missionTags = normalizeTags([
|
|
60
|
+
...normalizeTags(tags),
|
|
61
|
+
...normalizeTags(cfg?.addTags),
|
|
62
|
+
...normalizeTags(process.env.TESTRONAUT_ADD_TAGS),
|
|
63
|
+
]);
|
|
58
64
|
if (notes.length) {
|
|
59
65
|
console.warn(notes.join('\n'));
|
|
60
66
|
}
|
|
@@ -208,6 +214,10 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
208
214
|
return;
|
|
209
215
|
}
|
|
210
216
|
|
|
217
|
+
for (const result of success) {
|
|
218
|
+
result.tags = result.submissionType === 'mission' ? missionTags : [];
|
|
219
|
+
}
|
|
220
|
+
|
|
211
221
|
const missionStatus = success[0].steps[success[0].steps.length - 1].result;
|
|
212
222
|
if (missionStatus.toLowerCase().includes('failure')) {
|
|
213
223
|
success[0].status = 'failed';
|
|
@@ -143,6 +143,49 @@ describe('cli helpers', () => {
|
|
|
143
143
|
expect(res.invalid).toBe(true);
|
|
144
144
|
});
|
|
145
145
|
|
|
146
|
+
it('parses tag flags and aliases without leaving mission-like arguments', () => {
|
|
147
|
+
const { parseTagArgs } = __test__;
|
|
148
|
+
const res = parseTagArgs([
|
|
149
|
+
'--tags=smoke,authentication',
|
|
150
|
+
'--tag-match', 'all',
|
|
151
|
+
'--add_tags', 'staging',
|
|
152
|
+
'login.mission.js',
|
|
153
|
+
]);
|
|
154
|
+
expect(res.tags.value).toBe('smoke,authentication');
|
|
155
|
+
expect(res.tagMatch.value).toBe('all');
|
|
156
|
+
expect(res.addTags.value).toBe('staging');
|
|
157
|
+
expect(res.args).toEqual(['login.mission.js']);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it('combines repeated singular and plural tag flags', () => {
|
|
161
|
+
const { parseTagArgs } = __test__;
|
|
162
|
+
const res = parseTagArgs([
|
|
163
|
+
'--tag', 'authentication',
|
|
164
|
+
'--tag=smoke',
|
|
165
|
+
'--tags', 'checkout, navigation',
|
|
166
|
+
'--add-tag', 'staging',
|
|
167
|
+
'--add-tags=nightly,full-test-run',
|
|
168
|
+
]);
|
|
169
|
+
expect(res.tags.value).toBe('authentication,smoke,checkout, navigation');
|
|
170
|
+
expect(res.addTags.value).toBe('staging,nightly,full-test-run');
|
|
171
|
+
expect(res.hasUnquotedCommaSpace).toBe(false);
|
|
172
|
+
expect(res.args).toEqual([]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('detects the common unquoted comma-space form for a friendly error', () => {
|
|
176
|
+
const { parseTagArgs } = __test__;
|
|
177
|
+
const res = parseTagArgs(['--tags', 'authentication,', 'smoke']);
|
|
178
|
+
expect(res.hasUnquotedCommaSpace).toBe(true);
|
|
179
|
+
expect(res.args).toEqual(['smoke']);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('records missing values even when another repeated flag is valid', () => {
|
|
183
|
+
const { parseTagArgs } = __test__;
|
|
184
|
+
const res = parseTagArgs(['--tag', '--tag', 'smoke']);
|
|
185
|
+
expect(res.tags.missingValue).toBe(true);
|
|
186
|
+
expect(res.tags.value).toBe('smoke');
|
|
187
|
+
});
|
|
188
|
+
|
|
146
189
|
describe('detectCliName', () => {
|
|
147
190
|
const { detectCliName } = __test__;
|
|
148
191
|
|
|
@@ -4,6 +4,17 @@ import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../.
|
|
|
4
4
|
describe('model helpers', () => {
|
|
5
5
|
it('lists known models', () => {
|
|
6
6
|
expect(openAIModels()).toContain('gpt-4o');
|
|
7
|
+
expect(openAIModels()).toEqual(expect.arrayContaining([
|
|
8
|
+
'gpt-5.2',
|
|
9
|
+
'gpt-5.4',
|
|
10
|
+
'gpt-5.4-mini',
|
|
11
|
+
'gpt-5.4-nano',
|
|
12
|
+
'gpt-5.5',
|
|
13
|
+
'gpt-5.6',
|
|
14
|
+
'gpt-5.6-terra',
|
|
15
|
+
'gpt-5.6-luna',
|
|
16
|
+
]));
|
|
17
|
+
expect(openAIModels()).not.toContain('gpt-5.3');
|
|
7
18
|
expect(geminiModels()).toContain('gemini-2.5-flash');
|
|
8
19
|
});
|
|
9
20
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../../core/tags.js';
|
|
3
|
+
|
|
4
|
+
describe('tag helpers', () => {
|
|
5
|
+
it('normalizes case, order, and duplicates', () => {
|
|
6
|
+
expect(normalizeTags(['Smoke', 'authentication', 'smoke'])).toEqual(['authentication', 'smoke']);
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it('treats omitted and blank optional tag sources as no tags', () => {
|
|
10
|
+
expect(normalizeTags()).toEqual([]);
|
|
11
|
+
expect(normalizeTags(null)).toEqual([]);
|
|
12
|
+
expect(normalizeTags('')).toEqual([]);
|
|
13
|
+
expect(normalizeTags(' ')).toEqual([]);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('rejects spaces and the reserved untagged value', () => {
|
|
17
|
+
expect(() => normalizeTags(['human in loop'])).toThrow(/Invalid tag/);
|
|
18
|
+
expect(() => normalizeTags(['untagged'])).toThrow(/reserved/);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('supports any, all, and untagged matching', () => {
|
|
22
|
+
expect(matchesTagFilter(['smoke'], ['smoke', 'checkout'], 'any')).toBe(true);
|
|
23
|
+
expect(matchesTagFilter(['smoke'], ['smoke', 'checkout'], 'all')).toBe(false);
|
|
24
|
+
expect(matchesTagFilter([], ['untagged'], 'any')).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('defaults tag matching to any', () => {
|
|
28
|
+
expect(normalizeTagMatch()).toBe('any');
|
|
29
|
+
expect(() => normalizeTagMatch('some')).toThrow(/any.*all/);
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -67,6 +67,7 @@ vi.mock('../../tools/tokenControl.js', () => ({
|
|
|
67
67
|
totalTokensUsed: turnTimestamps.reduce((a, [, t]) => a + t, 0),
|
|
68
68
|
})),
|
|
69
69
|
updateLimitsFromHeaders: vi.fn(() => {}),
|
|
70
|
+
warnIfContextNearLimit: vi.fn(async () => ({ warned: false })),
|
|
70
71
|
}));
|
|
71
72
|
|
|
72
73
|
vi.mock('../../tools/toolSchema.js', () => ({
|
|
@@ -62,13 +62,13 @@ describe('resolveProviderModel', () => {
|
|
|
62
62
|
writeConfig(temp, { model: 'openai' });
|
|
63
63
|
|
|
64
64
|
const res = resolveProviderModel({ cwd: temp });
|
|
65
|
-
expect(res).toEqual({ provider: 'openai', model: 'gpt-
|
|
65
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-5.6' });
|
|
66
66
|
});
|
|
67
67
|
|
|
68
68
|
it('returns defaults when config missing', () => {
|
|
69
69
|
const temp = makeTempProject();
|
|
70
70
|
const res = resolveProviderModel({ cwd: temp });
|
|
71
|
-
expect(res).toEqual({ provider: 'openai', model: 'gpt-
|
|
71
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-5.6' });
|
|
72
72
|
});
|
|
73
73
|
|
|
74
74
|
it('returns defaults when JSON parsing fails', () => {
|
|
@@ -77,7 +77,7 @@ describe('resolveProviderModel', () => {
|
|
|
77
77
|
|
|
78
78
|
const res = resolveProviderModel({ cwd: temp });
|
|
79
79
|
expect(res.provider).toBe('openai');
|
|
80
|
-
expect(res.model).toBe('gpt-
|
|
80
|
+
expect(res.model).toBe('gpt-5.6');
|
|
81
81
|
});
|
|
82
82
|
|
|
83
83
|
it('trims env vars and treats empty strings as absent', () => {
|
|
@@ -90,7 +90,7 @@ describe('resolveProviderModel', () => {
|
|
|
90
90
|
process.env.TESTRONAUT_MODEL = ' ';
|
|
91
91
|
res = resolveProviderModel();
|
|
92
92
|
// falls back to defaults when env vars are just whitespace
|
|
93
|
-
expect(res).toEqual({ provider: 'openai', model: 'gpt-
|
|
93
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-5.6' });
|
|
94
94
|
});
|
|
95
95
|
|
|
96
96
|
it('prefers config when only provider is in env and model only in config', () => {
|
|
@@ -78,6 +78,24 @@ describe('OpenAIProvider', () => {
|
|
|
78
78
|
});
|
|
79
79
|
});
|
|
80
80
|
|
|
81
|
+
it.each(['gpt-5.6', 'gpt-5.6-terra', 'gpt-5.6-luna'])(
|
|
82
|
+
'disables reasoning for %s function tools on Chat Completions',
|
|
83
|
+
async (model) => {
|
|
84
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
85
|
+
const tools = [{ type: 'function', function: { name: 'doThing', parameters: { type: 'object' } } }];
|
|
86
|
+
|
|
87
|
+
await prov.chat({ model, messages: [{ role: 'user', content: 'go' }], tools });
|
|
88
|
+
|
|
89
|
+
expect(shared.lastCreateArgs).toMatchObject({ model, tools, reasoning_effort: 'none' });
|
|
90
|
+
}
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
it('does not add a reasoning override to GPT-5.6 requests without tools', async () => {
|
|
94
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
95
|
+
await prov.chat({ model: 'gpt-5.6', messages: [] });
|
|
96
|
+
expect(shared.lastCreateArgs).not.toHaveProperty('reasoning_effort');
|
|
97
|
+
});
|
|
98
|
+
|
|
81
99
|
it('returns normalized message, usage, and headers', async () => {
|
|
82
100
|
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
83
101
|
const { message, usage, headers } = await prov.chat({
|
|
@@ -137,6 +137,41 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
137
137
|
log.mockRestore();
|
|
138
138
|
});
|
|
139
139
|
|
|
140
|
+
it('adds normalized tags only to main mission submissions', async () => {
|
|
141
|
+
loadConfig.mockResolvedValue({ addTags: ['staging', 'Smoke'] });
|
|
142
|
+
enforceTurnBudget.mockReturnValue({ effectiveMax: 20, limits: {}, notes: [], strict: false });
|
|
143
|
+
process.env.TESTRONAUT_ADD_TAGS = 'full-test-run,smoke';
|
|
144
|
+
runAgent.mockResolvedValue([
|
|
145
|
+
{ submissionType: 'premission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
146
|
+
{ submissionType: 'mission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
147
|
+
{ submissionType: 'postmission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
148
|
+
]);
|
|
149
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
150
|
+
|
|
151
|
+
const result = await runMissions({ preMission: 'setup', mission: 'test', postMission: 'cleanup', tags: ['authentication'] }, 'Tagged');
|
|
152
|
+
|
|
153
|
+
expect(result[0].tags).toEqual([]);
|
|
154
|
+
expect(result[1].tags).toEqual(['authentication', 'full-test-run', 'smoke', 'staging']);
|
|
155
|
+
expect(result[2].tags).toEqual([]);
|
|
156
|
+
log.mockRestore();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('runs an untagged mission when optional additive tags are blank', async () => {
|
|
160
|
+
loadConfig.mockResolvedValue({ addTags: [] });
|
|
161
|
+
enforceTurnBudget.mockReturnValue({ effectiveMax: 20, limits: {}, notes: [], strict: false });
|
|
162
|
+
process.env.TESTRONAUT_ADD_TAGS = '';
|
|
163
|
+
runAgent.mockResolvedValue([
|
|
164
|
+
{ submissionType: 'mission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
165
|
+
]);
|
|
166
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
167
|
+
|
|
168
|
+
const result = await runMissions({ mission: 'untagged mission' }, 'Untagged');
|
|
169
|
+
|
|
170
|
+
expect(result[0].tags).toEqual([]);
|
|
171
|
+
expect(runAgent).toHaveBeenCalledOnce();
|
|
172
|
+
log.mockRestore();
|
|
173
|
+
});
|
|
174
|
+
|
|
140
175
|
it('normalizes single strings and functions into arrays and names missions', async () => {
|
|
141
176
|
loadConfig.mockResolvedValue({});
|
|
142
177
|
enforceTurnBudget.mockReturnValue({
|
|
@@ -27,6 +27,7 @@ describe('generateHtmlReport', () => {
|
|
|
27
27
|
submissionType: 'mission',
|
|
28
28
|
submissionName: 'Main',
|
|
29
29
|
status: 'passed',
|
|
30
|
+
tags: ['smoke'],
|
|
30
31
|
steps: [
|
|
31
32
|
{
|
|
32
33
|
turn: 0,
|
|
@@ -54,6 +55,13 @@ describe('generateHtmlReport', () => {
|
|
|
54
55
|
expect(html).toContain('Mission A');
|
|
55
56
|
expect(html).toContain('Turn 1 (re-attempt 1/5)');
|
|
56
57
|
expect(html).toContain('β οΈ Turn Issues');
|
|
58
|
+
expect(html).toContain('data-tag="smoke"');
|
|
59
|
+
expect(html).toContain('data-tags="smoke"');
|
|
60
|
+
expect(html).toContain('data-tag="untagged"');
|
|
61
|
+
expect(html).toContain('id="match-count"');
|
|
62
|
+
expect(html).toContain('No missions match these tags');
|
|
63
|
+
expect(html).toContain('Keep missions that do not match visible');
|
|
64
|
+
expect(html).not.toContain('<div class="pill">LLM:');
|
|
57
65
|
});
|
|
58
66
|
|
|
59
67
|
it('escapes unsafe text in output', () => {
|
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
tokenUseCoolOff,
|
|
34
34
|
recordTokenUsage,
|
|
35
35
|
pruneOldTokenUsage,
|
|
36
|
+
warnIfContextNearLimit,
|
|
36
37
|
__resetTokenControlForTests,
|
|
37
38
|
} from '../../tools/tokenControl.js';
|
|
38
39
|
|
|
@@ -111,6 +112,24 @@ describe('tokenControl', () => {
|
|
|
111
112
|
// sanity check it actually changed (unless defaults already 1234)
|
|
112
113
|
if (before.tpm !== 1234) expect(after.tpm).not.toBe(before.tpm);
|
|
113
114
|
});
|
|
115
|
+
|
|
116
|
+
it('uses tier-1 defaults without letting generic GPT-5 rules shadow variants', () => {
|
|
117
|
+
expect(getCurrentTokenLimit('gpt-5.6').tpm).toBe(500000);
|
|
118
|
+
expect(getCurrentTokenLimit('gpt-5.6-luna').tpm).toBe(500000);
|
|
119
|
+
expect(getCurrentTokenLimit('gpt-5.4-nano').tpm).toBe(200000);
|
|
120
|
+
expect(getCurrentTokenLimit('gpt-5-mini').tpm).toBe(240000);
|
|
121
|
+
expect(getCurrentTokenLimit('gpt-5-nano').tpm).toBe(600000);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it('warns without truncating when known context usage crosses the threshold', async () => {
|
|
126
|
+
tiktokenMocks.encoding_for_model_impl.mockReturnValue(encMock);
|
|
127
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
128
|
+
const result = await warnIfContextNearLimit('gpt-5.2', '1234567890', 0.00002);
|
|
129
|
+
expect(result.warned).toBe(true);
|
|
130
|
+
expect(result.contextWindow).toBe(400000);
|
|
131
|
+
expect(warn).toHaveBeenCalled();
|
|
132
|
+
warn.mockRestore();
|
|
114
133
|
});
|
|
115
134
|
|
|
116
135
|
describe('rolling window + cooldown', () => {
|
package/todo.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Testronaut CLI TODO
|
|
2
|
+
|
|
3
|
+
## OpenAI provider follow-ups
|
|
4
|
+
|
|
5
|
+
- Migrate the OpenAI adapter from Chat Completions to the Responses API in a
|
|
6
|
+
dedicated change. Preserve Testronaut's provider-neutral message and tool-call
|
|
7
|
+
contract, add response-ID/reasoning-state handling, normalize usage and raw
|
|
8
|
+
headers, and cover multi-turn tool continuation with unit and API smoke tests.
|
|
9
|
+
Once migrated, remove the GPT-5.6 Chat Completions compatibility override that
|
|
10
|
+
sets `reasoning_effort` to `none` for function-tool requests.
|
|
11
|
+
- Add active context budgeting after its behavior is designed and tested. Use
|
|
12
|
+
model context/output metadata to reserve response and reasoning capacity, then
|
|
13
|
+
apply explicit history compaction or summarization rules. Do not silently
|
|
14
|
+
truncate mission instructions, current DOM state, or unmatched tool calls.
|
|
15
|
+
- Learn effective OpenAI TPM/RPM limits from successful response headers and
|
|
16
|
+
rate-limit errors. Prefer observed project/model limits over inferring a named
|
|
17
|
+
account tier; retain the environment override and conservative startup values.
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import fs from 'fs';
|
|
19
19
|
import path from 'path';
|
|
20
|
+
import { normalizeTags } from '../core/tags.js';
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
23
|
* Render and write a Testronaut run report to disk.
|
|
@@ -27,6 +28,7 @@ import path from 'path';
|
|
|
27
28
|
*/
|
|
28
29
|
export function generateHtmlReport(report, outputPath) {
|
|
29
30
|
const { runId, startTime, endTime, missions = [], summary = {}, llm = {} } = report;
|
|
31
|
+
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
30
32
|
const durationSec =
|
|
31
33
|
(startTime && endTime)
|
|
32
34
|
? ((new Date(endTime) - new Date(startTime)) / 1000).toFixed(2)
|
|
@@ -40,6 +42,16 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
40
42
|
const badge = (status) =>
|
|
41
43
|
status === 'passed' ? 'β
Passed' :
|
|
42
44
|
status === 'failed' ? 'β Failed' : (status || 'β');
|
|
45
|
+
const tagPalette = [
|
|
46
|
+
['#60a5fa', 'rgba(96,165,250,.18)'], ['#a78bfa', 'rgba(167,139,250,.18)'],
|
|
47
|
+
['#34d399', 'rgba(52,211,153,.18)'], ['#f472b6', 'rgba(244,114,182,.18)'],
|
|
48
|
+
['#fbbf24', 'rgba(251,191,36,.18)'], ['#22d3ee', 'rgba(34,211,238,.18)'],
|
|
49
|
+
];
|
|
50
|
+
const tagStyle = (tag) => {
|
|
51
|
+
const index = [...tag].reduce((sum, char) => sum + char.charCodeAt(0), 0) % tagPalette.length;
|
|
52
|
+
const [color, background] = tagPalette[index];
|
|
53
|
+
return `color:${color};background:${background};border-color:${color}66`;
|
|
54
|
+
};
|
|
43
55
|
|
|
44
56
|
const submissionBlock = (m) => {
|
|
45
57
|
const mDurationSec =
|
|
@@ -128,6 +140,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
128
140
|
const firstStart = Math.min(...subs.map(s => s.startTime || 0).filter(Boolean));
|
|
129
141
|
const lastEnd = Math.max(...subs.map(s => s.endTime || 0).filter(Boolean));
|
|
130
142
|
const groupDur = (firstStart && lastEnd) ? ((lastEnd - firstStart) / 1000).toFixed(2) : 'β';
|
|
143
|
+
const tags = normalizeTags(subs.flatMap(s => s.submissionType === 'mission' ? (s.tags ?? []) : []));
|
|
131
144
|
|
|
132
145
|
// pre β mission β post
|
|
133
146
|
const order = { premission: 0, mission: 1, postmission: 2 };
|
|
@@ -137,9 +150,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
137
150
|
);
|
|
138
151
|
|
|
139
152
|
return `
|
|
140
|
-
<details class="mission-group">
|
|
153
|
+
<details class="mission-group" data-tags="${esc(tags.join(','))}">
|
|
141
154
|
<summary>
|
|
142
155
|
<span class="name">${esc(missionName)}</span>
|
|
156
|
+
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
143
157
|
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
144
158
|
<span class="meta">submissions: ${subs.length} β’ steps: ${totalSteps} β’ duration: ${groupDur}s</span>
|
|
145
159
|
<span class="toolbar">
|
|
@@ -233,6 +247,21 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
233
247
|
}
|
|
234
248
|
.pill.ok{ background: var(--chip-ok-bg); color: var(--ok); border-color: var(--chip-ok-border); font-weight:700; }
|
|
235
249
|
.pill.bad{ background: var(--chip-bad-bg); color: var(--bad); border-color: var(--chip-bad-border); font-weight:700; }
|
|
250
|
+
.tag-filter{max-width:928px;margin:0 auto 20px;padding:18px 16px;background:rgba(2,6,23,.30);}
|
|
251
|
+
.tag-filter-heading{display:flex;align-items:center;gap:7px;margin-bottom:12px;flex-wrap:wrap;}
|
|
252
|
+
.tag-filter-title{font-size:14px;color:var(--text);font-weight:800;}
|
|
253
|
+
.info{display:inline-grid;place-items:center;width:17px;height:17px;border:1px solid var(--chip-border);border-radius:50%;font-size:11px;color:var(--text-muted);cursor:help;}
|
|
254
|
+
.match-count{margin-left:auto;color:var(--text-muted);font-size:12px;}
|
|
255
|
+
.tag-buttons,.mission-tags{display:flex;gap:6px;flex-wrap:wrap;align-items:center;}
|
|
256
|
+
.tag-button,.tag-small{border:1px solid var(--chip-border);background:rgba(255,255,255,.06);color:var(--text);border-radius:999px;padding:5px 10px;font-size:11px;font-weight:700;}
|
|
257
|
+
.tag-button{cursor:pointer;transition:transform .15s ease,background .15s ease}.tag-button:hover{transform:translateY(-1px);background:rgba(255,255,255,.11)}.tag-button.active{box-shadow:0 0 0 2px rgba(96,165,250,.38);}
|
|
258
|
+
.filter-controls{display:flex;gap:16px;align-items:center;margin-top:14px;font-size:12px;color:var(--text-muted);flex-wrap:wrap;}
|
|
259
|
+
.filter-controls select{color:var(--text);background:#111a31;border:1px solid var(--chip-border);border-radius:8px;padding:6px 24px 6px 8px;}
|
|
260
|
+
.show-control{cursor:help;display:flex;align-items:center;gap:5px;}
|
|
261
|
+
.filter-empty{max-width:928px;margin:12px auto;padding:28px 16px;text-align:center;border:1px dashed var(--hairline-strong);border-radius:16px;color:var(--text-muted);}
|
|
262
|
+
.filter-empty strong{display:block;color:var(--text);font-size:18px;margin-bottom:6px;}
|
|
263
|
+
.mission-group.nonmatching{opacity:.45;}
|
|
264
|
+
@media(max-width:640px){.match-count{width:100%;margin-left:0}.tag-filter{padding:14px 12px}.meta{display:none}}
|
|
236
265
|
|
|
237
266
|
/* ===== Disclosure blocks ===== */
|
|
238
267
|
details{ background: rgba(255,255,255,.05); border:1px solid var(--hairline); border-radius:16px; margin:10px 0; overflow:hidden; }
|
|
@@ -328,14 +357,76 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
328
357
|
<div class="pill">Missions: ${esc(summary.totalMissions ?? totals.total)}</div>
|
|
329
358
|
<div class="pill ok">Passed: ${esc(summary.passed ?? totals.passed)}</div>
|
|
330
359
|
<div class="pill bad">Failed: ${esc(summary.failed ?? totals.failed)}</div>
|
|
331
|
-
|
|
360
|
+
</div>
|
|
361
|
+
|
|
362
|
+
<div class="tag-filter glass">
|
|
363
|
+
<div class="tag-filter-heading">
|
|
364
|
+
<span class="tag-filter-title">Filter missions by tag</span>
|
|
365
|
+
<span class="info" title="Select one or more tags to focus the report. Match Any uses OR; Match All requires every selected tag.">i</span>
|
|
366
|
+
<span class="match-count" id="match-count">${totals.total} of ${totals.total} missions match</span>
|
|
367
|
+
</div>
|
|
368
|
+
<div class="tag-buttons">
|
|
369
|
+
${reportTags.map(tag => `<button type="button" class="tag-button" data-tag="${esc(tag)}" style="${tagStyle(tag)}">${esc(tag)}</button>`).join('')}
|
|
370
|
+
<button type="button" class="tag-button" data-tag="untagged">untagged</button>
|
|
371
|
+
</div>
|
|
372
|
+
<div class="filter-controls">
|
|
373
|
+
<label>Match <select id="tag-match"><option value="any">any</option><option value="all">all</option></select></label>
|
|
374
|
+
<label class="show-control" title="Keep missions that do not match visible in a muted style instead of hiding them."><input id="show-nonmatching" type="checkbox"> Show nonmatching missions <span class="info">i</span></label>
|
|
375
|
+
<button type="button" id="clear-tags" class="tag-button" hidden>Clear filters</button>
|
|
376
|
+
</div>
|
|
332
377
|
</div>
|
|
333
378
|
|
|
334
379
|
<div class="container">
|
|
335
380
|
${groupsHtml || '<div class="glass empty">No missions recorded.</div>'}
|
|
336
381
|
</div>
|
|
382
|
+
<div class="filter-empty" id="filter-empty" hidden>
|
|
383
|
+
<strong>No missions match these tags</strong>
|
|
384
|
+
Try removing a tag, switching to βAny,β or showing nonmatching missions.
|
|
385
|
+
</div>
|
|
337
386
|
<script>
|
|
338
387
|
(function () {
|
|
388
|
+
var selectedTags = [];
|
|
389
|
+
function applyTagFilter() {
|
|
390
|
+
var match = document.getElementById('tag-match');
|
|
391
|
+
var show = document.getElementById('show-nonmatching');
|
|
392
|
+
var matchedCount = 0;
|
|
393
|
+
var totalCount = 0;
|
|
394
|
+
document.querySelectorAll('.mission-group').forEach(function (group) {
|
|
395
|
+
totalCount += 1;
|
|
396
|
+
var tags = (group.getAttribute('data-tags') || '').split(',').filter(Boolean);
|
|
397
|
+
var test = function (tag) { return tag === 'untagged' ? tags.length === 0 : tags.indexOf(tag) >= 0; };
|
|
398
|
+
var matches = !selectedTags.length || ((match && match.value === 'all') ? selectedTags.every(test) : selectedTags.some(test));
|
|
399
|
+
if (matches) matchedCount += 1;
|
|
400
|
+
group.hidden = !matches && !(show && show.checked);
|
|
401
|
+
group.classList.toggle('nonmatching', !matches);
|
|
402
|
+
});
|
|
403
|
+
var count = document.getElementById('match-count');
|
|
404
|
+
var empty = document.getElementById('filter-empty');
|
|
405
|
+
var clear = document.getElementById('clear-tags');
|
|
406
|
+
if (count) count.textContent = matchedCount + ' of ' + totalCount + ' missions match';
|
|
407
|
+
if (empty) empty.hidden = matchedCount !== 0 || !selectedTags.length || !!(show && show.checked);
|
|
408
|
+
if (clear) clear.hidden = selectedTags.length === 0;
|
|
409
|
+
}
|
|
410
|
+
document.querySelectorAll('.tag-button[data-tag]').forEach(function (button) {
|
|
411
|
+
button.addEventListener('click', function () {
|
|
412
|
+
var tag = button.getAttribute('data-tag');
|
|
413
|
+
var idx = selectedTags.indexOf(tag);
|
|
414
|
+
if (idx >= 0) selectedTags.splice(idx, 1); else selectedTags.push(tag);
|
|
415
|
+
button.classList.toggle('active', idx < 0);
|
|
416
|
+
applyTagFilter();
|
|
417
|
+
});
|
|
418
|
+
});
|
|
419
|
+
var tagMatch = document.getElementById('tag-match');
|
|
420
|
+
var showNonmatching = document.getElementById('show-nonmatching');
|
|
421
|
+
var clearTags = document.getElementById('clear-tags');
|
|
422
|
+
if (tagMatch) tagMatch.addEventListener('change', applyTagFilter);
|
|
423
|
+
if (showNonmatching) showNonmatching.addEventListener('change', applyTagFilter);
|
|
424
|
+
if (clearTags) clearTags.addEventListener('click', function () {
|
|
425
|
+
selectedTags = [];
|
|
426
|
+
document.querySelectorAll('.tag-button[data-tag]').forEach(function (button) { button.classList.remove('active'); });
|
|
427
|
+
applyTagFilter();
|
|
428
|
+
});
|
|
429
|
+
applyTagFilter();
|
|
339
430
|
function setOpenAll(root, selector, open) {
|
|
340
431
|
root.querySelectorAll(selector).forEach(function (el) {
|
|
341
432
|
if (el && 'open' in el) el.open = open;
|
package/tools/tokenControl.js
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
|
|
23
23
|
import { encoding_for_model, get_encoding } from '@dqbd/tiktoken';
|
|
24
24
|
import { wait } from './turnLoopUtils.js';
|
|
25
|
+
import { getOpenAIModel } from '../llm/openAI/models.js';
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* Dynamic token-per-minute limits by model family.
|
|
@@ -35,10 +36,16 @@ import { wait } from './turnLoopUtils.js';
|
|
|
35
36
|
*/
|
|
36
37
|
const DEFAULT_LIMITS = [
|
|
37
38
|
// ββ OpenAI (newer first) ββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
39
|
+
{ test: /^gpt-5\.6-luna(-|$)/i, tpm: 500000 },
|
|
40
|
+
{ test: /^gpt-5\.6(-|$)/i, tpm: 500000 },
|
|
41
|
+
{ test: /^gpt-5\.5(-|$)/i, tpm: 500000 },
|
|
42
|
+
{ test: /^gpt-5\.4-nano(-|$)/i, tpm: 200000 },
|
|
43
|
+
{ test: /^gpt-5\.4(-|$)/i, tpm: 500000 },
|
|
44
|
+
{ test: /^gpt-5\.2(-|$)/i, tpm: 500000 },
|
|
38
45
|
{ test: /^gpt-5.1(-|$)/i, tpm: 120000 },
|
|
39
|
-
{ test: /^gpt-5(-|$)/i, tpm: 90000 },
|
|
40
46
|
{ test: /^gpt-5-mini(-|$)/i, tpm: 240000 },
|
|
41
47
|
{ test: /^gpt-5-nano(-|$)/i, tpm: 600000 },
|
|
48
|
+
{ test: /^gpt-5(-|$)/i, tpm: 90000 },
|
|
42
49
|
|
|
43
50
|
{ test: /^gpt-4o(-|$)/i, tpm: 450000 },
|
|
44
51
|
{ test: /^gpt-4\.1(-|$)/i, tpm:1000000 },
|
|
@@ -143,6 +150,25 @@ export const tokenEstimate = async (model, text) => {
|
|
|
143
150
|
return approx;
|
|
144
151
|
};
|
|
145
152
|
|
|
153
|
+
/**
|
|
154
|
+
* Warn when a request is close to a known model context window. This is a
|
|
155
|
+
* diagnostic only: Testronaut does not truncate or compact content here.
|
|
156
|
+
*/
|
|
157
|
+
export async function warnIfContextNearLimit(model, payload, threshold = 0.9) {
|
|
158
|
+
const contextWindow = getOpenAIModel(model)?.contextWindow;
|
|
159
|
+
if (!contextWindow) return { warned: false };
|
|
160
|
+
|
|
161
|
+
const estimatedTokens = await tokenEstimate(model, payload);
|
|
162
|
+
const warned = estimatedTokens >= contextWindow * threshold;
|
|
163
|
+
if (warned) {
|
|
164
|
+
console.warn(
|
|
165
|
+
`β οΈ Estimated request context for ${model} is ${estimatedTokens}/${contextWindow} tokens. ` +
|
|
166
|
+
'Testronaut will send it unchanged; consider reducing mission or tool history.'
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
return { warned, estimatedTokens, contextWindow };
|
|
170
|
+
}
|
|
171
|
+
|
|
146
172
|
/* ---------------- Dynamic limit resolution ---------------- */
|
|
147
173
|
|
|
148
174
|
/**
|