testronaut 1.5.0 β 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +114 -0
- package/bin/createWelcomeMission.js +5 -2
- package/bin/initHelpers.js +3 -0
- package/core/tags.js +50 -0
- package/package.json +1 -1
- package/runner/testronaut.js +11 -1
- package/tests/binTests/cli.helpers.test.js +43 -0
- package/tests/coreTests/tags.test.js +31 -0
- package/tests/runnerTests/testronaut.test.js +35 -0
- package/tests/toolsTests/generateHtmlReport.test.js +8 -0
- package/tools/generateHtmlReport.js +93 -2
package/bin/cli.js
CHANGED
|
@@ -51,6 +51,8 @@ const exec = promisify(execCmd);
|
|
|
51
51
|
import url from 'url';
|
|
52
52
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
53
53
|
import { discoverMissionFiles } from '../core/missionDiscovery.js';
|
|
54
|
+
import { loadConfig } from '../core/config.js';
|
|
55
|
+
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../core/tags.js';
|
|
54
56
|
|
|
55
57
|
// Keep PW browsers inside the project to avoid global cache skew
|
|
56
58
|
process.env.PLAYWRIGHT_BROWSERS_PATH = process.env.PLAYWRIGHT_BROWSERS_PATH || '0';
|
|
@@ -311,6 +313,61 @@ export const __test__ = {
|
|
|
311
313
|
isDirectInvocation,
|
|
312
314
|
};
|
|
313
315
|
|
|
316
|
+
function parseTagArgs(argsList) {
|
|
317
|
+
const nextArgs = [];
|
|
318
|
+
const tagValues = [];
|
|
319
|
+
const addTagValues = [];
|
|
320
|
+
let tagMatchValue;
|
|
321
|
+
let tagsPresent = false;
|
|
322
|
+
let addTagsPresent = false;
|
|
323
|
+
let tagMatchPresent = false;
|
|
324
|
+
let tagsMissingValue = false;
|
|
325
|
+
let addTagsMissingValue = false;
|
|
326
|
+
|
|
327
|
+
const matchFlag = (raw, names) => names.some(name => raw === name || raw.startsWith(`${name}=`));
|
|
328
|
+
const readValue = (raw, index) => {
|
|
329
|
+
if (raw.includes('=')) return { value: raw.slice(raw.indexOf('=') + 1), consumed: 1 };
|
|
330
|
+
const candidate = argsList[index + 1];
|
|
331
|
+
return candidate && !candidate.startsWith('-')
|
|
332
|
+
? { value: candidate, consumed: 2 }
|
|
333
|
+
: { value: undefined, consumed: 1 };
|
|
334
|
+
};
|
|
335
|
+
|
|
336
|
+
for (let index = 0; index < argsList.length;) {
|
|
337
|
+
const raw = argsList[index];
|
|
338
|
+
if (matchFlag(raw, ['--tag', '--tags'])) {
|
|
339
|
+
const { value, consumed } = readValue(raw, index);
|
|
340
|
+
tagsPresent = true;
|
|
341
|
+
if (value !== undefined) tagValues.push(value);
|
|
342
|
+
else tagsMissingValue = true;
|
|
343
|
+
index += consumed;
|
|
344
|
+
} else if (matchFlag(raw, ['--add-tag', '--add-tags', '--add_tag', '--add_tags'])) {
|
|
345
|
+
const { value, consumed } = readValue(raw, index);
|
|
346
|
+
addTagsPresent = true;
|
|
347
|
+
if (value !== undefined) addTagValues.push(value);
|
|
348
|
+
else addTagsMissingValue = true;
|
|
349
|
+
index += consumed;
|
|
350
|
+
} else if (matchFlag(raw, ['--tag-match', '--tag_match'])) {
|
|
351
|
+
const { value, consumed } = readValue(raw, index);
|
|
352
|
+
tagMatchPresent = true;
|
|
353
|
+
tagMatchValue = value;
|
|
354
|
+
index += consumed;
|
|
355
|
+
} else {
|
|
356
|
+
nextArgs.push(raw);
|
|
357
|
+
index += 1;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
return {
|
|
362
|
+
args: nextArgs,
|
|
363
|
+
tags: { present: tagsPresent, value: tagValues.length ? tagValues.join(',') : undefined, missingValue: tagsMissingValue },
|
|
364
|
+
addTags: { present: addTagsPresent, value: addTagValues.length ? addTagValues.join(',') : undefined, missingValue: addTagsMissingValue },
|
|
365
|
+
tagMatch: { present: tagMatchPresent, value: tagMatchValue },
|
|
366
|
+
hasUnquotedCommaSpace: [...tagValues, ...addTagValues].some(value => value.trimEnd().endsWith(',')),
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
__test__.parseTagArgs = parseTagArgs;
|
|
370
|
+
|
|
314
371
|
// Look for --model=<id> or --model <id>
|
|
315
372
|
let modelOverride;
|
|
316
373
|
const modelFlagIndex = args.findIndex(a => a === '--model' || a.startsWith('--model='));
|
|
@@ -356,6 +413,26 @@ if (cliMfaName) {
|
|
|
356
413
|
console.log(`π MFA nickname override: ${process.env.TESTRONAUT_MFA_NAME}`);
|
|
357
414
|
}
|
|
358
415
|
|
|
416
|
+
const tagArgs = parseTagArgs(args);
|
|
417
|
+
args = tagArgs.args;
|
|
418
|
+
let cliTags;
|
|
419
|
+
let cliAddTags = [];
|
|
420
|
+
let cliTagMatch;
|
|
421
|
+
try {
|
|
422
|
+
if (tagArgs.hasUnquotedCommaSpace) {
|
|
423
|
+
throw new Error('A tag list ends with a comma. Quote comma-and-space lists (for example --tags "authentication, smoke") or repeat --tag for each tag.');
|
|
424
|
+
}
|
|
425
|
+
if (tagArgs.tags.missingValue || (tagArgs.tags.present && !tagArgs.tags.value)) throw new Error('--tag/--tags requires a tag value.');
|
|
426
|
+
if (tagArgs.addTags.missingValue || (tagArgs.addTags.present && !tagArgs.addTags.value)) throw new Error('--add-tag/--add-tags requires a tag value.');
|
|
427
|
+
if (tagArgs.tagMatch.present && !tagArgs.tagMatch.value) throw new Error('--tag-match requires "any" or "all".');
|
|
428
|
+
if (tagArgs.tags.present) cliTags = normalizeTags(tagArgs.tags.value, { allowUntagged: true });
|
|
429
|
+
if (tagArgs.addTags.present) cliAddTags = normalizeTags(tagArgs.addTags.value);
|
|
430
|
+
if (tagArgs.tagMatch.present) cliTagMatch = normalizeTagMatch(tagArgs.tagMatch.value);
|
|
431
|
+
} catch (error) {
|
|
432
|
+
console.error(`β ${error.message}`);
|
|
433
|
+
process.exit(1);
|
|
434
|
+
}
|
|
435
|
+
|
|
359
436
|
// Look for --debug / --debug=<bool> / --no-debug
|
|
360
437
|
// Normalize CLI boolean strings into true/false/null for optional flags.
|
|
361
438
|
const parseBool = (v) => {
|
|
@@ -545,6 +622,11 @@ Options:
|
|
|
545
622
|
--human-input-timeout=<s> Override human input wait timeout in seconds (default: 60)
|
|
546
623
|
--help Show this help message
|
|
547
624
|
--retry_limit=<n> Override agent turn retry limits (minimum 1, maximum 10)
|
|
625
|
+
--tag=<tag> Run missions matching a tag; repeat for multiple tags
|
|
626
|
+
--tags=<tag,...> Compact comma-list form (OR/any by default)
|
|
627
|
+
--tag-match=<any|all> Match any or all requested tags
|
|
628
|
+
--add-tag=<tag> Add one report tag; repeat for multiple tags
|
|
629
|
+
--add-tags=<tag,...> Compact comma-list form for report tags
|
|
548
630
|
|
|
549
631
|
Examples:
|
|
550
632
|
${cliName}
|
|
@@ -552,6 +634,8 @@ Examples:
|
|
|
552
634
|
${cliName} upload
|
|
553
635
|
${cliName} serve
|
|
554
636
|
${cliName} --init
|
|
637
|
+
${cliName} --tag authentication --tag smoke
|
|
638
|
+
${cliName} --add-tag staging --add-tag full-test-run
|
|
555
639
|
`;
|
|
556
640
|
|
|
557
641
|
async function main() {
|
|
@@ -619,6 +703,21 @@ if (args.includes('serve') || args.includes('view')) {
|
|
|
619
703
|
}
|
|
620
704
|
|
|
621
705
|
const { root: missionsRoot, files: discoveredMissions } = await discoverMissionFiles({ cwd: process.cwd() });
|
|
706
|
+
const tagConfig = await loadConfig(process.cwd());
|
|
707
|
+
const explicitFiles = args.length > 0;
|
|
708
|
+
let requestedTags = [];
|
|
709
|
+
let tagMatch = 'any';
|
|
710
|
+
try {
|
|
711
|
+
requestedTags = explicitFiles ? [] : (cliTags ?? normalizeTags(tagConfig?.tags, { allowUntagged: true }));
|
|
712
|
+
tagMatch = explicitFiles ? 'any' : normalizeTagMatch(cliTagMatch ?? tagConfig?.tagMatch);
|
|
713
|
+
process.env.TESTRONAUT_ADD_TAGS = normalizeTags([
|
|
714
|
+
...normalizeTags(tagConfig?.addTags),
|
|
715
|
+
...cliAddTags,
|
|
716
|
+
]).join(',');
|
|
717
|
+
} catch (error) {
|
|
718
|
+
console.error(`β ${error.message}`);
|
|
719
|
+
process.exit(1);
|
|
720
|
+
}
|
|
622
721
|
|
|
623
722
|
if (!fs.existsSync(missionsRoot)) {
|
|
624
723
|
console.error(`β Missions directory not found: ${path.relative(process.cwd(), missionsRoot)}`);
|
|
@@ -630,17 +729,24 @@ const runFile = async (filePath) => {
|
|
|
630
729
|
const modulePath = path.resolve(missionsRoot, filePath);
|
|
631
730
|
const missionsModule = await import(`file://${modulePath}`);
|
|
632
731
|
|
|
732
|
+
if (!explicitFiles) {
|
|
733
|
+
const moduleTags = normalizeTags(missionsModule.tags);
|
|
734
|
+
if (!matchesTagFilter(moduleTags, requestedTags, tagMatch)) return false;
|
|
735
|
+
}
|
|
736
|
+
|
|
633
737
|
if (typeof missionsModule.executeMission === 'function') {
|
|
634
738
|
const result = await missionsModule.executeMission();
|
|
635
739
|
allResults.push({
|
|
636
740
|
file: filePath,
|
|
637
741
|
result
|
|
638
742
|
});
|
|
743
|
+
return true;
|
|
639
744
|
}
|
|
640
745
|
} catch (err) {
|
|
641
746
|
console.error(`β Error running mission: ${filePath}`);
|
|
642
747
|
console.error(err);
|
|
643
748
|
}
|
|
749
|
+
return false;
|
|
644
750
|
};
|
|
645
751
|
|
|
646
752
|
if (args.length > 0) {
|
|
@@ -655,6 +761,11 @@ if (args.length > 0) {
|
|
|
655
761
|
}
|
|
656
762
|
}
|
|
657
763
|
|
|
764
|
+
if (!explicitFiles && requestedTags.length && allResults.length === 0) {
|
|
765
|
+
console.error(`β No missions matched ${tagMatch === 'all' ? 'all' : 'any'} of: ${requestedTags.join(', ')}`);
|
|
766
|
+
process.exit(1);
|
|
767
|
+
}
|
|
768
|
+
|
|
658
769
|
const endTime = new Date();
|
|
659
770
|
|
|
660
771
|
const flatMissions = allResults.flatMap(entry => {
|
|
@@ -696,6 +807,9 @@ const report = {
|
|
|
696
807
|
},
|
|
697
808
|
missions: flatMissions
|
|
698
809
|
};
|
|
810
|
+
report.tags = normalizeTags(flatMissions.flatMap(m =>
|
|
811
|
+
m.submissionType === 'mission' ? (m.tags ?? []) : []
|
|
812
|
+
));
|
|
699
813
|
|
|
700
814
|
const outputDir = './missions/mission_reports';
|
|
701
815
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
@@ -15,7 +15,9 @@ export const createWelcomeMission = async () => {
|
|
|
15
15
|
return;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
const content = `import { runMissions } from 'testronaut';
|
|
19
|
+
|
|
20
|
+
export const tags = [];
|
|
19
21
|
|
|
20
22
|
const welcomeGoal = \`
|
|
21
23
|
Welcome to Testronaut!
|
|
@@ -29,7 +31,8 @@ If you cannot reach this message, report FAILURE.
|
|
|
29
31
|
|
|
30
32
|
export async function executeMission() {
|
|
31
33
|
return await runMissions({
|
|
32
|
-
mission: welcomeGoal
|
|
34
|
+
mission: welcomeGoal,
|
|
35
|
+
tags
|
|
33
36
|
}, 'Welcome Mission!');
|
|
34
37
|
}
|
|
35
38
|
|
package/bin/initHelpers.js
CHANGED
package/core/tags.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export const TAG_PATTERN = /^[a-z0-9_-]+$/;
|
|
2
|
+
export const MAX_TAG_LENGTH = 40;
|
|
3
|
+
export const MAX_TAGS_PER_MISSION = 50;
|
|
4
|
+
export const UNTAGGED_FILTER = 'untagged';
|
|
5
|
+
|
|
6
|
+
export function normalizeTag(value) {
|
|
7
|
+
const tag = String(value ?? '').trim().toLowerCase();
|
|
8
|
+
if (!tag) throw new Error('Tags cannot be empty.');
|
|
9
|
+
if (tag.length > MAX_TAG_LENGTH) {
|
|
10
|
+
throw new Error(`Tag "${tag}" exceeds the ${MAX_TAG_LENGTH}-character limit.`);
|
|
11
|
+
}
|
|
12
|
+
if (!TAG_PATTERN.test(tag)) {
|
|
13
|
+
throw new Error(`Invalid tag "${tag}". Use only letters, numbers, hyphens, and underscores.`);
|
|
14
|
+
}
|
|
15
|
+
return tag;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function normalizeTags(values, { allowUntagged = false } = {}) {
|
|
19
|
+
const input = Array.isArray(values)
|
|
20
|
+
? values
|
|
21
|
+
: typeof values === 'string'
|
|
22
|
+
? (values.trim() ? values.split(',') : [])
|
|
23
|
+
: values == null
|
|
24
|
+
? []
|
|
25
|
+
: [values];
|
|
26
|
+
const tags = [...new Set(input.map(normalizeTag))].sort();
|
|
27
|
+
if (!allowUntagged && tags.includes(UNTAGGED_FILTER)) {
|
|
28
|
+
throw new Error(`"${UNTAGGED_FILTER}" is reserved for filtering and cannot be saved as a tag.`);
|
|
29
|
+
}
|
|
30
|
+
if (tags.length > MAX_TAGS_PER_MISSION) {
|
|
31
|
+
throw new Error(`A mission can have at most ${MAX_TAGS_PER_MISSION} tags.`);
|
|
32
|
+
}
|
|
33
|
+
return tags;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function matchesTagFilter(missionTags, requestedTags, match = 'any') {
|
|
37
|
+
const actual = normalizeTags(missionTags);
|
|
38
|
+
const requested = normalizeTags(requestedTags, { allowUntagged: true });
|
|
39
|
+
if (!requested.length) return true;
|
|
40
|
+
const has = (tag) => tag === UNTAGGED_FILTER ? actual.length === 0 : actual.includes(tag);
|
|
41
|
+
return match === 'all' ? requested.every(has) : requested.some(has);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function normalizeTagMatch(value) {
|
|
45
|
+
const match = String(value ?? 'any').trim().toLowerCase();
|
|
46
|
+
if (match !== 'any' && match !== 'all') {
|
|
47
|
+
throw new Error('tagMatch must be "any" or "all".');
|
|
48
|
+
}
|
|
49
|
+
return match;
|
|
50
|
+
}
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -28,6 +28,7 @@ import path from 'path';
|
|
|
28
28
|
import { runAgent } from '../core/agent.js';
|
|
29
29
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
30
|
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
|
+
import { normalizeTags } from '../core/tags.js';
|
|
31
32
|
|
|
32
33
|
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
33
34
|
const isDebugEnabled = () => {
|
|
@@ -44,7 +45,7 @@ const formatListLimit = (v) => v === Infinity ? 'all' : v;
|
|
|
44
45
|
* @param {{ preMission?: any|any[], mission?: any|any[], postMission?: any|any[] }} params
|
|
45
46
|
* @param {string} missionName
|
|
46
47
|
*/
|
|
47
|
-
export async function runMissions({ preMission, mission, postMission }, missionName) {
|
|
48
|
+
export async function runMissions({ preMission, mission, postMission, tags = [] }, missionName) {
|
|
48
49
|
// 1) Config and turn budget (with guardrails)
|
|
49
50
|
const cfg = await loadConfig();
|
|
50
51
|
const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
|
|
@@ -55,6 +56,11 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
55
56
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
56
57
|
const humanInput = getHumanInputConfig(cfg);
|
|
57
58
|
const debugEnabled = isDebugEnabled();
|
|
59
|
+
const missionTags = normalizeTags([
|
|
60
|
+
...normalizeTags(tags),
|
|
61
|
+
...normalizeTags(cfg?.addTags),
|
|
62
|
+
...normalizeTags(process.env.TESTRONAUT_ADD_TAGS),
|
|
63
|
+
]);
|
|
58
64
|
if (notes.length) {
|
|
59
65
|
console.warn(notes.join('\n'));
|
|
60
66
|
}
|
|
@@ -208,6 +214,10 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
208
214
|
return;
|
|
209
215
|
}
|
|
210
216
|
|
|
217
|
+
for (const result of success) {
|
|
218
|
+
result.tags = result.submissionType === 'mission' ? missionTags : [];
|
|
219
|
+
}
|
|
220
|
+
|
|
211
221
|
const missionStatus = success[0].steps[success[0].steps.length - 1].result;
|
|
212
222
|
if (missionStatus.toLowerCase().includes('failure')) {
|
|
213
223
|
success[0].status = 'failed';
|
|
@@ -143,6 +143,49 @@ describe('cli helpers', () => {
|
|
|
143
143
|
expect(res.invalid).toBe(true);
|
|
144
144
|
});
|
|
145
145
|
|
|
146
|
+
it('parses tag flags and aliases without leaving mission-like arguments', () => {
|
|
147
|
+
const { parseTagArgs } = __test__;
|
|
148
|
+
const res = parseTagArgs([
|
|
149
|
+
'--tags=smoke,authentication',
|
|
150
|
+
'--tag-match', 'all',
|
|
151
|
+
'--add_tags', 'staging',
|
|
152
|
+
'login.mission.js',
|
|
153
|
+
]);
|
|
154
|
+
expect(res.tags.value).toBe('smoke,authentication');
|
|
155
|
+
expect(res.tagMatch.value).toBe('all');
|
|
156
|
+
expect(res.addTags.value).toBe('staging');
|
|
157
|
+
expect(res.args).toEqual(['login.mission.js']);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it('combines repeated singular and plural tag flags', () => {
|
|
161
|
+
const { parseTagArgs } = __test__;
|
|
162
|
+
const res = parseTagArgs([
|
|
163
|
+
'--tag', 'authentication',
|
|
164
|
+
'--tag=smoke',
|
|
165
|
+
'--tags', 'checkout, navigation',
|
|
166
|
+
'--add-tag', 'staging',
|
|
167
|
+
'--add-tags=nightly,full-test-run',
|
|
168
|
+
]);
|
|
169
|
+
expect(res.tags.value).toBe('authentication,smoke,checkout, navigation');
|
|
170
|
+
expect(res.addTags.value).toBe('staging,nightly,full-test-run');
|
|
171
|
+
expect(res.hasUnquotedCommaSpace).toBe(false);
|
|
172
|
+
expect(res.args).toEqual([]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('detects the common unquoted comma-space form for a friendly error', () => {
|
|
176
|
+
const { parseTagArgs } = __test__;
|
|
177
|
+
const res = parseTagArgs(['--tags', 'authentication,', 'smoke']);
|
|
178
|
+
expect(res.hasUnquotedCommaSpace).toBe(true);
|
|
179
|
+
expect(res.args).toEqual(['smoke']);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('records missing values even when another repeated flag is valid', () => {
|
|
183
|
+
const { parseTagArgs } = __test__;
|
|
184
|
+
const res = parseTagArgs(['--tag', '--tag', 'smoke']);
|
|
185
|
+
expect(res.tags.missingValue).toBe(true);
|
|
186
|
+
expect(res.tags.value).toBe('smoke');
|
|
187
|
+
});
|
|
188
|
+
|
|
146
189
|
describe('detectCliName', () => {
|
|
147
190
|
const { detectCliName } = __test__;
|
|
148
191
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { matchesTagFilter, normalizeTagMatch, normalizeTags } from '../../core/tags.js';
|
|
3
|
+
|
|
4
|
+
describe('tag helpers', () => {
|
|
5
|
+
it('normalizes case, order, and duplicates', () => {
|
|
6
|
+
expect(normalizeTags(['Smoke', 'authentication', 'smoke'])).toEqual(['authentication', 'smoke']);
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it('treats omitted and blank optional tag sources as no tags', () => {
|
|
10
|
+
expect(normalizeTags()).toEqual([]);
|
|
11
|
+
expect(normalizeTags(null)).toEqual([]);
|
|
12
|
+
expect(normalizeTags('')).toEqual([]);
|
|
13
|
+
expect(normalizeTags(' ')).toEqual([]);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('rejects spaces and the reserved untagged value', () => {
|
|
17
|
+
expect(() => normalizeTags(['human in loop'])).toThrow(/Invalid tag/);
|
|
18
|
+
expect(() => normalizeTags(['untagged'])).toThrow(/reserved/);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('supports any, all, and untagged matching', () => {
|
|
22
|
+
expect(matchesTagFilter(['smoke'], ['smoke', 'checkout'], 'any')).toBe(true);
|
|
23
|
+
expect(matchesTagFilter(['smoke'], ['smoke', 'checkout'], 'all')).toBe(false);
|
|
24
|
+
expect(matchesTagFilter([], ['untagged'], 'any')).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('defaults tag matching to any', () => {
|
|
28
|
+
expect(normalizeTagMatch()).toBe('any');
|
|
29
|
+
expect(() => normalizeTagMatch('some')).toThrow(/any.*all/);
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -137,6 +137,41 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
137
137
|
log.mockRestore();
|
|
138
138
|
});
|
|
139
139
|
|
|
140
|
+
it('adds normalized tags only to main mission submissions', async () => {
|
|
141
|
+
loadConfig.mockResolvedValue({ addTags: ['staging', 'Smoke'] });
|
|
142
|
+
enforceTurnBudget.mockReturnValue({ effectiveMax: 20, limits: {}, notes: [], strict: false });
|
|
143
|
+
process.env.TESTRONAUT_ADD_TAGS = 'full-test-run,smoke';
|
|
144
|
+
runAgent.mockResolvedValue([
|
|
145
|
+
{ submissionType: 'premission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
146
|
+
{ submissionType: 'mission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
147
|
+
{ submissionType: 'postmission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
148
|
+
]);
|
|
149
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
150
|
+
|
|
151
|
+
const result = await runMissions({ preMission: 'setup', mission: 'test', postMission: 'cleanup', tags: ['authentication'] }, 'Tagged');
|
|
152
|
+
|
|
153
|
+
expect(result[0].tags).toEqual([]);
|
|
154
|
+
expect(result[1].tags).toEqual(['authentication', 'full-test-run', 'smoke', 'staging']);
|
|
155
|
+
expect(result[2].tags).toEqual([]);
|
|
156
|
+
log.mockRestore();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('runs an untagged mission when optional additive tags are blank', async () => {
|
|
160
|
+
loadConfig.mockResolvedValue({ addTags: [] });
|
|
161
|
+
enforceTurnBudget.mockReturnValue({ effectiveMax: 20, limits: {}, notes: [], strict: false });
|
|
162
|
+
process.env.TESTRONAUT_ADD_TAGS = '';
|
|
163
|
+
runAgent.mockResolvedValue([
|
|
164
|
+
{ submissionType: 'mission', steps: [{ result: 'SUCCESS' }], status: 'passed' },
|
|
165
|
+
]);
|
|
166
|
+
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
167
|
+
|
|
168
|
+
const result = await runMissions({ mission: 'untagged mission' }, 'Untagged');
|
|
169
|
+
|
|
170
|
+
expect(result[0].tags).toEqual([]);
|
|
171
|
+
expect(runAgent).toHaveBeenCalledOnce();
|
|
172
|
+
log.mockRestore();
|
|
173
|
+
});
|
|
174
|
+
|
|
140
175
|
it('normalizes single strings and functions into arrays and names missions', async () => {
|
|
141
176
|
loadConfig.mockResolvedValue({});
|
|
142
177
|
enforceTurnBudget.mockReturnValue({
|
|
@@ -27,6 +27,7 @@ describe('generateHtmlReport', () => {
|
|
|
27
27
|
submissionType: 'mission',
|
|
28
28
|
submissionName: 'Main',
|
|
29
29
|
status: 'passed',
|
|
30
|
+
tags: ['smoke'],
|
|
30
31
|
steps: [
|
|
31
32
|
{
|
|
32
33
|
turn: 0,
|
|
@@ -54,6 +55,13 @@ describe('generateHtmlReport', () => {
|
|
|
54
55
|
expect(html).toContain('Mission A');
|
|
55
56
|
expect(html).toContain('Turn 1 (re-attempt 1/5)');
|
|
56
57
|
expect(html).toContain('β οΈ Turn Issues');
|
|
58
|
+
expect(html).toContain('data-tag="smoke"');
|
|
59
|
+
expect(html).toContain('data-tags="smoke"');
|
|
60
|
+
expect(html).toContain('data-tag="untagged"');
|
|
61
|
+
expect(html).toContain('id="match-count"');
|
|
62
|
+
expect(html).toContain('No missions match these tags');
|
|
63
|
+
expect(html).toContain('Keep missions that do not match visible');
|
|
64
|
+
expect(html).not.toContain('<div class="pill">LLM:');
|
|
57
65
|
});
|
|
58
66
|
|
|
59
67
|
it('escapes unsafe text in output', () => {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import fs from 'fs';
|
|
19
19
|
import path from 'path';
|
|
20
|
+
import { normalizeTags } from '../core/tags.js';
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
23
|
* Render and write a Testronaut run report to disk.
|
|
@@ -27,6 +28,7 @@ import path from 'path';
|
|
|
27
28
|
*/
|
|
28
29
|
export function generateHtmlReport(report, outputPath) {
|
|
29
30
|
const { runId, startTime, endTime, missions = [], summary = {}, llm = {} } = report;
|
|
31
|
+
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
30
32
|
const durationSec =
|
|
31
33
|
(startTime && endTime)
|
|
32
34
|
? ((new Date(endTime) - new Date(startTime)) / 1000).toFixed(2)
|
|
@@ -40,6 +42,16 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
40
42
|
const badge = (status) =>
|
|
41
43
|
status === 'passed' ? 'β
Passed' :
|
|
42
44
|
status === 'failed' ? 'β Failed' : (status || 'β');
|
|
45
|
+
const tagPalette = [
|
|
46
|
+
['#60a5fa', 'rgba(96,165,250,.18)'], ['#a78bfa', 'rgba(167,139,250,.18)'],
|
|
47
|
+
['#34d399', 'rgba(52,211,153,.18)'], ['#f472b6', 'rgba(244,114,182,.18)'],
|
|
48
|
+
['#fbbf24', 'rgba(251,191,36,.18)'], ['#22d3ee', 'rgba(34,211,238,.18)'],
|
|
49
|
+
];
|
|
50
|
+
const tagStyle = (tag) => {
|
|
51
|
+
const index = [...tag].reduce((sum, char) => sum + char.charCodeAt(0), 0) % tagPalette.length;
|
|
52
|
+
const [color, background] = tagPalette[index];
|
|
53
|
+
return `color:${color};background:${background};border-color:${color}66`;
|
|
54
|
+
};
|
|
43
55
|
|
|
44
56
|
const submissionBlock = (m) => {
|
|
45
57
|
const mDurationSec =
|
|
@@ -128,6 +140,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
128
140
|
const firstStart = Math.min(...subs.map(s => s.startTime || 0).filter(Boolean));
|
|
129
141
|
const lastEnd = Math.max(...subs.map(s => s.endTime || 0).filter(Boolean));
|
|
130
142
|
const groupDur = (firstStart && lastEnd) ? ((lastEnd - firstStart) / 1000).toFixed(2) : 'β';
|
|
143
|
+
const tags = normalizeTags(subs.flatMap(s => s.submissionType === 'mission' ? (s.tags ?? []) : []));
|
|
131
144
|
|
|
132
145
|
// pre β mission β post
|
|
133
146
|
const order = { premission: 0, mission: 1, postmission: 2 };
|
|
@@ -137,9 +150,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
137
150
|
);
|
|
138
151
|
|
|
139
152
|
return `
|
|
140
|
-
<details class="mission-group">
|
|
153
|
+
<details class="mission-group" data-tags="${esc(tags.join(','))}">
|
|
141
154
|
<summary>
|
|
142
155
|
<span class="name">${esc(missionName)}</span>
|
|
156
|
+
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
143
157
|
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
144
158
|
<span class="meta">submissions: ${subs.length} β’ steps: ${totalSteps} β’ duration: ${groupDur}s</span>
|
|
145
159
|
<span class="toolbar">
|
|
@@ -233,6 +247,21 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
233
247
|
}
|
|
234
248
|
.pill.ok{ background: var(--chip-ok-bg); color: var(--ok); border-color: var(--chip-ok-border); font-weight:700; }
|
|
235
249
|
.pill.bad{ background: var(--chip-bad-bg); color: var(--bad); border-color: var(--chip-bad-border); font-weight:700; }
|
|
250
|
+
.tag-filter{max-width:928px;margin:0 auto 20px;padding:18px 16px;background:rgba(2,6,23,.30);}
|
|
251
|
+
.tag-filter-heading{display:flex;align-items:center;gap:7px;margin-bottom:12px;flex-wrap:wrap;}
|
|
252
|
+
.tag-filter-title{font-size:14px;color:var(--text);font-weight:800;}
|
|
253
|
+
.info{display:inline-grid;place-items:center;width:17px;height:17px;border:1px solid var(--chip-border);border-radius:50%;font-size:11px;color:var(--text-muted);cursor:help;}
|
|
254
|
+
.match-count{margin-left:auto;color:var(--text-muted);font-size:12px;}
|
|
255
|
+
.tag-buttons,.mission-tags{display:flex;gap:6px;flex-wrap:wrap;align-items:center;}
|
|
256
|
+
.tag-button,.tag-small{border:1px solid var(--chip-border);background:rgba(255,255,255,.06);color:var(--text);border-radius:999px;padding:5px 10px;font-size:11px;font-weight:700;}
|
|
257
|
+
.tag-button{cursor:pointer;transition:transform .15s ease,background .15s ease}.tag-button:hover{transform:translateY(-1px);background:rgba(255,255,255,.11)}.tag-button.active{box-shadow:0 0 0 2px rgba(96,165,250,.38);}
|
|
258
|
+
.filter-controls{display:flex;gap:16px;align-items:center;margin-top:14px;font-size:12px;color:var(--text-muted);flex-wrap:wrap;}
|
|
259
|
+
.filter-controls select{color:var(--text);background:#111a31;border:1px solid var(--chip-border);border-radius:8px;padding:6px 24px 6px 8px;}
|
|
260
|
+
.show-control{cursor:help;display:flex;align-items:center;gap:5px;}
|
|
261
|
+
.filter-empty{max-width:928px;margin:12px auto;padding:28px 16px;text-align:center;border:1px dashed var(--hairline-strong);border-radius:16px;color:var(--text-muted);}
|
|
262
|
+
.filter-empty strong{display:block;color:var(--text);font-size:18px;margin-bottom:6px;}
|
|
263
|
+
.mission-group.nonmatching{opacity:.45;}
|
|
264
|
+
@media(max-width:640px){.match-count{width:100%;margin-left:0}.tag-filter{padding:14px 12px}.meta{display:none}}
|
|
236
265
|
|
|
237
266
|
/* ===== Disclosure blocks ===== */
|
|
238
267
|
details{ background: rgba(255,255,255,.05); border:1px solid var(--hairline); border-radius:16px; margin:10px 0; overflow:hidden; }
|
|
@@ -328,14 +357,76 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
328
357
|
<div class="pill">Missions: ${esc(summary.totalMissions ?? totals.total)}</div>
|
|
329
358
|
<div class="pill ok">Passed: ${esc(summary.passed ?? totals.passed)}</div>
|
|
330
359
|
<div class="pill bad">Failed: ${esc(summary.failed ?? totals.failed)}</div>
|
|
331
|
-
|
|
360
|
+
</div>
|
|
361
|
+
|
|
362
|
+
<div class="tag-filter glass">
|
|
363
|
+
<div class="tag-filter-heading">
|
|
364
|
+
<span class="tag-filter-title">Filter missions by tag</span>
|
|
365
|
+
<span class="info" title="Select one or more tags to focus the report. Match Any uses OR; Match All requires every selected tag.">i</span>
|
|
366
|
+
<span class="match-count" id="match-count">${totals.total} of ${totals.total} missions match</span>
|
|
367
|
+
</div>
|
|
368
|
+
<div class="tag-buttons">
|
|
369
|
+
${reportTags.map(tag => `<button type="button" class="tag-button" data-tag="${esc(tag)}" style="${tagStyle(tag)}">${esc(tag)}</button>`).join('')}
|
|
370
|
+
<button type="button" class="tag-button" data-tag="untagged">untagged</button>
|
|
371
|
+
</div>
|
|
372
|
+
<div class="filter-controls">
|
|
373
|
+
<label>Match <select id="tag-match"><option value="any">any</option><option value="all">all</option></select></label>
|
|
374
|
+
<label class="show-control" title="Keep missions that do not match visible in a muted style instead of hiding them."><input id="show-nonmatching" type="checkbox"> Show nonmatching missions <span class="info">i</span></label>
|
|
375
|
+
<button type="button" id="clear-tags" class="tag-button" hidden>Clear filters</button>
|
|
376
|
+
</div>
|
|
332
377
|
</div>
|
|
333
378
|
|
|
334
379
|
<div class="container">
|
|
335
380
|
${groupsHtml || '<div class="glass empty">No missions recorded.</div>'}
|
|
336
381
|
</div>
|
|
382
|
+
<div class="filter-empty" id="filter-empty" hidden>
|
|
383
|
+
<strong>No missions match these tags</strong>
|
|
384
|
+
Try removing a tag, switching to βAny,β or showing nonmatching missions.
|
|
385
|
+
</div>
|
|
337
386
|
<script>
|
|
338
387
|
(function () {
|
|
388
|
+
var selectedTags = [];
|
|
389
|
+
function applyTagFilter() {
|
|
390
|
+
var match = document.getElementById('tag-match');
|
|
391
|
+
var show = document.getElementById('show-nonmatching');
|
|
392
|
+
var matchedCount = 0;
|
|
393
|
+
var totalCount = 0;
|
|
394
|
+
document.querySelectorAll('.mission-group').forEach(function (group) {
|
|
395
|
+
totalCount += 1;
|
|
396
|
+
var tags = (group.getAttribute('data-tags') || '').split(',').filter(Boolean);
|
|
397
|
+
var test = function (tag) { return tag === 'untagged' ? tags.length === 0 : tags.indexOf(tag) >= 0; };
|
|
398
|
+
var matches = !selectedTags.length || ((match && match.value === 'all') ? selectedTags.every(test) : selectedTags.some(test));
|
|
399
|
+
if (matches) matchedCount += 1;
|
|
400
|
+
group.hidden = !matches && !(show && show.checked);
|
|
401
|
+
group.classList.toggle('nonmatching', !matches);
|
|
402
|
+
});
|
|
403
|
+
var count = document.getElementById('match-count');
|
|
404
|
+
var empty = document.getElementById('filter-empty');
|
|
405
|
+
var clear = document.getElementById('clear-tags');
|
|
406
|
+
if (count) count.textContent = matchedCount + ' of ' + totalCount + ' missions match';
|
|
407
|
+
if (empty) empty.hidden = matchedCount !== 0 || !selectedTags.length || !!(show && show.checked);
|
|
408
|
+
if (clear) clear.hidden = selectedTags.length === 0;
|
|
409
|
+
}
|
|
410
|
+
document.querySelectorAll('.tag-button[data-tag]').forEach(function (button) {
|
|
411
|
+
button.addEventListener('click', function () {
|
|
412
|
+
var tag = button.getAttribute('data-tag');
|
|
413
|
+
var idx = selectedTags.indexOf(tag);
|
|
414
|
+
if (idx >= 0) selectedTags.splice(idx, 1); else selectedTags.push(tag);
|
|
415
|
+
button.classList.toggle('active', idx < 0);
|
|
416
|
+
applyTagFilter();
|
|
417
|
+
});
|
|
418
|
+
});
|
|
419
|
+
var tagMatch = document.getElementById('tag-match');
|
|
420
|
+
var showNonmatching = document.getElementById('show-nonmatching');
|
|
421
|
+
var clearTags = document.getElementById('clear-tags');
|
|
422
|
+
if (tagMatch) tagMatch.addEventListener('change', applyTagFilter);
|
|
423
|
+
if (showNonmatching) showNonmatching.addEventListener('change', applyTagFilter);
|
|
424
|
+
if (clearTags) clearTags.addEventListener('click', function () {
|
|
425
|
+
selectedTags = [];
|
|
426
|
+
document.querySelectorAll('.tag-button[data-tag]').forEach(function (button) { button.classList.remove('active'); });
|
|
427
|
+
applyTagFilter();
|
|
428
|
+
});
|
|
429
|
+
applyTagFilter();
|
|
339
430
|
function setOpenAll(root, selector, open) {
|
|
340
431
|
root.querySelectorAll(selector).forEach(function (el) {
|
|
341
432
|
if (el && 'open' in el) el.open = open;
|