qa-engineer 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -59
- package/package.json +1 -1
- package/packages/engine/bin/qa-engine.mjs +232 -8
- package/packages/engine/lib/analysis/har.mjs +34 -0
- package/packages/engine/lib/analysis/network.mjs +237 -0
- package/packages/engine/lib/analysis/report-html.mjs +47 -734
- package/packages/engine/lib/artifacts/manager.mjs +453 -0
- package/packages/engine/lib/artifacts/mime.mjs +109 -0
- package/packages/engine/lib/artifacts/zip-write.mjs +143 -0
- package/packages/engine/lib/report/components/charts.mjs +424 -0
- package/packages/engine/lib/report/components/evidence.mjs +207 -0
- package/packages/engine/lib/report/components/findings.mjs +258 -0
- package/packages/engine/lib/report/components/nav.mjs +99 -0
- package/packages/engine/lib/report/components/primitives.mjs +246 -0
- package/packages/engine/lib/report/components/runtime.mjs +246 -0
- package/packages/engine/lib/report/components/timeline.mjs +65 -0
- package/packages/engine/lib/report/core/model.mjs +270 -0
- package/packages/engine/lib/report/core/normalize.mjs +226 -0
- package/packages/engine/lib/report/core/sections.mjs +978 -0
- package/packages/engine/lib/report/export/bundle.mjs +293 -0
- package/packages/engine/lib/report/export/html.mjs +183 -0
- package/packages/engine/lib/report/export/machine.mjs +290 -0
- package/packages/engine/lib/report/export/markdown.mjs +323 -0
- package/packages/engine/lib/report/schemas/qa-report.schema.json +555 -0
- package/packages/engine/lib/report/theme/css.mjs +529 -0
- package/packages/engine/lib/report/theme/tokens.mjs +137 -0
- package/packages/engine/lib/report/version.mjs +78 -0
- package/packages/engine/package.json +2 -2
- package/packages/installer/lib/agents/targets.mjs +90 -0
- package/packages/installer/lib/agents/user-level.mjs +80 -0
- package/packages/installer/lib/cli/flags.mjs +20 -2
- package/packages/installer/lib/commands/doctor.mjs +6 -4
- package/packages/installer/lib/commands/install.mjs +134 -93
- package/packages/installer/lib/commands/repair.mjs +10 -6
- package/packages/installer/lib/commands/self-test.mjs +4 -3
- package/packages/installer/lib/commands/uninstall.mjs +14 -8
- package/packages/installer/lib/commands/update.mjs +9 -4
- package/packages/installer/lib/commands/verify.mjs +13 -12
- package/packages/installer/lib/constants.mjs +13 -0
- package/packages/installer/lib/core/conflict.mjs +5 -4
- package/packages/installer/lib/core/fs-safe.mjs +146 -6
- package/packages/installer/lib/core/integrity.mjs +59 -0
- package/packages/installer/lib/core/lockfile.mjs +19 -3
- package/packages/installer/lib/core/plan.mjs +213 -0
- package/packages/installer/lib/core/qa-home.mjs +145 -0
- package/packages/installer/lib/core/scope.mjs +274 -0
- package/packages/installer/lib/core/validate-install.mjs +37 -12
- package/packages/installer/package.json +1 -1
- package/packages/installer/schemas/qa-lock.schema.json +119 -21
- package/shared/tooling/qa-tool.mjs +41 -5
- package/skills/qa-api/scripts/qa-tool.mjs +41 -5
- package/skills/qa-audit/scripts/qa-tool.mjs +41 -5
- package/skills/qa-debug/scripts/qa-tool.mjs +41 -5
- package/skills/qa-explore/SKILL.md +26 -10
- package/skills/qa-explore/contracts/explore-result.schema.json +517 -11
- package/skills/qa-explore/references/api-replay.md +40 -1
- package/skills/qa-explore/references/report-pipeline.md +265 -95
- package/skills/qa-explore/scripts/qa-tool.mjs +41 -5
- package/skills/qa-fix/scripts/qa-tool.mjs +41 -5
- package/skills/qa-flaky/scripts/qa-tool.mjs +41 -5
- package/skills/qa-init/scripts/qa-tool.mjs +41 -5
- package/skills/qa-report/scripts/qa-tool.mjs +41 -5
- package/skills/qa-run/scripts/qa-tool.mjs +41 -5
|
@@ -25,6 +25,7 @@ import path from 'node:path';
|
|
|
25
25
|
|
|
26
26
|
import * as junit from '../lib/analysis/junit.mjs';
|
|
27
27
|
import * as har from '../lib/analysis/har.mjs';
|
|
28
|
+
import * as network from '../lib/analysis/network.mjs';
|
|
28
29
|
import * as discovery from '../lib/analysis/discovery.mjs';
|
|
29
30
|
import * as diffGuard from '../lib/analysis/diff-guard.mjs';
|
|
30
31
|
import * as redaction from '../lib/analysis/redaction.mjs';
|
|
@@ -33,6 +34,11 @@ import * as taxonomy from '../lib/analysis/taxonomy.mjs';
|
|
|
33
34
|
import * as contextModule from '../lib/analysis/context.mjs';
|
|
34
35
|
import * as branding from '../lib/analysis/branding.mjs';
|
|
35
36
|
import * as reportHtml from '../lib/analysis/report-html.mjs';
|
|
37
|
+
import * as markdownExport from '../lib/report/export/markdown.mjs';
|
|
38
|
+
import * as machineExport from '../lib/report/export/machine.mjs';
|
|
39
|
+
import * as bundleExport from '../lib/report/export/bundle.mjs';
|
|
40
|
+
import * as artifacts from '../lib/artifacts/manager.mjs';
|
|
41
|
+
import { SchemaError } from '../lib/report/core/normalize.mjs';
|
|
36
42
|
import * as diagnostics from '../lib/diagnostics/engine.mjs';
|
|
37
43
|
import {
|
|
38
44
|
InternalContractError, validateAnalysisResult, validateExecutionResultMin, validateDiagnosis,
|
|
@@ -43,13 +49,16 @@ import * as junitFrameworks from '../lib/frameworks/junit-frameworks.mjs';
|
|
|
43
49
|
const USAGE = `usage: qa-engine <tool> <subcommand> [args]
|
|
44
50
|
|
|
45
51
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
46
|
-
|
|
52
|
+
read .qa/context.md, render a report, print the footer
|
|
53
|
+
artifacts check that the evidence a result points at is really on disk
|
|
47
54
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
48
55
|
playwright normalize a Playwright report or summarize a trace
|
|
49
56
|
|
|
50
57
|
analysis subcommands
|
|
51
58
|
junit <report.xml> normalized {tests, executed}
|
|
52
59
|
har <file.har> [--slow-ms N] redacted network summary
|
|
60
|
+
network <file.har> [--slow-ms N] the report's network block: totals,
|
|
61
|
+
[--large-bytes N] per-endpoint timings, and an issue flag
|
|
53
62
|
discover [--root DIR] [--path P ...] artifacts found, by state
|
|
54
63
|
diff-guard <diff-file> {issues, safe} — safe:false blocks a change
|
|
55
64
|
redact <file> the file's text with secrets masked
|
|
@@ -57,8 +66,25 @@ analysis subcommands
|
|
|
57
66
|
classify "<message>" [--http-status N] {classification, confidence, reason}
|
|
58
67
|
context [--root DIR] [--path P] parsed, schema-checked project context
|
|
59
68
|
report-html <result.json> [--out FILE] a self-contained HTML report
|
|
69
|
+
[--embed] [--mode M] --embed inlines images as data URIs
|
|
70
|
+
M: full executive developer artifact
|
|
71
|
+
(artifact = body only, for an embedding host)
|
|
72
|
+
report-export <result.json> --format F F: html markdown sarif junit csv json bundle
|
|
73
|
+
[--out FILE] [--embed]
|
|
74
|
+
[--mode M]
|
|
75
|
+
report-bundle <result.json> [--out DIR] a portable folder — index.html + assets/ —
|
|
76
|
+
[--zip] [--zip-out FILE] with every link verified to resolve inside
|
|
77
|
+
[--mode M] [--force] it. The canonical local output.
|
|
78
|
+
report-schema [--out FILE] the canonical producer-neutral contract
|
|
79
|
+
report-versions schema / theme / renderer versions in force
|
|
60
80
|
branding [--format html|markdown|text] the exact footer bytes for a report
|
|
61
81
|
|
|
82
|
+
artifacts subcommands
|
|
83
|
+
verify <result.json> [--base-dir D] {ok, stats, missing}; exit 1 when a file
|
|
84
|
+
[--out-dir D] a finding points at is not there
|
|
85
|
+
scan <directory> [--root D] every file found, with size and type
|
|
86
|
+
hash <file> the file's SHA-256
|
|
87
|
+
|
|
62
88
|
diagnostics subcommands
|
|
63
89
|
diagnose --execution-result P [--analysis-result P]
|
|
64
90
|
plan-repairs --diagnosis P
|
|
@@ -71,7 +97,11 @@ playwright subcommands
|
|
|
71
97
|
|
|
72
98
|
examples
|
|
73
99
|
qa-engine analysis junit test-results/results.xml
|
|
74
|
-
qa-engine
|
|
100
|
+
qa-engine artifacts verify qa-artifacts/explore-R/explore-result.json
|
|
101
|
+
qa-engine analysis report-html qa-artifacts/explore-R/explore-result.json \\
|
|
102
|
+
--out qa-artifacts/explore-R/explore-report.html
|
|
103
|
+
qa-engine analysis report-export qa-artifacts/explore-R/explore-result.json \\
|
|
104
|
+
--format sarif --out qa-artifacts/explore-R/findings.sarif
|
|
75
105
|
qa-engine diagnostics report --execution-result qa-artifacts/run.json
|
|
76
106
|
`;
|
|
77
107
|
|
|
@@ -99,8 +129,15 @@ function readJson(file) {
|
|
|
99
129
|
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
100
130
|
}
|
|
101
131
|
|
|
102
|
-
/**
|
|
103
|
-
|
|
132
|
+
/**
|
|
133
|
+
* Flags, parsed the way the Python argparse CLIs accepted them.
|
|
134
|
+
*
|
|
135
|
+
* `boolean` names matter: without them a switch consumes the token after it, so
|
|
136
|
+
* `--embed --out report.html` set `embed` to `"--out"` and left the output path as a
|
|
137
|
+
* stray positional. The report then went to stdout and the flag did nothing — a
|
|
138
|
+
* failure that looks like the feature is broken rather than the parser.
|
|
139
|
+
*/
|
|
140
|
+
function parseFlags(argv, { repeatable = [], boolean = [] } = {}) {
|
|
104
141
|
const flags = {};
|
|
105
142
|
const positional = [];
|
|
106
143
|
for (let index = 0; index < argv.length; index += 1) {
|
|
@@ -111,6 +148,10 @@ function parseFlags(argv, { repeatable = [] } = {}) {
|
|
|
111
148
|
}
|
|
112
149
|
const [name, inline] = token.slice(2).split('=', 2);
|
|
113
150
|
const camel = name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
151
|
+
if (boolean.includes(camel)) {
|
|
152
|
+
flags[camel] = inline === undefined ? true : inline !== 'false' && inline !== '0';
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
114
155
|
const value = inline !== undefined ? inline : argv[++index];
|
|
115
156
|
if (repeatable.includes(camel)) {
|
|
116
157
|
flags[camel] = [...(flags[camel] ?? []), value];
|
|
@@ -126,9 +167,24 @@ function parseFlags(argv, { repeatable = [] } = {}) {
|
|
|
126
167
|
// everywhere handed `context` an array and it failed on "path must be a string".
|
|
127
168
|
const REPEATABLE = { discover: ['path'] };
|
|
128
169
|
|
|
170
|
+
// Switches that take no value. Declared per subcommand so `--embed --out f.html` reads
|
|
171
|
+
// as two flags rather than one flag whose value is the word "--out".
|
|
172
|
+
const BOOLEAN = {
|
|
173
|
+
'report-html': ['embed'],
|
|
174
|
+
'report-export': ['embed'],
|
|
175
|
+
// `--zip` is a switch and `--zip-out` carries the path. One flag doing both would
|
|
176
|
+
// need a third parser mode that guesses from the next token, and a guess here writes
|
|
177
|
+
// the archive to a filename taken from an unrelated flag.
|
|
178
|
+
'report-bundle': ['force', 'zip', 'markdown'],
|
|
179
|
+
branding: ['metadata'],
|
|
180
|
+
};
|
|
181
|
+
|
|
129
182
|
function analysis(argv) {
|
|
130
183
|
const [subcommand, ...rest] = argv;
|
|
131
|
-
const { flags, positional } = parseFlags(rest, {
|
|
184
|
+
const { flags, positional } = parseFlags(rest, {
|
|
185
|
+
repeatable: REPEATABLE[subcommand] ?? [],
|
|
186
|
+
boolean: BOOLEAN[subcommand] ?? [],
|
|
187
|
+
});
|
|
132
188
|
|
|
133
189
|
switch (subcommand) {
|
|
134
190
|
case 'junit':
|
|
@@ -141,6 +197,18 @@ function analysis(argv) {
|
|
|
141
197
|
emit(har.parseHar(positional[0], { slowMs: Number(flags.slowMs ?? 1000) }));
|
|
142
198
|
return 0;
|
|
143
199
|
|
|
200
|
+
case 'network': {
|
|
201
|
+
// The report contract's `network` block, ready to paste into a result and
|
|
202
|
+
// validate. `har` gives the raw entries; this gives the counted, flagged,
|
|
203
|
+
// report-shaped answer, so no skill has to count requests by eye.
|
|
204
|
+
requirePositional(positional, 1, 'network <file.har>');
|
|
205
|
+
const options = {};
|
|
206
|
+
if (flags.slowMs !== undefined) options.slowMs = Number(flags.slowMs);
|
|
207
|
+
if (flags.largeBytes !== undefined) options.largeBytes = Number(flags.largeBytes);
|
|
208
|
+
emit(network.analyzeHar(positional[0], options));
|
|
209
|
+
return 0;
|
|
210
|
+
}
|
|
211
|
+
|
|
144
212
|
case 'discover':
|
|
145
213
|
emit(discovery.discover({ root: flags.root ?? '.', explicit: flags.path ?? null }));
|
|
146
214
|
return 0;
|
|
@@ -187,9 +255,22 @@ function analysis(argv) {
|
|
|
187
255
|
|
|
188
256
|
case 'report-html': {
|
|
189
257
|
requirePositional(positional, 1, 'report-html <result.json>');
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
258
|
+
// The output path is handed to the renderer, not just used to write the file:
|
|
259
|
+
// every evidence href is computed relative to it. Rendering to stdout falls back
|
|
260
|
+
// to the result's own directory, which is where a report normally lands.
|
|
261
|
+
const outPath = flags.out ? path.resolve(flags.out) : null;
|
|
262
|
+
const document = reportHtml.renderFile(positional[0], {
|
|
263
|
+
title: flags.title ?? null,
|
|
264
|
+
outPath,
|
|
265
|
+
// --embed inlines every image as a data URI, producing one file that survives
|
|
266
|
+
// being forwarded without its screenshots folder.
|
|
267
|
+
embed: flags.embed === true,
|
|
268
|
+
// --mode picks the audience: full, executive, developer, or artifact (the
|
|
269
|
+
// body-only rendering a host page supplies its own <head> for).
|
|
270
|
+
mode: flags.mode ?? 'full',
|
|
271
|
+
});
|
|
272
|
+
if (outPath) {
|
|
273
|
+
fs.writeFileSync(outPath, document);
|
|
193
274
|
emit({ written: flags.out, bytes: document.length });
|
|
194
275
|
} else {
|
|
195
276
|
process.stdout.write(document);
|
|
@@ -197,6 +278,52 @@ function analysis(argv) {
|
|
|
197
278
|
return 0;
|
|
198
279
|
}
|
|
199
280
|
|
|
281
|
+
case 'report-export': {
|
|
282
|
+
requirePositional(positional, 1, 'report-export <result.json> --format <format>');
|
|
283
|
+
return reportExport(positional[0], flags);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
case 'report-bundle': {
|
|
287
|
+
requirePositional(positional, 1, 'report-bundle <result.json> [--out DIR]');
|
|
288
|
+
const file = path.resolve(positional[0]);
|
|
289
|
+
const summary = bundleExport.writeBundle(readJson(file), {
|
|
290
|
+
resultPath: file,
|
|
291
|
+
outDir: flags.out ? path.resolve(flags.out) : undefined,
|
|
292
|
+
mode: flags.mode ?? 'full',
|
|
293
|
+
zip: flags.zipOut ?? flags.zip === true,
|
|
294
|
+
markdown: flags.markdown !== false,
|
|
295
|
+
force: flags.force === true,
|
|
296
|
+
});
|
|
297
|
+
emit(summary);
|
|
298
|
+
return 0;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
case 'report-schema': {
|
|
302
|
+
// Any agent can fetch the canonical contract and validate against it before it
|
|
303
|
+
// hands anything over. Emitting it from the engine rather than documenting it in
|
|
304
|
+
// prose is what keeps the schema an agent writes to and the schema the renderer
|
|
305
|
+
// reads the same object.
|
|
306
|
+
const schema = readJson(
|
|
307
|
+
new URL('../lib/report/schemas/qa-report.schema.json', import.meta.url).pathname,
|
|
308
|
+
);
|
|
309
|
+
if (flags.out) {
|
|
310
|
+
fs.writeFileSync(flags.out, `${JSON.stringify(schema, null, 2)}\n`);
|
|
311
|
+
emit({ written: flags.out });
|
|
312
|
+
} else {
|
|
313
|
+
process.stdout.write(`${JSON.stringify(schema, null, 2)}\n`);
|
|
314
|
+
}
|
|
315
|
+
return 0;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
case 'report-versions':
|
|
319
|
+
// What this renderer is: schema it reads, theme it paints, code that did it.
|
|
320
|
+
emit({
|
|
321
|
+
...reportHtml.versionStamp(),
|
|
322
|
+
supportedContracts: reportHtml.supportedContracts(),
|
|
323
|
+
supportedModes: reportHtml.supportedModes(),
|
|
324
|
+
});
|
|
325
|
+
return 0;
|
|
326
|
+
|
|
200
327
|
case 'branding':
|
|
201
328
|
// Written to stdout verbatim, not as JSON: the caller embeds these exact
|
|
202
329
|
// bytes in a rendered report.
|
|
@@ -209,6 +336,97 @@ function analysis(argv) {
|
|
|
209
336
|
}
|
|
210
337
|
}
|
|
211
338
|
|
|
339
|
+
// Every non-HTML rendering of a validated result. Text formats go to stdout or a file
|
|
340
|
+
// verbatim; JSON formats are emitted through the same sorted serializer as everything
|
|
341
|
+
// else, so a diff of two runs is a diff of the findings and not of key ordering.
|
|
342
|
+
const EXPORT_FORMATS = {
|
|
343
|
+
html: { text: true, render: (result, options) => reportHtml.render(result, options) },
|
|
344
|
+
markdown: { text: true, render: (result, options) => markdownExport.renderMarkdown(result, options) },
|
|
345
|
+
md: { text: true, render: (result, options) => markdownExport.renderMarkdown(result, options) },
|
|
346
|
+
sarif: { text: false, render: (result, options) => machineExport.renderSarif(result, options) },
|
|
347
|
+
junit: { text: true, render: (result, options) => machineExport.renderJUnit(result, options) },
|
|
348
|
+
csv: { text: true, render: (result, options) => machineExport.renderCsv(result, options) },
|
|
349
|
+
json: { text: false, render: (result) => result },
|
|
350
|
+
bundle: { text: false, render: (result, options) => machineExport.bundleManifest(result, options) },
|
|
351
|
+
};
|
|
352
|
+
|
|
353
|
+
function reportExport(file, flags) {
|
|
354
|
+
const format = String(flags.format ?? 'markdown').toLowerCase();
|
|
355
|
+
const exporter = EXPORT_FORMATS[format];
|
|
356
|
+
if (!exporter) {
|
|
357
|
+
throw new UsageError(
|
|
358
|
+
`unknown export format '${format}'; expected one of: ${Object.keys(EXPORT_FORMATS).sort().join(', ')}`,
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const result = readJson(file);
|
|
363
|
+
const outPath = flags.out ? path.resolve(flags.out) : null;
|
|
364
|
+
const rendered = exporter.render(result, {
|
|
365
|
+
resultPath: path.resolve(file),
|
|
366
|
+
outPath,
|
|
367
|
+
embed: flags.embed === true,
|
|
368
|
+
mode: flags.mode ?? 'full',
|
|
369
|
+
});
|
|
370
|
+
const body = exporter.text ? rendered : `${JSON.stringify(rendered, sortedReplacer(), 2)}\n`;
|
|
371
|
+
|
|
372
|
+
if (outPath) {
|
|
373
|
+
fs.writeFileSync(outPath, body);
|
|
374
|
+
emit({ written: flags.out, format, bytes: body.length });
|
|
375
|
+
} else {
|
|
376
|
+
process.stdout.write(body);
|
|
377
|
+
}
|
|
378
|
+
return 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* The artifact tool group: does the evidence this result points at actually exist?
|
|
383
|
+
*
|
|
384
|
+
* Separate from `analysis` because it answers a question about the *filesystem* rather
|
|
385
|
+
* than about a document, and because a skill runs it at a different moment — after
|
|
386
|
+
* capture and before rendering, as the gate that stops a report full of broken images
|
|
387
|
+
* from being written at all.
|
|
388
|
+
*/
|
|
389
|
+
function artifactsCommand(argv) {
|
|
390
|
+
const [subcommand, ...rest] = argv;
|
|
391
|
+
const { flags, positional } = parseFlags(rest);
|
|
392
|
+
|
|
393
|
+
switch (subcommand) {
|
|
394
|
+
case 'verify': {
|
|
395
|
+
requirePositional(positional, 1, 'verify <result.json>');
|
|
396
|
+
const file = path.resolve(positional[0]);
|
|
397
|
+
const report = artifacts.verify(readJson(file), {
|
|
398
|
+
baseDir: flags.baseDir ? path.resolve(flags.baseDir) : path.dirname(file),
|
|
399
|
+
outDir: flags.outDir ? path.resolve(flags.outDir) : undefined,
|
|
400
|
+
});
|
|
401
|
+
emit(report);
|
|
402
|
+
// Exit 1, matching `validate`: the document was readable and did not hold up.
|
|
403
|
+
return report.ok ? 0 : 1;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
case 'scan': {
|
|
407
|
+
requirePositional(positional, 1, 'scan <directory>');
|
|
408
|
+
const found = artifacts.scan(positional[0], { root: flags.root ?? positional[0] });
|
|
409
|
+
emit({
|
|
410
|
+
directory: positional[0],
|
|
411
|
+
count: found.length,
|
|
412
|
+
empty: found.filter((entry) => entry.empty).length,
|
|
413
|
+
bytes: found.reduce((sum, entry) => sum + entry.bytes, 0),
|
|
414
|
+
files: found,
|
|
415
|
+
});
|
|
416
|
+
return 0;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
case 'hash': {
|
|
420
|
+
requirePositional(positional, 1, 'hash <file>');
|
|
421
|
+
emit({ path: positional[0], sha256: artifacts.hashFile(path.resolve(positional[0])) });
|
|
422
|
+
return 0;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
default:
|
|
426
|
+
throw new UsageError(`unknown artifacts subcommand: ${subcommand ?? '(none)'}`);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
212
430
|
function diagnosticsCommand(argv) {
|
|
213
431
|
const [subcommand, ...rest] = argv;
|
|
214
432
|
const { flags } = parseFlags(rest);
|
|
@@ -306,6 +524,7 @@ export function main(argv = process.argv.slice(2)) {
|
|
|
306
524
|
const [tool, ...rest] = argv;
|
|
307
525
|
try {
|
|
308
526
|
if (tool === 'analysis') return analysis(rest);
|
|
527
|
+
if (tool === 'artifacts') return artifactsCommand(rest);
|
|
309
528
|
if (tool === 'diagnostics') return diagnosticsCommand(rest);
|
|
310
529
|
if (tool === 'playwright') return playwrightCommand(rest);
|
|
311
530
|
process.stderr.write(USAGE);
|
|
@@ -314,6 +533,11 @@ export function main(argv = process.argv.slice(2)) {
|
|
|
314
533
|
if (error instanceof junit.MalformedArtifact) return fail('malformed-artifact', error.message);
|
|
315
534
|
if (error instanceof contextModule.MalformedContext) return fail('malformed-context', error.message);
|
|
316
535
|
if (error instanceof reportHtml.ReportError) return fail('report-error', error.message);
|
|
536
|
+
// The non-HTML exporters go through the normalizer directly, so its refusals
|
|
537
|
+
// surface here rather than wrapped as a ReportError.
|
|
538
|
+
if (error instanceof SchemaError) return fail('schema-error', error.message);
|
|
539
|
+
if (error instanceof bundleExport.BundleError) return fail('bundle-error', error.message);
|
|
540
|
+
if (error instanceof artifacts.ArtifactError) return fail('artifact-error', error.message);
|
|
317
541
|
if (error instanceof branding.BrandingError) return fail('branding-error', error.message);
|
|
318
542
|
if (error instanceof InternalContractError) return fail('invalid-payload', error.message);
|
|
319
543
|
if (error instanceof UsageError) return fail('usage-error', error.message);
|
|
@@ -36,6 +36,16 @@ export function parseHarData(data, { slowMs = 1000, label = '<input>' } = {}) {
|
|
|
36
36
|
durationMs: toMillis(item?.time, label),
|
|
37
37
|
requestHeaders: redactHeaders(request.headers ?? []),
|
|
38
38
|
responseHeaders: redactHeaders(response.headers ?? []),
|
|
39
|
+
// Three facts a report needs that a summary of failures cannot supply: how big
|
|
40
|
+
// the response was, when it started relative to the others, and whether the
|
|
41
|
+
// server said anything about caching. All three are read straight from the HAR
|
|
42
|
+
// — none is inferred — and all three are absent from many HARs, which is why
|
|
43
|
+
// each has an explicit "unknown" rather than a plausible zero.
|
|
44
|
+
bytes: bodyBytes(response),
|
|
45
|
+
startedAt: typeof item.startedDateTime === 'string' ? item.startedDateTime : null,
|
|
46
|
+
cacheControl: headerValue(response.headers, 'cache-control'),
|
|
47
|
+
etag: headerValue(response.headers, 'etag'),
|
|
48
|
+
resourceType: typeof item._resourceType === 'string' ? item._resourceType : null,
|
|
39
49
|
};
|
|
40
50
|
});
|
|
41
51
|
|
|
@@ -58,6 +68,30 @@ export function parseHar(path, { slowMs = 1000 } = {}) {
|
|
|
58
68
|
return parseHarData(data, { slowMs, label: path });
|
|
59
69
|
}
|
|
60
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Response body size in bytes, or null when the HAR does not say.
|
|
73
|
+
*
|
|
74
|
+
* `content.size` is the decoded size and `bodySize` the bytes on the wire; either is
|
|
75
|
+
* `-1` when the producer did not record it. Null rather than 0, because "we do not
|
|
76
|
+
* know how big this was" and "this response was empty" are different findings and a
|
|
77
|
+
* report that conflates them invents a fact.
|
|
78
|
+
*/
|
|
79
|
+
function bodyBytes(response) {
|
|
80
|
+
for (const candidate of [response?.content?.size, response?.bodySize]) {
|
|
81
|
+
const number = Number(candidate);
|
|
82
|
+
if (Number.isFinite(number) && number >= 0) return Math.trunc(number);
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** A header's value, case-insensitively, or null. */
|
|
88
|
+
function headerValue(headers, name) {
|
|
89
|
+
if (!Array.isArray(headers)) return null;
|
|
90
|
+
const wanted = name.toLowerCase();
|
|
91
|
+
const found = headers.find((header) => String(header?.name ?? '').toLowerCase() === wanted);
|
|
92
|
+
return found ? String(found.value ?? '') : null;
|
|
93
|
+
}
|
|
94
|
+
|
|
61
95
|
/** A status that is absent or unreadable is 0 — "no response", which is a failure. */
|
|
62
96
|
function toInt(value) {
|
|
63
97
|
const number = Number(value);
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
// Turning a HAR into the report's `network` block.
|
|
2
|
+
//
|
|
3
|
+
// ## Why this exists
|
|
4
|
+
//
|
|
5
|
+
// `qa-explore` claimed an API dimension and backed it with forty-eight lines of prose
|
|
6
|
+
// telling the model what to look for. The model then counted requests by eye, decided
|
|
7
|
+
// which were slow, and judged which were duplicates — three things a model is bad at
|
|
8
|
+
// and a parser is perfect at. The pack's founding rule is that deterministic code owns
|
|
9
|
+
// facts and the model owns explanation; for this dimension the model owned both.
|
|
10
|
+
//
|
|
11
|
+
// So the counting moves here. What comes out is the `network` block of the report
|
|
12
|
+
// contract, ready to validate: totals, per-endpoint timings and sizes, and a flag on
|
|
13
|
+
// each endpoint saying what is wrong with it. The model's job becomes what it is
|
|
14
|
+
// actually good at — deciding whether a duplicate request *matters* to this product,
|
|
15
|
+
// and writing the finding.
|
|
16
|
+
//
|
|
17
|
+
// ## Facts, and inferences that are labelled as such
|
|
18
|
+
//
|
|
19
|
+
// Two different kinds of claim come out of a HAR, and conflating them is how a report
|
|
20
|
+
// starts asserting things it cannot support:
|
|
21
|
+
//
|
|
22
|
+
// fact count, status, duration, byte size, start offset, header presence.
|
|
23
|
+
// Read directly. If the HAR does not record it, the answer is null, never
|
|
24
|
+
// a plausible zero.
|
|
25
|
+
// inference duplicate, polling, n+1, uncached. Derived by a stated rule, over a
|
|
26
|
+
// stated threshold, and every one is reported with the evidence that
|
|
27
|
+
// triggered it so a reader can disagree.
|
|
28
|
+
//
|
|
29
|
+
// Nothing here decides severity. A duplicated analytics beacon and a duplicated payment
|
|
30
|
+
// request are the same shape in a HAR and wildly different findings, and only something
|
|
31
|
+
// that understands the product can tell them apart.
|
|
32
|
+
|
|
33
|
+
import { parseHar, parseHarData } from './har.mjs';
|
|
34
|
+
|
|
35
|
+
/** Defaults, all overridable, all reported back in the output so a reader sees them. */
|
|
36
|
+
export const DEFAULTS = Object.freeze({
|
|
37
|
+
slowMs: 1000,
|
|
38
|
+
// 512 KB. Below this a payload is rarely the thing worth reporting; above it, on a
|
|
39
|
+
// mobile connection, it is.
|
|
40
|
+
largeBytes: 512 * 1024,
|
|
41
|
+
// Three or more identical requests at a regular cadence reads as polling rather than
|
|
42
|
+
// as a burst of duplicates. Two is a double-submit; ten evenly spaced is a timer.
|
|
43
|
+
pollingMinimum: 3,
|
|
44
|
+
// Requests to the same path shape within this window look like one screen's worth of
|
|
45
|
+
// work, which is what makes an N+1 visible rather than just "a busy session".
|
|
46
|
+
nPlusOneWindowMs: 2000,
|
|
47
|
+
nPlusOneMinimum: 4,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/** The path portion of a URL, without query or fragment. */
|
|
51
|
+
function pathOf(url) {
|
|
52
|
+
const text = String(url ?? '');
|
|
53
|
+
const withoutScheme = text.includes('://') ? text.slice(text.indexOf('://') + 3) : text;
|
|
54
|
+
const slash = withoutScheme.indexOf('/');
|
|
55
|
+
const rest = slash === -1 ? '/' : withoutScheme.slice(slash);
|
|
56
|
+
return rest.split('?')[0].split('#')[0] || '/';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* A path with its identifier segments replaced, so `/users/41` and `/users/42` collapse.
|
|
61
|
+
*
|
|
62
|
+
* This is what makes an N+1 detectable. The rule is deliberately narrow — numeric
|
|
63
|
+
* segments, UUIDs, and long hex strings — because a looser one collapses genuinely
|
|
64
|
+
* different endpoints and reports an N+1 that is not there.
|
|
65
|
+
*/
|
|
66
|
+
export function templatePath(url) {
|
|
67
|
+
return pathOf(url)
|
|
68
|
+
.split('/')
|
|
69
|
+
.map((segment) => {
|
|
70
|
+
if (/^\d+$/.test(segment)) return ':id';
|
|
71
|
+
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(segment)) return ':uuid';
|
|
72
|
+
if (/^[0-9a-f]{16,}$/i.test(segment)) return ':hash';
|
|
73
|
+
return segment;
|
|
74
|
+
})
|
|
75
|
+
.join('/');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Milliseconds from the first request, or null when the HAR has no timestamps. */
|
|
79
|
+
function startOffsets(entries) {
|
|
80
|
+
const times = entries.map((entry) => {
|
|
81
|
+
const parsed = entry.startedAt ? Date.parse(entry.startedAt) : Number.NaN;
|
|
82
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
83
|
+
});
|
|
84
|
+
const known = times.filter((value) => value !== null);
|
|
85
|
+
if (known.length === 0) return times.map(() => null);
|
|
86
|
+
const origin = Math.min(...known);
|
|
87
|
+
return times.map((value) => (value === null ? null : value - origin));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Is this response cacheable and uncached?
|
|
92
|
+
*
|
|
93
|
+
* Only asked of static assets, and only when the server said nothing at all. A 200 for
|
|
94
|
+
* a script with neither `Cache-Control` nor `ETag` is a real finding; the same absence
|
|
95
|
+
* on a JSON API response usually is not, because that response is often meant to be
|
|
96
|
+
* fresh every time. Guessing on the second produces noise that trains a reader to
|
|
97
|
+
* ignore the whole section.
|
|
98
|
+
*/
|
|
99
|
+
function looksUncached(entry) {
|
|
100
|
+
if (entry.status !== 200) return false;
|
|
101
|
+
if (entry.cacheControl || entry.etag) return false;
|
|
102
|
+
const path = pathOf(entry.url).toLowerCase();
|
|
103
|
+
return /\.(js|mjs|css|png|jpe?g|gif|webp|avif|svg|woff2?|ttf|eot|ico)$/.test(path);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Evenly spaced repeats read as a timer rather than as a burst. */
|
|
107
|
+
function looksLikePolling(offsets, minimum) {
|
|
108
|
+
const known = offsets.filter((value) => value !== null).sort((a, b) => a - b);
|
|
109
|
+
if (known.length < minimum) return false;
|
|
110
|
+
const gaps = known.slice(1).map((value, index) => value - known[index]);
|
|
111
|
+
if (gaps.length < 2) return false;
|
|
112
|
+
const mean = gaps.reduce((sum, gap) => sum + gap, 0) / gaps.length;
|
|
113
|
+
if (mean < 250) return false; // a burst, not a poll
|
|
114
|
+
// Within 35% of the mean gap, every time: a scheduler, not a user.
|
|
115
|
+
return gaps.every((gap) => Math.abs(gap - mean) <= mean * 0.35);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Build the contract's `network` block from parsed HAR entries.
|
|
120
|
+
*
|
|
121
|
+
* Exported separately from the file-reading path so a caller that already has entries —
|
|
122
|
+
* a browser adapter capturing them live, say — can use the same analysis without
|
|
123
|
+
* writing a HAR to disk first.
|
|
124
|
+
*/
|
|
125
|
+
export function analyzeEntries(entries, options = {}) {
|
|
126
|
+
const config = { ...DEFAULTS, ...options };
|
|
127
|
+
const offsets = startOffsets(entries);
|
|
128
|
+
const enriched = entries.map((entry, index) => ({ ...entry, startedMs: offsets[index] }));
|
|
129
|
+
|
|
130
|
+
// Group by what actually identifies a call: the method and the full URL. Two GETs of
|
|
131
|
+
// the same path with different query strings are different requests, and collapsing
|
|
132
|
+
// them would report a duplicate that is not one.
|
|
133
|
+
const groups = new Map();
|
|
134
|
+
for (const entry of enriched) {
|
|
135
|
+
const key = `${entry.method} ${entry.url}`;
|
|
136
|
+
if (!groups.has(key)) groups.set(key, []);
|
|
137
|
+
groups.get(key).push(entry);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// N+1: many calls to one path *shape* inside a short window.
|
|
141
|
+
const byTemplate = new Map();
|
|
142
|
+
for (const entry of enriched) {
|
|
143
|
+
const key = `${entry.method} ${templatePath(entry.url)}`;
|
|
144
|
+
if (!byTemplate.has(key)) byTemplate.set(key, []);
|
|
145
|
+
byTemplate.get(key).push(entry);
|
|
146
|
+
}
|
|
147
|
+
const nPlusOne = new Set();
|
|
148
|
+
for (const [key, group] of byTemplate) {
|
|
149
|
+
if (group.length < config.nPlusOneMinimum) continue;
|
|
150
|
+
// Distinct URLs, or it is a duplicate rather than an N+1.
|
|
151
|
+
if (new Set(group.map((entry) => entry.url)).size < config.nPlusOneMinimum) continue;
|
|
152
|
+
const known = group.map((entry) => entry.startedMs).filter((value) => value !== null);
|
|
153
|
+
if (known.length > 1 && Math.max(...known) - Math.min(...known) > config.nPlusOneWindowMs) continue;
|
|
154
|
+
nPlusOne.add(key);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const endpoints = [];
|
|
158
|
+
for (const [, group] of groups) {
|
|
159
|
+
const first = group[0];
|
|
160
|
+
const durations = group.map((entry) => entry.durationMs).filter((value) => Number.isFinite(value));
|
|
161
|
+
const sizes = group.map((entry) => entry.bytes).filter((value) => value !== null);
|
|
162
|
+
const worstStatus = group.reduce(
|
|
163
|
+
(worst, entry) => (statusRank(entry.status) > statusRank(worst) ? entry.status : worst),
|
|
164
|
+
first.status,
|
|
165
|
+
);
|
|
166
|
+
const slowest = durations.length > 0 ? Math.max(...durations) : 0;
|
|
167
|
+
const bytes = sizes.length > 0 ? Math.max(...sizes) : null;
|
|
168
|
+
const templateKey = `${first.method} ${templatePath(first.url)}`;
|
|
169
|
+
|
|
170
|
+
// One flag per endpoint, worst first: a failing request that is also slow is
|
|
171
|
+
// reported as failing, because that is what the reader must act on.
|
|
172
|
+
let issue;
|
|
173
|
+
if (worstStatus === 0 || worstStatus >= 400) issue = 'failed';
|
|
174
|
+
else if (slowest >= config.slowMs) issue = 'slow';
|
|
175
|
+
else if (looksLikePolling(group.map((entry) => entry.startedMs), config.pollingMinimum)) issue = 'polling';
|
|
176
|
+
else if (group.length > 1) issue = 'duplicate';
|
|
177
|
+
else if (nPlusOne.has(templateKey)) issue = 'n-plus-one';
|
|
178
|
+
else if (bytes !== null && bytes >= config.largeBytes) issue = 'large-payload';
|
|
179
|
+
else if (looksUncached(first)) issue = 'uncached';
|
|
180
|
+
|
|
181
|
+
endpoints.push({
|
|
182
|
+
method: first.method,
|
|
183
|
+
url: first.url,
|
|
184
|
+
...(Number.isFinite(worstStatus) ? { status: worstStatus } : {}),
|
|
185
|
+
durationMs: slowest,
|
|
186
|
+
...(bytes !== null ? { bytes } : {}),
|
|
187
|
+
count: group.length,
|
|
188
|
+
...(first.startedMs !== null ? { startedMs: first.startedMs } : {}),
|
|
189
|
+
...(issue ? { issue } : {}),
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Worst first, then slowest: the order a reader wants without sorting anything.
|
|
194
|
+
endpoints.sort(
|
|
195
|
+
(a, b) => issueRank(a.issue) - issueRank(b.issue) || (b.durationMs ?? 0) - (a.durationMs ?? 0),
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
const totalBytes = enriched.reduce((sum, entry) => sum + (entry.bytes ?? 0), 0);
|
|
199
|
+
const anySizes = enriched.some((entry) => entry.bytes !== null);
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
totalRequests: enriched.length,
|
|
203
|
+
failedRequests: enriched.filter((entry) => entry.status === 0 || entry.status >= 400).length,
|
|
204
|
+
slowRequests: enriched.filter((entry) => entry.durationMs >= config.slowMs).length,
|
|
205
|
+
// Requests beyond the first for each identical call — the count of *wasted* calls,
|
|
206
|
+
// not the count of endpoints that were repeated.
|
|
207
|
+
duplicateRequests: [...groups.values()].reduce((sum, group) => sum + (group.length - 1), 0),
|
|
208
|
+
...(anySizes ? { totalBytes } : {}),
|
|
209
|
+
slowThresholdMs: config.slowMs,
|
|
210
|
+
endpoints,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function statusRank(status) {
|
|
215
|
+
if (status === 0) return 3;
|
|
216
|
+
if (status >= 500) return 2;
|
|
217
|
+
if (status >= 400) return 1;
|
|
218
|
+
return 0;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const ISSUE_ORDER = ['failed', 'slow', 'n-plus-one', 'duplicate', 'polling', 'large-payload', 'uncached'];
|
|
222
|
+
function issueRank(issue) {
|
|
223
|
+
const index = ISSUE_ORDER.indexOf(issue);
|
|
224
|
+
return index === -1 ? ISSUE_ORDER.length : index;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** Analyze a HAR file on disk. Returns the contract's `network` block. */
|
|
228
|
+
export function analyzeHar(path, options = {}) {
|
|
229
|
+
const parsed = parseHar(path, { slowMs: options.slowMs ?? DEFAULTS.slowMs });
|
|
230
|
+
return analyzeEntries(parsed.entries, options);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Analyze already-parsed HAR JSON, for a caller holding the document in memory. */
|
|
234
|
+
export function analyzeHarData(data, options = {}) {
|
|
235
|
+
const parsed = parseHarData(data, { slowMs: options.slowMs ?? DEFAULTS.slowMs });
|
|
236
|
+
return analyzeEntries(parsed.entries, options);
|
|
237
|
+
}
|