suitevitals 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +169 -2
- package/dist/adapters/dispatch.js +19 -0
- package/dist/adapters/junit.js +176 -0
- package/dist/adapters/mochawesome.js +78 -0
- package/dist/adapters/playwright-blob.js +146 -0
- package/dist/artifacts/extract.js +93 -0
- package/dist/artifacts/fetch.js +167 -0
- package/dist/cli.js +55 -0
- package/dist/collect/config.js +155 -0
- package/dist/collect/filter.js +45 -0
- package/dist/collect/matrix.js +69 -0
- package/dist/collect/parse.js +89 -0
- package/dist/collect/rerun.js +86 -0
- package/dist/collect/watermark.js +28 -0
- package/dist/collect/window.js +28 -0
- package/dist/colors.js +35 -0
- package/dist/commands/collect.js +466 -0
- package/dist/commands/compact.js +72 -0
- package/dist/commands/dashboard.js +116 -0
- package/dist/commands/doctor.js +110 -0
- package/dist/commands/flags.js +21 -0
- package/dist/commands/prune.js +87 -0
- package/dist/commands/repo-flags.js +23 -0
- package/dist/commands/report.js +138 -0
- package/dist/doctor/diagnose.js +171 -0
- package/dist/events/error.js +94 -0
- package/dist/events/event.js +0 -0
- package/dist/github/artifacts.js +72 -0
- package/dist/github/auth.js +47 -0
- package/dist/github/client.js +62 -0
- package/dist/github/contents.js +74 -0
- package/dist/github/repo.js +26 -0
- package/dist/github/runs.js +129 -0
- package/dist/report/aggregate.js +0 -0
- package/dist/report/dashboard.js +187 -0
- package/dist/report/rank.js +46 -0
- package/dist/report/read.js +95 -0
- package/dist/report/render.js +77 -0
- package/dist/report/score.js +50 -0
- package/dist/state/lock.js +134 -0
- package/dist/state/state.js +48 -0
- package/dist/store/compact.js +119 -0
- package/dist/store/compress.js +33 -0
- package/dist/store/dedup.js +0 -0
- package/dist/store/durable.js +40 -0
- package/dist/store/ndjson.js +125 -0
- package/dist/store/partition.js +16 -0
- package/dist/store/prune.js +51 -0
- package/package.json +62 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Adrian Jiga
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,170 @@
|
|
|
1
|
-
#
|
|
1
|
+
# suitevitals
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://github.com/adrianjiga/TestSuiteVitals/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/suitevitals)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Flaky-test reports for GitHub Actions, built from the test-report artifacts your CI already
|
|
8
|
+
uploads.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
npx suitevitals collect --repo owner/name
|
|
12
|
+
npx suitevitals report --repo owner/name
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## The problem
|
|
16
|
+
|
|
17
|
+
Flaky tests are easy to notice and hard to measure:
|
|
18
|
+
|
|
19
|
+
- a test fails, passes on retry, and the build goes green, so nobody records that it failed;
|
|
20
|
+
- the evidence sits in CI artifacts that expire after a few weeks and that nobody reads;
|
|
21
|
+
- the tools that track flakiness usually need a server, an account, or a change to how the
|
|
22
|
+
tests run.
|
|
23
|
+
|
|
24
|
+
suitevitals reads the artifacts you already have, keeps the results as plain files you own, and
|
|
25
|
+
tells you which tests are flaky, how badly, and where. No server and no changes to your tests.
|
|
26
|
+
|
|
27
|
+
## What it does
|
|
28
|
+
|
|
29
|
+
1. **Finds the reports** in your GitHub Actions runs: artifacts with report-like names first,
|
|
30
|
+
and if a run has none, it looks inside the other artifacts too.
|
|
31
|
+
2. **Reads JUnit XML, Playwright blob reports and Mochawesome JSON**, one record per test per
|
|
32
|
+
run per matrix cell, with error messages redacted.
|
|
33
|
+
3. **Stores the results** as monthly NDJSON files under `data/<owner>__<repo>/`. Re-running
|
|
34
|
+
`collect` picks up where it left off and never writes a duplicate.
|
|
35
|
+
4. **Ranks the tests** by a flake score, as a terminal table, JSON or a self-contained HTML
|
|
36
|
+
dashboard.
|
|
37
|
+
|
|
38
|
+
## How tests are scored
|
|
39
|
+
|
|
40
|
+
Each test gets a score from 0 to 100. Flaky means _inconsistent_, so a test that always passes,
|
|
41
|
+
or always fails the same way (broken, not flaky), scores near zero. The score blends four
|
|
42
|
+
signals:
|
|
43
|
+
|
|
44
|
+
| Signal | Weight |
|
|
45
|
+
| ------------------------------------------------------- | ------ |
|
|
46
|
+
| Passed only after a retry | 0.45 |
|
|
47
|
+
| Both passed and failed on the same commit | 0.30 |
|
|
48
|
+
| Failed in more than one way (distinct stack signatures) | 0.15 |
|
|
49
|
+
| The workflow run was re-run | 0.10 |
|
|
50
|
+
|
|
51
|
+
Skipped runs do not count. A test with fewer than 5 runs that
|
|
52
|
+
passed or failed (`--min-runs`) is scored but set aside, since the sample is too small to trust. The
|
|
53
|
+
report shows the signals next to the score, so every ranking can be explained.
|
|
54
|
+
|
|
55
|
+
```text
|
|
56
|
+
Flaky tests · owner/repo · 3 tests
|
|
57
|
+
|
|
58
|
+
SCORE TEST RUNS RETRY% INTERMIT MODES
|
|
59
|
+
50 webTables.spec.js › should delete a record 8 100% 0/8 2
|
|
60
|
+
0 login.spec.js › remembers the session 8 0% 0/8 0
|
|
61
|
+
|
|
62
|
+
1 test(s) below the min-runs threshold hidden (--all to show).
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Commands
|
|
66
|
+
|
|
67
|
+
| Command | What it does |
|
|
68
|
+
| ----------------------- | ----------------------------------------------------------------------------------------------------------- |
|
|
69
|
+
| `suitevitals collect` | Download and parse test reports into `./data` (last 72 hours on the first run, then from where it left off) |
|
|
70
|
+
| `suitevitals report` | Rank a repo's tests by flake score (`--since`, `--branch`, `--workflow`, `--by-matrix`, `--json`) |
|
|
71
|
+
| `suitevitals dashboard` | Write the same ranking as a self-contained HTML file with a sparkline per test |
|
|
72
|
+
| `suitevitals doctor` | Check that a repo uploads parseable reports, and print the reporter and upload YAML if not |
|
|
73
|
+
| `suitevitals compact` | Compress closed months to `.ndjson.zst`; the current month stays uncompressed |
|
|
74
|
+
| `suitevitals prune` | Delete months older than `--keep-months` (`--dry-run` to preview) |
|
|
75
|
+
|
|
76
|
+
Run `suitevitals <command> --help` for every flag.
|
|
77
|
+
|
|
78
|
+
A typical first run:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
suitevitals doctor --repo owner/name # will collection work for this repo?
|
|
82
|
+
suitevitals collect --repo owner/name # writes state.json and data/ in the current directory
|
|
83
|
+
suitevitals report --repo owner/name # rank the tests
|
|
84
|
+
suitevitals dashboard --repo owner/name # writes owner__name.html
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Run the commands from the same directory, or pass the same `--data-dir` to each.
|
|
88
|
+
|
|
89
|
+
### Settings that stick
|
|
90
|
+
|
|
91
|
+
To collect several repos with the same options every time, list them in a JSON file and pass it
|
|
92
|
+
with `--config`:
|
|
93
|
+
|
|
94
|
+
```json
|
|
95
|
+
{
|
|
96
|
+
"hoursBack": 72,
|
|
97
|
+
"excludeTriggers": ["pull_request"],
|
|
98
|
+
"repos": [
|
|
99
|
+
"owner/web-app",
|
|
100
|
+
{ "repo": "owner/api", "hoursBack": 168, "workflow": "e2e-*" },
|
|
101
|
+
{ "repo": "owner/web", "excludeArtifacts": ["final-test-report"] }
|
|
102
|
+
]
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
suitevitals collect --config repos.json
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A repo's own settings override the command-line flags, which override the top-level defaults in
|
|
111
|
+
the file. [`repos.example.json`](repos.example.json) is a working example. See the
|
|
112
|
+
[CLI and config reference](docs/API-Interface-Design.md) for every key.
|
|
113
|
+
|
|
114
|
+
## Install
|
|
115
|
+
|
|
116
|
+
Needs Node.js 24 or newer.
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
npx suitevitals --help # run without installing
|
|
120
|
+
npm install -g suitevitals # or install it globally
|
|
121
|
+
npm install --save-dev suitevitals # or per project
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The command is `suitevitals` in every case.
|
|
125
|
+
|
|
126
|
+
### GitHub token
|
|
127
|
+
|
|
128
|
+
suitevitals looks for a token in this order:
|
|
129
|
+
|
|
130
|
+
1. `--token <token>`
|
|
131
|
+
2. the `GITHUB_TOKEN` environment variable
|
|
132
|
+
3. `gh auth token`, if the GitHub CLI is installed and signed in
|
|
133
|
+
|
|
134
|
+
It only reads. A fine-grained token needs **Actions: read** and **Contents: read** on the repos
|
|
135
|
+
you collect (`doctor` uses Contents). A classic token needs the `repo` scope for private repos and
|
|
136
|
+
no scope for public ones.
|
|
137
|
+
|
|
138
|
+
## Supported
|
|
139
|
+
|
|
140
|
+
- **Report formats**: JUnit XML (any `<testsuites>` or `<testsuite>` document, such as Maven
|
|
141
|
+
Surefire, TestNG or the Jest and Vitest JUnit reporters), Playwright blob reports, and Mochawesome JSON (Cypress, Mocha).
|
|
142
|
+
- **CI**: GitHub Actions workflow artifacts.
|
|
143
|
+
- **Matrix builds**: Playwright reports carry their own project name. For the other formats, the
|
|
144
|
+
cell comes from the artifact name, and `--matrix-from-artifact` can split it into named
|
|
145
|
+
dimensions such as browser and viewport.
|
|
146
|
+
- **`doctor`** recognizes Playwright, Cypress, Vitest, Jest, Mocha and Java (Maven or Gradle)
|
|
147
|
+
projects, including monorepos.
|
|
148
|
+
- **Output**: a terminal table, JSON, and a self-contained HTML dashboard with no runtime
|
|
149
|
+
dependencies.
|
|
150
|
+
|
|
151
|
+
## Learn more
|
|
152
|
+
|
|
153
|
+
- [Troubleshooting](docs/troubleshooting.md): no reports found, missing tests, token and lock
|
|
154
|
+
errors.
|
|
155
|
+
- [CLI and config reference](docs/API-Interface-Design.md): every flag, its default, and the config
|
|
156
|
+
file.
|
|
157
|
+
- [Stored data and dashboard reference](docs/output-reference.md): what each record and file
|
|
158
|
+
contains.
|
|
159
|
+
- [Architecture](docs/Architecture.md), [technical design](docs/Technical-Design.md) and
|
|
160
|
+
[design decisions](docs/decisions/).
|
|
161
|
+
- [Changelog](CHANGELOG.md) and [releases](https://github.com/adrianjiga/TestSuiteVitals/releases).
|
|
162
|
+
|
|
163
|
+
## Contributing
|
|
164
|
+
|
|
165
|
+
Bugs and ideas go on the [issue tracker](https://github.com/adrianjiga/TestSuiteVitals/issues).
|
|
166
|
+
For the development setup and how changes get merged, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { JUNIT_FRAMEWORK, looksLikeJUnit, parseJUnit } from "./junit.js";
|
|
2
|
+
import { looksLikeMochawesome, MOCHAWESOME_FRAMEWORK, parseMochawesome } from "./mochawesome.js";
|
|
3
|
+
import { looksLikePlaywrightBlob, PLAYWRIGHT_FRAMEWORK, parsePlaywrightBlob, } from "./playwright-blob.js";
|
|
4
|
+
/**
|
|
5
|
+
* Sniffs report content and parses it with the matching adapter, or returns null if no
|
|
6
|
+
* adapter recognizes it. Each adapter emits the same `ParsedTest` shape.
|
|
7
|
+
*/
|
|
8
|
+
export function detectAndParse(content, options = {}) {
|
|
9
|
+
if (looksLikeJUnit(content)) {
|
|
10
|
+
return { framework: JUNIT_FRAMEWORK, tests: parseJUnit(content, options) };
|
|
11
|
+
}
|
|
12
|
+
if (looksLikePlaywrightBlob(content)) {
|
|
13
|
+
return { framework: PLAYWRIGHT_FRAMEWORK, tests: parsePlaywrightBlob(content, options) };
|
|
14
|
+
}
|
|
15
|
+
if (looksLikeMochawesome(content)) {
|
|
16
|
+
return { framework: MOCHAWESOME_FRAMEWORK, tests: parseMochawesome(content, options) };
|
|
17
|
+
}
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { XMLParser } from "fast-xml-parser";
|
|
2
|
+
import { makeTestError } from "../events/error.js";
|
|
3
|
+
export const JUNIT_FRAMEWORK = "junit";
|
|
4
|
+
/**
|
|
5
|
+
* Cheap head-anchored sniff: after an optional BOM, XML prolog, comments, and DOCTYPE, the
|
|
6
|
+
* document's root element must be `<testsuites>` or `<testsuite>`. Anchoring matters: dispatch
|
|
7
|
+
* tries JUnit first, so a JSON report that merely *mentions* the tag mid-document (e.g. an error
|
|
8
|
+
* message quoting XML) must not be routed here — parsing it as XML yields zero tests and silently
|
|
9
|
+
* loses the real report.
|
|
10
|
+
*/
|
|
11
|
+
const JUNIT_ROOT = /^\uFEFF?\s*(?:<\?xml[^>]*>\s*)?(?:<!--[\s\S]*?-->\s*)*(?:<!DOCTYPE[^[>]*(?:\[[^\]]*\])?[^>]*>\s*)?<testsuites?[\s/>]/;
|
|
12
|
+
export function looksLikeJUnit(content) {
|
|
13
|
+
return JUNIT_ROOT.test(content.slice(0, 4096));
|
|
14
|
+
}
|
|
15
|
+
const XML_OPTIONS = {
|
|
16
|
+
ignoreAttributes: false,
|
|
17
|
+
attributeNamePrefix: "@_",
|
|
18
|
+
textNodeName: "#text",
|
|
19
|
+
parseTagValue: false,
|
|
20
|
+
trimValues: true,
|
|
21
|
+
isArray: (name) => [
|
|
22
|
+
"testsuite",
|
|
23
|
+
"testcase",
|
|
24
|
+
"failure",
|
|
25
|
+
"error",
|
|
26
|
+
"flakyFailure",
|
|
27
|
+
"flakyError",
|
|
28
|
+
"rerunFailure",
|
|
29
|
+
"rerunError",
|
|
30
|
+
].includes(name),
|
|
31
|
+
};
|
|
32
|
+
/** Surefire's records of failed runs before a retry passed. */
|
|
33
|
+
const FLAKY_TAGS = ["flakyFailure", "flakyError"];
|
|
34
|
+
/** Surefire's records of retries after the first failure, when every retry also failed. */
|
|
35
|
+
const RERUN_TAGS = ["rerunFailure", "rerunError"];
|
|
36
|
+
const parser = new XMLParser(XML_OPTIONS);
|
|
37
|
+
/**
|
|
38
|
+
* For DOCTYPE-bearing documents only: fast-xml-parser expands `<!ENTITY>` definitions by default,
|
|
39
|
+
* and although it blocks *recursive* entities, one long value referenced many times still expands
|
|
40
|
+
* quadratically — a ~50 MB report inflating to gigabytes in memory (F1). No real JUnit emitter
|
|
41
|
+
* writes a DOCTYPE, so those documents are parsed with entity processing off (references stay as
|
|
42
|
+
* literal `&name;` text) rather than being dropped.
|
|
43
|
+
*/
|
|
44
|
+
const noEntitiesParser = new XMLParser({ ...XML_OPTIONS, processEntities: false });
|
|
45
|
+
const HAS_DOCTYPE = /<!DOCTYPE/i;
|
|
46
|
+
/**
|
|
47
|
+
* Parses a JUnit XML report into framework-agnostic tests. A test yields one attempt, unless
|
|
48
|
+
* Surefire's `rerunFailingTestsCount` retried it: then each recorded retry becomes an attempt.
|
|
49
|
+
*/
|
|
50
|
+
export function parseJUnit(xml, options = {}) {
|
|
51
|
+
const activeParser = HAS_DOCTYPE.test(xml) ? noEntitiesParser : parser;
|
|
52
|
+
const root = activeParser.parse(xml);
|
|
53
|
+
const suites = collectSuites(root);
|
|
54
|
+
const tests = [];
|
|
55
|
+
for (const { suite, file } of suites) {
|
|
56
|
+
const suiteName = str(suite["@_name"]);
|
|
57
|
+
for (const testcase of asArray(suite.testcase)) {
|
|
58
|
+
tests.push(parseTestcase(testcase, file, suiteName, options));
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return tests;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Gathers every `<testsuite>` at any depth, under a `<testsuites>` root or a lone `<testsuite>`.
|
|
65
|
+
* An inner suite without its own `file` inherits the enclosing suite's.
|
|
66
|
+
*/
|
|
67
|
+
function collectSuites(node, inheritedFile) {
|
|
68
|
+
const found = [];
|
|
69
|
+
for (const container of asArray(node.testsuites)) {
|
|
70
|
+
found.push(...collectSuites(container, inheritedFile));
|
|
71
|
+
}
|
|
72
|
+
for (const suite of asArray(node.testsuite)) {
|
|
73
|
+
const file = str(suite["@_file"]) ?? inheritedFile;
|
|
74
|
+
found.push({ suite, file }, ...collectSuites(suite, file));
|
|
75
|
+
}
|
|
76
|
+
return found;
|
|
77
|
+
}
|
|
78
|
+
function parseTestcase(tc, suiteFile, suiteName, options) {
|
|
79
|
+
const name = str(tc["@_name"]) ?? "(unnamed test)";
|
|
80
|
+
const classname = str(tc["@_classname"]);
|
|
81
|
+
/**
|
|
82
|
+
* Resolution order matters for test identity (D5: `id = hash(file + title_path)`).
|
|
83
|
+
*
|
|
84
|
+
* `classname` is preferred over `<testsuite name>` because the suite name is a *run
|
|
85
|
+
* configuration label*, not a location — and the same test emits a different one depending
|
|
86
|
+
* on how the run was invoked. TestNG/Surefire is the clearest case: one test yields
|
|
87
|
+
* `name="TestSuite"` under `testng.xml`, `name="UI Tests"` under `testng-ui.xml`, and a
|
|
88
|
+
* class-named suite element when Surefire emits per-class files. Ranking that ahead of
|
|
89
|
+
* `classname` minted three identities for one test and split its history three ways.
|
|
90
|
+
* pytest has the same shape at lower stakes (`<testsuite name="pytest">` for the whole run).
|
|
91
|
+
*
|
|
92
|
+
* Genuine path attributes still win: `<testcase file>` then `<testsuite file>` are real
|
|
93
|
+
* locations when an emitter provides them. The suite name survives last, so a report with
|
|
94
|
+
* neither a path nor a classname is unaffected.
|
|
95
|
+
*/
|
|
96
|
+
const file = str(tc["@_file"]) ?? suiteFile ?? classname ?? suiteName ?? "unknown";
|
|
97
|
+
const titlePath = classname && classname !== file ? [classname, name] : [name];
|
|
98
|
+
const durationMs = Math.round((Number(tc["@_time"]) || 0) * 1000);
|
|
99
|
+
const status = testcaseStatus(tc);
|
|
100
|
+
const attempts = retryAttempts(tc, status, options);
|
|
101
|
+
const final = { n: attempts.length + 1, status, duration_ms: durationMs };
|
|
102
|
+
if (status === "failed") {
|
|
103
|
+
const error = errorFrom(asArray(tc.failure)[0] ?? asArray(tc.error)[0], options);
|
|
104
|
+
if (error) {
|
|
105
|
+
final.error = error;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
file,
|
|
110
|
+
title_path: titlePath,
|
|
111
|
+
status,
|
|
112
|
+
flaky: status === "passed" && attempts.length > 0,
|
|
113
|
+
duration_ms: durationMs,
|
|
114
|
+
attempts: status === "failed" ? failedThenReruns(final, tc, options) : [...attempts, final],
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/** Failed runs that preceded a passing retry (`<flakyFailure>`/`<flakyError>`). */
|
|
118
|
+
function retryAttempts(tc, status, options) {
|
|
119
|
+
if (status !== "passed") {
|
|
120
|
+
return [];
|
|
121
|
+
}
|
|
122
|
+
return FLAKY_TAGS.flatMap((tag) => asArray(tc[tag])).map((node, i) => failedAttempt(i + 1, node, options));
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* A test that failed every retry: the `<failure>`/`<error>` is the first run, and each
|
|
126
|
+
* `<rerunFailure>`/`<rerunError>` is a retry after it.
|
|
127
|
+
*/
|
|
128
|
+
function failedThenReruns(first, tc, options) {
|
|
129
|
+
const reruns = RERUN_TAGS.flatMap((tag) => asArray(tc[tag]));
|
|
130
|
+
return [{ ...first, n: 1 }, ...reruns.map((node, i) => failedAttempt(i + 2, node, options))];
|
|
131
|
+
}
|
|
132
|
+
function failedAttempt(n, node, options) {
|
|
133
|
+
const attempt = { n, status: "failed", duration_ms: 0 };
|
|
134
|
+
const error = errorFrom(node, options);
|
|
135
|
+
if (error) {
|
|
136
|
+
attempt.error = error;
|
|
137
|
+
}
|
|
138
|
+
return attempt;
|
|
139
|
+
}
|
|
140
|
+
function testcaseStatus(tc) {
|
|
141
|
+
if ("skipped" in tc) {
|
|
142
|
+
return "skipped";
|
|
143
|
+
}
|
|
144
|
+
const failures = [tc.failure, tc.error, ...RERUN_TAGS.map((tag) => tc[tag])];
|
|
145
|
+
if (failures.some((value) => asArray(value).length > 0)) {
|
|
146
|
+
return "failed";
|
|
147
|
+
}
|
|
148
|
+
return "passed";
|
|
149
|
+
}
|
|
150
|
+
function errorFrom(node, options) {
|
|
151
|
+
if (node === undefined) {
|
|
152
|
+
return undefined;
|
|
153
|
+
}
|
|
154
|
+
if (typeof node === "string") {
|
|
155
|
+
return makeTestError({ message: node }, options);
|
|
156
|
+
}
|
|
157
|
+
const detail = node;
|
|
158
|
+
const message = str(detail["@_message"]) ?? str(detail["#text"]) ?? "Test failed";
|
|
159
|
+
const stack = str(detail["#text"]) ?? "";
|
|
160
|
+
return makeTestError({ message, stack }, options);
|
|
161
|
+
}
|
|
162
|
+
function asArray(value) {
|
|
163
|
+
if (Array.isArray(value)) {
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
return value === undefined ? [] : [value];
|
|
167
|
+
}
|
|
168
|
+
function str(value) {
|
|
169
|
+
if (typeof value === "string") {
|
|
170
|
+
return value;
|
|
171
|
+
}
|
|
172
|
+
if (typeof value === "number") {
|
|
173
|
+
return String(value);
|
|
174
|
+
}
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { makeTestError } from "../events/error.js";
|
|
2
|
+
export const MOCHAWESOME_FRAMEWORK = "mochawesome";
|
|
3
|
+
/**
|
|
4
|
+
* Cheap content sniff: a mochawesome report is a single JSON object whose `stats`
|
|
5
|
+
* carry the distinctive `passPercent` + `testsRegistered` keys. Both appear at the
|
|
6
|
+
* very start of the document, so a head slice is enough.
|
|
7
|
+
*/
|
|
8
|
+
export function looksLikeMochawesome(content) {
|
|
9
|
+
const head = content.slice(0, 2000);
|
|
10
|
+
return /"passPercent"\s*:/.test(head) && /"testsRegistered"\s*:/.test(head);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Parses a mochawesome JSON report (Cypress/Mocha) into framework-agnostic tests.
|
|
14
|
+
* Each `results[]` entry is a per-spec root suite holding a tree of `suites`/`tests`;
|
|
15
|
+
* the walk accumulates describe titles into the title path, skipping the root node's
|
|
16
|
+
* title (the spec filename, not a describe). Mochawesome is flat per test — one attempt,
|
|
17
|
+
* never flaky; the richer `attempts[]` comes from Playwright and Surefire retries.
|
|
18
|
+
*/
|
|
19
|
+
export function parseMochawesome(content, options = {}) {
|
|
20
|
+
const report = JSON.parse(content);
|
|
21
|
+
const tests = [];
|
|
22
|
+
for (const result of report.results ?? []) {
|
|
23
|
+
const file = result.file || result.fullFile || "unknown";
|
|
24
|
+
walkSuite(result, file, [], tests, options);
|
|
25
|
+
}
|
|
26
|
+
return tests;
|
|
27
|
+
}
|
|
28
|
+
/** Recursively walks a suite tree, accumulating describe titles into the title path. */
|
|
29
|
+
function walkSuite(suite, file, titlePath, tests, options) {
|
|
30
|
+
for (const test of suite.tests ?? []) {
|
|
31
|
+
tests.push(toTest(test, file, titlePath, options));
|
|
32
|
+
}
|
|
33
|
+
for (const child of suite.suites ?? []) {
|
|
34
|
+
// The root node's title is the spec filename, not a real describe — skip it.
|
|
35
|
+
const childPath = child.root || !child.title ? titlePath : [...titlePath, child.title];
|
|
36
|
+
walkSuite(child, file, childPath, tests, options);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function toTest(test, file, titlePath, options) {
|
|
40
|
+
const status = mapStatus(test);
|
|
41
|
+
const durationMs = test.duration ?? 0;
|
|
42
|
+
const attempt = { n: 1, status, duration_ms: durationMs };
|
|
43
|
+
if (status === "failed") {
|
|
44
|
+
const error = toError(test.err, options);
|
|
45
|
+
if (error) {
|
|
46
|
+
attempt.error = error;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
file,
|
|
51
|
+
title_path: [...titlePath, test.title ?? "(unnamed test)"],
|
|
52
|
+
status,
|
|
53
|
+
flaky: false,
|
|
54
|
+
duration_ms: durationMs,
|
|
55
|
+
attempts: [attempt],
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Maps mochawesome's redundant status fields; a pending/skipped test is never a real status. */
|
|
59
|
+
function mapStatus(test) {
|
|
60
|
+
if (test.pending || test.skipped) {
|
|
61
|
+
return "skipped";
|
|
62
|
+
}
|
|
63
|
+
if (test.state === "failed" || test.fail) {
|
|
64
|
+
return "failed";
|
|
65
|
+
}
|
|
66
|
+
if (test.state === "passed" || test.pass) {
|
|
67
|
+
return "passed";
|
|
68
|
+
}
|
|
69
|
+
return "skipped";
|
|
70
|
+
}
|
|
71
|
+
function toError(err, options) {
|
|
72
|
+
if (!err) {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
const message = err.message ?? "Test failed";
|
|
76
|
+
const stack = err.estack ?? err.stack ?? "";
|
|
77
|
+
return makeTestError({ message, stack }, options);
|
|
78
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { makeTestError } from "../events/error.js";
|
|
2
|
+
import { deriveFlaky, deriveStatus, } from "../events/event.js";
|
|
3
|
+
export const PLAYWRIGHT_FRAMEWORK = "playwright";
|
|
4
|
+
/** Strips ANSI color codes that Playwright embeds in error messages. */
|
|
5
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escapes are control chars by definition.
|
|
6
|
+
const ANSI = /\[[0-9;]*m/g;
|
|
7
|
+
/**
|
|
8
|
+
* Sniffs a Playwright blob report (`report.jsonl`): a JSON-lines stream of teleReporter
|
|
9
|
+
* events whose first lines carry `"method":"on…"`.
|
|
10
|
+
*/
|
|
11
|
+
export function looksLikePlaywrightBlob(content) {
|
|
12
|
+
return /"method"\s*:\s*"on(Configure|BlobReportMetadata|Project|Begin)"/.test(content.slice(0, 1000));
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Parses a Playwright blob report into tests. The teleReporter `onProject` events carry
|
|
16
|
+
* the suite tree (file → describe → test via `entries`) and the project name (the matrix
|
|
17
|
+
* cell); `onTestEnd` events carry per-retry results, which become `attempts[]`.
|
|
18
|
+
*/
|
|
19
|
+
export function parsePlaywrightBlob(content, options = {}) {
|
|
20
|
+
const events = parseJsonl(content);
|
|
21
|
+
const tree = new Map();
|
|
22
|
+
const results = new Map();
|
|
23
|
+
for (const event of events) {
|
|
24
|
+
if (event.method === "onProject") {
|
|
25
|
+
indexProject(event.params?.project, tree);
|
|
26
|
+
}
|
|
27
|
+
else if (event.method === "onTestEnd") {
|
|
28
|
+
indexResult(event.params, results);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const tests = [];
|
|
32
|
+
for (const [testId, rawAttempts] of results) {
|
|
33
|
+
const info = tree.get(testId);
|
|
34
|
+
if (!info) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
rawAttempts.sort((a, b) => a.retry - b.retry);
|
|
38
|
+
const attempts = rawAttempts.map((raw) => toAttempt(raw, options));
|
|
39
|
+
tests.push({
|
|
40
|
+
file: info.file,
|
|
41
|
+
title_path: info.titlePath,
|
|
42
|
+
status: deriveStatus(attempts),
|
|
43
|
+
flaky: deriveFlaky(attempts),
|
|
44
|
+
duration_ms: attempts.at(-1)?.duration_ms ?? 0,
|
|
45
|
+
attempts,
|
|
46
|
+
matrix: info.matrix,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return tests;
|
|
50
|
+
}
|
|
51
|
+
function indexProject(project, tree) {
|
|
52
|
+
if (!project) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const matrix = project.name ? { project: project.name } : {};
|
|
56
|
+
for (const fileSuite of project.suites ?? []) {
|
|
57
|
+
const file = fileSuite.location?.file ?? fileSuite.title ?? "unknown";
|
|
58
|
+
for (const entry of fileSuite.entries ?? []) {
|
|
59
|
+
walk(entry, [], file, matrix, tree);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/** Recursively walks the suite tree, accumulating describe titles into the title path. */
|
|
64
|
+
function walk(node, titlePath, file, matrix, tree) {
|
|
65
|
+
const nodeFile = node.location?.file ?? file;
|
|
66
|
+
if (node.testId) {
|
|
67
|
+
tree.set(node.testId, { file: nodeFile, titlePath: [...titlePath, node.title ?? ""], matrix });
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const childPath = node.title ? [...titlePath, node.title] : titlePath;
|
|
71
|
+
for (const entry of node.entries ?? []) {
|
|
72
|
+
walk(entry, childPath, nodeFile, matrix, tree);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function indexResult(params, results) {
|
|
76
|
+
const test = params?.test;
|
|
77
|
+
const result = params?.result;
|
|
78
|
+
if (!test?.testId || !result) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const attempt = {
|
|
82
|
+
retry: result.retry ?? 0,
|
|
83
|
+
status: mapStatus(result.status, test.expectedStatus),
|
|
84
|
+
duration_ms: result.duration ?? 0,
|
|
85
|
+
errors: result.errors ?? [],
|
|
86
|
+
};
|
|
87
|
+
const bucket = results.get(test.testId);
|
|
88
|
+
if (bucket) {
|
|
89
|
+
bucket.push(attempt);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
results.set(test.testId, [attempt]);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function toAttempt(raw, options) {
|
|
96
|
+
const attempt = {
|
|
97
|
+
n: raw.retry + 1,
|
|
98
|
+
status: raw.status,
|
|
99
|
+
duration_ms: raw.duration_ms,
|
|
100
|
+
};
|
|
101
|
+
if (raw.status === "failed") {
|
|
102
|
+
const error = toError(raw.errors, options);
|
|
103
|
+
if (error) {
|
|
104
|
+
attempt.error = error;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return attempt;
|
|
108
|
+
}
|
|
109
|
+
function toError(errors, options) {
|
|
110
|
+
const first = errors[0];
|
|
111
|
+
if (!first) {
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
const message = stripAnsi(first.message ?? first.value ?? "Test failed");
|
|
115
|
+
const stack = stripAnsi(first.stack ?? "");
|
|
116
|
+
return makeTestError({ message, stack }, options);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Playwright judges a result against the test's `expectedStatus`, not in absolute terms: a
|
|
120
|
+
* `test.fail()` test that fails is passing, and one that unexpectedly passes is a failure.
|
|
121
|
+
*/
|
|
122
|
+
function mapStatus(status, expectedStatus = "passed") {
|
|
123
|
+
if (status === "skipped" || expectedStatus === "skipped") {
|
|
124
|
+
return "skipped";
|
|
125
|
+
}
|
|
126
|
+
return status === expectedStatus ? "passed" : "failed";
|
|
127
|
+
}
|
|
128
|
+
function stripAnsi(text) {
|
|
129
|
+
return text.replace(ANSI, "");
|
|
130
|
+
}
|
|
131
|
+
function parseJsonl(content) {
|
|
132
|
+
const events = [];
|
|
133
|
+
for (const line of content.split("\n")) {
|
|
134
|
+
const trimmed = line.trim();
|
|
135
|
+
if (trimmed.length === 0) {
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
events.push(JSON.parse(trimmed));
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Skip malformed lines rather than fail the whole report.
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return events;
|
|
146
|
+
}
|