tewip-ai 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/TEAM-GUIDE.md +3 -0
- package/dist/cli/tewip.js +19 -1
- package/dist/engine/autofix/fix-selector.js +93 -0
- package/dist/engine/autofix/selector-source.js +162 -0
- package/dist/engine/import/junit.js +40 -4
- package/dist/engine/import/other-formats.js +127 -0
- package/dist/engine/review/coverage.js +188 -0
- package/dist/engine/review/review-command.js +44 -0
- package/dist/engine/review/review.js +127 -0
- package/dist/engine/work/azure-linear.js +116 -0
- package/dist/engine/work/bug-command.js +20 -25
- package/dist/engine/work/github-issues.js +63 -0
- package/dist/engine/work/rally.js +84 -0
- package/dist/engine/work/tracker.js +48 -0
- package/dist/engine/work/trackers.js +81 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -166,6 +166,36 @@ that lost its markup) and **5 of 5 harmless changes ignored** (capitals, extra s
|
|
|
166
166
|
implemented as a link, reordered sections). No designer has judged the output yet, so the roadmap's
|
|
167
167
|
own metric is still open.
|
|
168
168
|
|
|
169
|
+
## More runners: Robot Framework, JMeter, k6 (new)
|
|
170
|
+
|
|
171
|
+
Tewip now reads **Robot Framework's output.xml**, **JMeter's .jtl** and a **k6 summary** as well as
|
|
172
|
+
JUnit XML, TRX, NUnit and Playwright's own reports. A load test does not fail test by test, so a
|
|
173
|
+
JMeter report becomes one line per request ("4 of 5 requests failed, an assertion failed") and a k6
|
|
174
|
+
run becomes one record per crossed threshold, with the number that crossed it.
|
|
175
|
+
|
|
176
|
+
Appium and Selenium need no importer: those suites report through Mocha, pytest or JUnit, which
|
|
177
|
+
Tewip already reads, and their own error messages ("Could not find a connected device", "stale
|
|
178
|
+
element reference") are already understood.
|
|
179
|
+
|
|
180
|
+
Each reader was checked against real reports published in public repositories, which is how three
|
|
181
|
+
defects were found before release - a Robot timestamp that is not a date, k6's numbers living under
|
|
182
|
+
`values`, and a JMeter line that read "mostly 200" when an assertion had failed.
|
|
183
|
+
|
|
184
|
+
## Before anyone reads the diff (new)
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
tewip review --diff main...HEAD
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Which tests cover this change and **how** (they import the file, they import the package that
|
|
191
|
+
exports it, or their words merely match); which of them have flaked before, so a red run is not
|
|
192
|
+
blamed on your change; what has gone wrong in these areas before; what nothing tests at all; and the
|
|
193
|
+
command to run the telling tests first. From your project's own history, so it costs nothing.
|
|
194
|
+
|
|
195
|
+
Measured on **16 real commits** from two public projects, where the developers changed source *and*
|
|
196
|
+
its test: given only the source files, Tewip **named the right test in 16 of 16, ranked it first in
|
|
197
|
+
12**.
|
|
198
|
+
|
|
169
199
|
## Which flaky tests to quarantine, and what they cost (new)
|
|
170
200
|
|
|
171
201
|
```bash
|
package/TEAM-GUIDE.md
CHANGED
|
@@ -127,6 +127,7 @@ collects environment failures in one issue, and (optionally) opens a fix pull re
|
|
|
127
127
|
| `tewip spec "<feature>"` | An epic, its user stories and acceptance criteria, grounded in your code; files them in Jira with `--project PROJ --create` | Before the work is broken down |
|
|
128
128
|
| `tewip report report.xml --gate` | The CI command for GitLab, Jenkins, CircleCI, Azure, Bitbucket and the rest: diagnoses the run, comments on the GitLab merge request, and fails the build only on failures that need a person | Your CI is not GitHub Actions |
|
|
129
129
|
| `tewip watch --cmd "npm test"` | Runs your tests when you save a file and explains each failure straight away | While you are writing code |
|
|
130
|
+
| `tewip review --diff main...HEAD` | Which tests cover your change, which have flaked before, what nothing tests, and what to run first | Before you open a pull request |
|
|
130
131
|
| `tewip flaky --days 30` | Which tests to quarantine, which to fix, what they cost in CI minutes - and when a quarantined one can come back | Monday morning |
|
|
131
132
|
| `tewip note --days 7` | The weekly quality note for a lead, from runs already diagnosed (no model calls, free) | Monday morning |
|
|
132
133
|
| `tewip fix --code report.xml` | Opt-in: proposes the smallest change to application code for a real bug, and keeps it only if the failing test passes and the whole suite still passes | "It is a real bug and I want a starting patch" |
|
|
@@ -140,6 +141,8 @@ Notes that save you time:
|
|
|
140
141
|
- `write` never touches your application code. It only adds new test files, and you review and
|
|
141
142
|
commit them yourself.
|
|
142
143
|
- Everything it writes locally goes to the `.tewip` folder in your project.
|
|
144
|
+
- Work items go to **Jira, Rally, GitHub Issues, Azure Boards or Linear** - set `TEWIP_TRACKER`, or just the credentials
|
|
145
|
+
of the one you use. GitHub Issues uses your existing `gh` login, so it needs no new secret.
|
|
143
146
|
- Jira works with **Cloud, Server and Data Center**. To read tickets from a tracker anyone can read
|
|
144
147
|
(many open-source ones), `TEWIP_JIRA_URL` alone is enough - no account. Writing needs a token.
|
|
145
148
|
- `tewip jira` shows what it *would* file and creates nothing until you add `--create`. It never
|
package/dist/cli/tewip.js
CHANGED
|
@@ -67,6 +67,12 @@ Usage:
|
|
|
67
67
|
the GitLab merge request, tells Slack or Teams only when a
|
|
68
68
|
failure needs a person, and with --gate fails the build only on
|
|
69
69
|
those failures.
|
|
70
|
+
tewip review [--diff main...HEAD] [--out review.md]
|
|
71
|
+
Before anyone reads the diff: which tests cover this change,
|
|
72
|
+
which of them have flaked before, what has gone wrong in these
|
|
73
|
+
areas, what nothing tests at all, and the command that runs the
|
|
74
|
+
telling tests first. From this project's own history, so it
|
|
75
|
+
costs nothing.
|
|
70
76
|
tewip watch [--cmd "npm test"] [--report <path>] [--paths src,tests]
|
|
71
77
|
Beside your editor: runs your tests when a file changes (or
|
|
72
78
|
watches for new reports) and explains each failure as it
|
|
@@ -92,7 +98,9 @@ Usage:
|
|
|
92
98
|
tewip --version Print the version.
|
|
93
99
|
|
|
94
100
|
Reports it reads: Playwright, pytest/unittest, Jest, Vitest, Mocha, node:test, JUnit/TestNG
|
|
95
|
-
(Maven, Gradle), Go (gotestsum), .NET (TRX or JUnit), RSpec, PHPUnit, Rust (nextest)
|
|
101
|
+
(Maven, Gradle), Go (gotestsum), .NET (TRX or JUnit), NUnit 3, RSpec, PHPUnit, Rust (nextest),
|
|
102
|
+
Robot Framework (output.xml), JMeter (.jtl) and k6 (--summary-export). Appium, Selenium and
|
|
103
|
+
WebdriverIO suites report through one of the above, and their own error messages are understood.
|
|
96
104
|
Set OPENAI_API_KEY to let the AI layer decide what the rules leave open.`;
|
|
97
105
|
const flag = (args, name) => {
|
|
98
106
|
const i = args.indexOf(`--${name}`);
|
|
@@ -248,6 +256,16 @@ async function main() {
|
|
|
248
256
|
process.exitCode = 1;
|
|
249
257
|
return;
|
|
250
258
|
}
|
|
259
|
+
case 'review': {
|
|
260
|
+
const { runReview } = await import('../engine/review/review-command.js');
|
|
261
|
+
await runReview({
|
|
262
|
+
root: process.cwd(),
|
|
263
|
+
diff: flag(args, 'diff') ?? 'main...HEAD',
|
|
264
|
+
out: flag(args, 'out'),
|
|
265
|
+
name: flag(args, 'name'),
|
|
266
|
+
});
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
251
269
|
case 'watch': {
|
|
252
270
|
const { runWatch } = await import('../engine/watch/watch-command.js');
|
|
253
271
|
await runWatch({
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// ACT + VERIFY for a drifted selector in a Cypress, Selenium or WebdriverIO test.
|
|
2
|
+
//
|
|
3
|
+
// The same loop as every other fix Tewip makes: change one thing in the test file, run that test,
|
|
4
|
+
// keep the change only if it passes, and otherwise put the file back exactly as it was. What is new
|
|
5
|
+
// is where the replacement comes from - the repository, not a page snapshot - which is why these
|
|
6
|
+
// tools can be fixed at all without a trace.
|
|
7
|
+
//
|
|
8
|
+
// It will not touch application code, will not change what the test asserts, and will not offer a
|
|
9
|
+
// replacement it cannot point at in the source or in a commit.
|
|
10
|
+
import { relative, resolve, sep } from 'node:path';
|
|
11
|
+
import { driverLocator } from '../driver-errors.js';
|
|
12
|
+
import { likeItInSource, renamedInGit, tokenOf, writeSelector } from './selector-source.js';
|
|
13
|
+
/**
|
|
14
|
+
* Where a selector is written in a test: the whole line, so the replacement reads in context.
|
|
15
|
+
* Cypress and Selenium put the selector in a plain string, so this is simpler than Playwright's
|
|
16
|
+
* chained calls - and looking for the quoted form keeps it from matching a comment about it.
|
|
17
|
+
*/
|
|
18
|
+
export function findSelectorSites(source, selector) {
|
|
19
|
+
const quoted = [`'${selector}'`, `"${selector}"`, `\`${selector}\``];
|
|
20
|
+
const sites = [];
|
|
21
|
+
source.split('\n').forEach((text, index) => {
|
|
22
|
+
if (quoted.some((q) => text.includes(q)))
|
|
23
|
+
sites.push({ line: index + 1, text });
|
|
24
|
+
});
|
|
25
|
+
return sites;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* One drifted selector, fixed from what the repository knows. `keep` false checks a suggestion
|
|
29
|
+
* without leaving it in the file, exactly as the locator fix does.
|
|
30
|
+
*/
|
|
31
|
+
export async function attemptSelectorFix({ record, diagnosis, action, specFile }, deps, { keep = true } = {}) {
|
|
32
|
+
const title = record.testId.split(' > ').at(-1) ?? record.testId;
|
|
33
|
+
const result = { testId: record.testId, specFile, status: 'not-eligible', applied: false, log: [] };
|
|
34
|
+
const stop = (status, why) => ({ ...result, status, log: [...result.log, why] });
|
|
35
|
+
if (diagnosis.category !== 'LOCATOR_DRIFT' || action.kind !== 'fix-locator') {
|
|
36
|
+
return stop('not-eligible', `only a renamed element is fixed this way; this one is ${diagnosis.category} (${action.kind})`);
|
|
37
|
+
}
|
|
38
|
+
const rel = relative(resolve(deps.allowedRoot), resolve(specFile));
|
|
39
|
+
if (rel.startsWith('..') || rel.startsWith(sep) || /^[a-zA-Z]:/.test(rel))
|
|
40
|
+
return stop('outside-sandbox', `refused: ${specFile} is outside ${deps.allowedRoot}`);
|
|
41
|
+
// What the tool said it was looking for.
|
|
42
|
+
const selector = driverLocator(record.errorMessage) ?? '';
|
|
43
|
+
if (!selector)
|
|
44
|
+
return stop('no-selector', 'the error does not say which selector was not found');
|
|
45
|
+
result.log.push(`the test looked for ${selector}`);
|
|
46
|
+
// What the line in the test really holds: Selenium prints "css selector=#x" but writes "#x".
|
|
47
|
+
const written = /^[a-z ]+=\s*/i.test(selector) ? selector.replace(/^[a-z ]+=\s*/i, '') : selector;
|
|
48
|
+
const token = tokenOf(selector);
|
|
49
|
+
if (!token) {
|
|
50
|
+
return stop('not-identifiable', `${selector} is not an id, class or data attribute, so there is nothing in the source to compare it against. A person should look at the page.`);
|
|
51
|
+
}
|
|
52
|
+
// Git first: somebody renamed it, and the commit says to what. The source is the weaker answer.
|
|
53
|
+
const found = renamedInGit(deps.projectRoot, token) ?? likeItInSource(deps.projectRoot, token, deps.sources);
|
|
54
|
+
if (!found) {
|
|
55
|
+
return stop('no-replacement', `nothing in the source or in the history looks like "${token.value}" renamed. Tewip will not invent a selector.`);
|
|
56
|
+
}
|
|
57
|
+
const evidence = found.how === 'renamed in git'
|
|
58
|
+
? `commit ${found.commit} changed "${token.value}" to "${found.value}" in ${found.file}`
|
|
59
|
+
: `"${token.value}" is nowhere in the source any more, and ${found.file} now has "${found.value}"`;
|
|
60
|
+
result.evidence = evidence;
|
|
61
|
+
result.log.push(evidence);
|
|
62
|
+
const replacement = writeSelector(token, found.value);
|
|
63
|
+
const source = deps.readFile(specFile);
|
|
64
|
+
const sites = findSelectorSites(source, written);
|
|
65
|
+
const site = sites[0];
|
|
66
|
+
if (!site)
|
|
67
|
+
return stop('selector-not-in-test', `${selector} is not written in ${rel}: it comes from a helper or a page object, which a person should change`);
|
|
68
|
+
result.line = site.line;
|
|
69
|
+
result.original = site.text;
|
|
70
|
+
result.replacement = site.text.replace(written, replacement);
|
|
71
|
+
deps.writeFile(specFile, source.replace(site.text, result.replacement));
|
|
72
|
+
try {
|
|
73
|
+
const run = await deps.verify(specFile, title);
|
|
74
|
+
if (!run.passed) {
|
|
75
|
+
deps.writeFile(specFile, source);
|
|
76
|
+
return stop('still-failing', `with ${replacement} the test still failed, so the file was put back: ${run.detail.slice(-200)}`);
|
|
77
|
+
}
|
|
78
|
+
result.log.push(`with ${replacement} the test passes`);
|
|
79
|
+
if (!keep)
|
|
80
|
+
deps.writeFile(specFile, source);
|
|
81
|
+
return { ...result, status: 'fixed', applied: keep };
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
deps.writeFile(specFile, source);
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/** The one-line diff a reviewer reads. */
|
|
89
|
+
export function selectorFixDiff(result) {
|
|
90
|
+
if (!result.original || !result.replacement)
|
|
91
|
+
return '';
|
|
92
|
+
return [`--- a/${result.specFile}`, `+++ b/${result.specFile}`, `@@ line ${result.line} @@`, `-${result.original.trim()}`, `+${result.replacement.trim()}`].join('\n');
|
|
93
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// Phase 5, for the tools that do not have a page snapshot: Cypress, Selenium and WebdriverIO.
|
|
2
|
+
//
|
|
3
|
+
// Playwright's traces tell Tewip what was on the page, so a drifted locator can be matched against
|
|
4
|
+
// the real elements. A Cypress or Selenium failure in a JUnit report carries none of that - only
|
|
5
|
+
// "Expected to find element: '#submit-btn', but never found it". Inventing a replacement from that
|
|
6
|
+
// would be exactly the kind of guess this project exists not to make.
|
|
7
|
+
//
|
|
8
|
+
// So the answer comes from the repository instead. If a test looks for `#submit-btn` and the app
|
|
9
|
+
// now renders `id="submit-button"`, the new name is in the application source, and git knows which
|
|
10
|
+
// commit renamed it. That is evidence, not a guess - and the fix is still only kept if the test
|
|
11
|
+
// passes afterwards.
|
|
12
|
+
import { execFileSync } from 'node:child_process';
|
|
13
|
+
import { readFileSync } from 'node:fs';
|
|
14
|
+
/** What a CSS or XPath selector is really identifying, when it is something identifiable. */
|
|
15
|
+
export function tokenOf(selector) {
|
|
16
|
+
// Selenium and WebdriverIO name the strategy first: "css selector=#submit-btn", "id=submit-btn".
|
|
17
|
+
const strategy = /^(css selector|css|xpath|id|name|class name|link text|partial link text|tag name)\s*=\s*(.+)$/i.exec(selector.trim());
|
|
18
|
+
const s = (strategy ? strategy[2] : selector).trim();
|
|
19
|
+
if (strategy) {
|
|
20
|
+
const method = strategy[1].toLowerCase();
|
|
21
|
+
if (method === 'id')
|
|
22
|
+
return { kind: 'id', value: s };
|
|
23
|
+
if (method === 'class name')
|
|
24
|
+
return { kind: 'class', value: s.split(/\s+/)[0] };
|
|
25
|
+
if (method === 'link text' || method === 'partial link text')
|
|
26
|
+
return { kind: 'text', value: s };
|
|
27
|
+
}
|
|
28
|
+
const data = /\[\s*([\w-]*data[\w-]*)\s*[~|^$*]?=\s*["']?([^"'\]]+)["']?\s*\]/i.exec(s);
|
|
29
|
+
if (data)
|
|
30
|
+
return { kind: 'data', attribute: data[1], value: data[2] };
|
|
31
|
+
const id = /^#([\w-]+)$|\[\s*id\s*=\s*["']([^"']+)["']\s*\]|@id\s*=\s*["']([^"']+)["']/i.exec(s);
|
|
32
|
+
if (id)
|
|
33
|
+
return { kind: 'id', value: id[1] ?? id[2] ?? id[3] };
|
|
34
|
+
const cls = /^\.([\w-]+)$|\[\s*class\s*=\s*["']([^"']+)["']\s*\]/i.exec(s);
|
|
35
|
+
if (cls)
|
|
36
|
+
return { kind: 'class', value: (cls[1] ?? cls[2]).split(/\s+/)[0] };
|
|
37
|
+
const text = /text\(\)\s*=\s*["']([^"']+)["']|contains\(\s*(?:text\(\)|\.)\s*,\s*["']([^"']+)["']/i.exec(s);
|
|
38
|
+
if (text)
|
|
39
|
+
return { kind: 'text', value: text[1] ?? text[2] };
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
/** How close two names are, 0 to 1, by the words in them - "submit-btn" and "submit-button". */
|
|
43
|
+
export function closeness(a, b) {
|
|
44
|
+
const words = (s) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
|
|
45
|
+
const left = words(a);
|
|
46
|
+
const right = words(b);
|
|
47
|
+
if (left.size === 0 || right.size === 0)
|
|
48
|
+
return 0;
|
|
49
|
+
let shared = 0;
|
|
50
|
+
for (const w of left)
|
|
51
|
+
if (right.has(w))
|
|
52
|
+
shared++;
|
|
53
|
+
// A shared prefix counts too: "submitBtn" and "submitButton" have no word in common.
|
|
54
|
+
const prefix = commonPrefix(a.toLowerCase(), b.toLowerCase()) / Math.max(a.length, b.length);
|
|
55
|
+
return Math.max(shared / Math.max(left.size, right.size), prefix);
|
|
56
|
+
}
|
|
57
|
+
function commonPrefix(a, b) {
|
|
58
|
+
let i = 0;
|
|
59
|
+
while (i < a.length && i < b.length && a[i] === b[i])
|
|
60
|
+
i++;
|
|
61
|
+
return i;
|
|
62
|
+
}
|
|
63
|
+
const git = (root, args) => {
|
|
64
|
+
try {
|
|
65
|
+
return execFileSync('git', args, { cwd: root, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: 60_000 });
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return '';
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* The commit that removed this value, and what took its place on the same line. This is the
|
|
73
|
+
* strongest evidence there is: somebody renamed it, and the diff says to what.
|
|
74
|
+
*/
|
|
75
|
+
export function renamedInGit(root, token, maxCommits = 5) {
|
|
76
|
+
const shas = git(root, ['log', '-S', token.value, '--format=%H', `-n${maxCommits}`, '--', '.'])
|
|
77
|
+
.split('\n')
|
|
78
|
+
.map((s) => s.trim())
|
|
79
|
+
.filter(Boolean);
|
|
80
|
+
for (const sha of shas) {
|
|
81
|
+
const diff = git(root, ['show', '--unified=0', '--format=', sha]);
|
|
82
|
+
if (!diff)
|
|
83
|
+
continue;
|
|
84
|
+
let file = '';
|
|
85
|
+
const removed = [];
|
|
86
|
+
const added = [];
|
|
87
|
+
for (const line of diff.split('\n')) {
|
|
88
|
+
if (line.startsWith('+++ b/'))
|
|
89
|
+
file = line.slice(6);
|
|
90
|
+
else if (line.startsWith('-') && line.includes(token.value))
|
|
91
|
+
removed.push(line);
|
|
92
|
+
else if (line.startsWith('+'))
|
|
93
|
+
added.push(line);
|
|
94
|
+
}
|
|
95
|
+
if (removed.length === 0 || added.length === 0)
|
|
96
|
+
continue;
|
|
97
|
+
// The best replacement is the attribute value on an added line that most resembles the old one.
|
|
98
|
+
let best = null;
|
|
99
|
+
for (const line of added) {
|
|
100
|
+
for (const value of attributeValues(line, token)) {
|
|
101
|
+
const score = closeness(token.value, value);
|
|
102
|
+
if (value !== token.value && score >= 0.5 && (!best || score > best.score)) {
|
|
103
|
+
best = { value, file, how: 'renamed in git', commit: sha.slice(0, 8), score };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (best)
|
|
108
|
+
return best;
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
/** Values of the same kind of attribute written on a line of source. */
|
|
113
|
+
function attributeValues(line, token) {
|
|
114
|
+
const out = [];
|
|
115
|
+
const attribute = token.kind === 'data' ? token.attribute ?? 'data-[\\w-]+' : token.kind === 'id' ? 'id' : token.kind === 'class' ? 'class(?:Name)?' : '';
|
|
116
|
+
if (!attribute)
|
|
117
|
+
return out;
|
|
118
|
+
const pattern = new RegExp(`${attribute}\\s*=\\s*["'\`{]?\\s*["']?([\\w\\s-]+)["']?`, 'gi');
|
|
119
|
+
for (const m of line.matchAll(pattern)) {
|
|
120
|
+
for (const value of m[1].trim().split(/\s+/))
|
|
121
|
+
if (value)
|
|
122
|
+
out.push(value);
|
|
123
|
+
}
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* The closest value of the same kind anywhere in the application source now. Weaker than git,
|
|
128
|
+
* because nobody wrote down that it is a rename - so it is offered only when it is clearly close.
|
|
129
|
+
*/
|
|
130
|
+
export function likeItInSource(root, token, sources, min = 0.6) {
|
|
131
|
+
let best = null;
|
|
132
|
+
for (const file of sources) {
|
|
133
|
+
let text;
|
|
134
|
+
try {
|
|
135
|
+
text = readFileSync(file, 'utf8');
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (text.includes(token.value))
|
|
141
|
+
return null; // it is still there; this is not a rename
|
|
142
|
+
for (const line of text.split('\n')) {
|
|
143
|
+
for (const value of attributeValues(line, token)) {
|
|
144
|
+
const score = closeness(token.value, value);
|
|
145
|
+
if (value !== token.value && score >= min && (!best || score > best.score)) {
|
|
146
|
+
best = { value, file, how: 'found in the source', score };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return best;
|
|
152
|
+
}
|
|
153
|
+
/** The selector written the way each tool writes it, so the fix reads like the rest of the test. */
|
|
154
|
+
export function writeSelector(token, value) {
|
|
155
|
+
if (token.kind === 'id')
|
|
156
|
+
return `#${value}`;
|
|
157
|
+
if (token.kind === 'class')
|
|
158
|
+
return `.${value}`;
|
|
159
|
+
if (token.kind === 'data')
|
|
160
|
+
return `[${token.attribute}="${value}"]`;
|
|
161
|
+
return value;
|
|
162
|
+
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
// Imports test reports from any language or framework into the agent's records: JUnit XML, which
|
|
2
2
|
// pytest, Jest, Vitest, Mocha, node:test, Maven Surefire, Gradle, gotestsum, PHPUnit, RSpec,
|
|
3
|
-
// cargo-nextest and Cypress write
|
|
3
|
+
// cargo-nextest and Cypress write; .NET's own TRX; NUnit 3's result file; Robot Framework's
|
|
4
|
+
// output.xml; JMeter's .jtl; and a k6 summary, where a run fails by crossing a threshold rather
|
|
5
|
+
// than test by test. Playwright keeps its own importer
|
|
4
6
|
// (blob.ts), which also reads traces.
|
|
5
7
|
//
|
|
6
8
|
// What these reports can and cannot say: each test's name, outcome, failure message and stack,
|
|
@@ -11,6 +13,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
|
11
13
|
import { join } from 'node:path';
|
|
12
14
|
import { SCHEMA_VERSION } from '../../reporters/types.js';
|
|
13
15
|
import { descendants, parseXml } from './xml.js';
|
|
16
|
+
import { jmeterCases, k6Cases, looksLikeK6, robotCases, robotTime } from './other-formats.js';
|
|
14
17
|
const PASSED = { failed: false, message: '', stack: '' };
|
|
15
18
|
const FAILURE = new Set(['failure', 'error']);
|
|
16
19
|
/** Maven Surefire's records of earlier attempts: flaky* when a later one passed, rerun* when not. */
|
|
@@ -138,7 +141,8 @@ function nunitCases(root) {
|
|
|
138
141
|
};
|
|
139
142
|
});
|
|
140
143
|
}
|
|
141
|
-
|
|
144
|
+
// .jtl is JMeter's, and a k6 summary is JSON; both are read below.
|
|
145
|
+
const REPORT_FILE = /\.(xml|trx|jtl|json)$/i;
|
|
142
146
|
/** Report files under `path` (a file, or a directory searched recursively). */
|
|
143
147
|
export function junitFiles(path) {
|
|
144
148
|
if (!statSync(path).isDirectory())
|
|
@@ -162,7 +166,26 @@ export function importJunit(path, meta) {
|
|
|
162
166
|
const unread = [];
|
|
163
167
|
let parsedFiles = 0;
|
|
164
168
|
for (const file of junitFiles(path)) {
|
|
165
|
-
const
|
|
169
|
+
const raw = readFileSync(file, 'utf8');
|
|
170
|
+
// k6 writes JSON, not XML: a performance run fails by crossing a threshold, not test by test.
|
|
171
|
+
if (/\.json$/i.test(file)) {
|
|
172
|
+
let parsed;
|
|
173
|
+
try {
|
|
174
|
+
parsed = JSON.parse(raw);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
unread.push(`${file} (not JSON)`);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (!looksLikeK6(parsed)) {
|
|
181
|
+
unread.push(`${file} (JSON, but not a k6 summary)`);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
parsedFiles++;
|
|
185
|
+
addCases(k6Cases(parsed));
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const root = parseXml(raw);
|
|
166
189
|
let cases;
|
|
167
190
|
if (root.name === 'TestRun') {
|
|
168
191
|
cases = trxCases(root);
|
|
@@ -182,6 +205,15 @@ export function importJunit(path, meta) {
|
|
|
182
205
|
if (start && (!startedAt || start < startedAt))
|
|
183
206
|
startedAt = start;
|
|
184
207
|
}
|
|
208
|
+
else if (root.name === 'robot') {
|
|
209
|
+
cases = robotCases(root);
|
|
210
|
+
const start = robotTime(root.attrs.generated ?? '');
|
|
211
|
+
if (start && !startedAt)
|
|
212
|
+
startedAt = start;
|
|
213
|
+
}
|
|
214
|
+
else if (root.name === 'testResults') {
|
|
215
|
+
cases = jmeterCases(root);
|
|
216
|
+
}
|
|
185
217
|
else {
|
|
186
218
|
// Not a report Tewip knows. Staying silent here would report a failing run as clean, so the
|
|
187
219
|
// file is remembered and, if nothing at all could be read, the caller is told.
|
|
@@ -189,6 +221,9 @@ export function importJunit(path, meta) {
|
|
|
189
221
|
continue;
|
|
190
222
|
}
|
|
191
223
|
parsedFiles++;
|
|
224
|
+
addCases(cases);
|
|
225
|
+
}
|
|
226
|
+
function addCases(cases) {
|
|
192
227
|
for (const c of cases) {
|
|
193
228
|
attempts[c.testId] = (attempts[c.testId] ?? 0) + c.attempts.length;
|
|
194
229
|
c.attempts.forEach((attempt, retry) => {
|
|
@@ -226,7 +261,8 @@ export function importJunit(path, meta) {
|
|
|
226
261
|
if (unread.length > 0) {
|
|
227
262
|
const list = unread.slice(0, 5).join(', ');
|
|
228
263
|
if (parsedFiles === 0) {
|
|
229
|
-
throw new Error(`no test report Tewip can read in ${path}: ${list}. It reads JUnit XML (<testsuites>/<testsuite>), NUnit 3 (<test-run>)
|
|
264
|
+
throw new Error(`no test report Tewip can read in ${path}: ${list}. It reads JUnit XML (<testsuites>/<testsuite>), NUnit 3 (<test-run>), .NET TRX (<TestRun>), ` +
|
|
265
|
+
'Robot Framework output.xml (<robot>), JMeter .jtl (<testResults>) and a k6 summary (--summary-export). ' +
|
|
230
266
|
'Most runners can write JUnit XML; for TestNG use the Surefire report, and for NUnit use --result=x.xml.');
|
|
231
267
|
}
|
|
232
268
|
console.error(`[tewip] ignored ${unread.length} file(s) that are not a test report: ${list}`);
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { descendants } from './xml.js';
|
|
2
|
+
const text = (element) => (element ? element.text.trim() : '');
|
|
3
|
+
/**
|
|
4
|
+
* Robot Framework writes output.xml by default, and only writes JUnit if asked. A suite holds
|
|
5
|
+
* tests, a test holds a status with the failure message inside it, and suites nest - so the test's
|
|
6
|
+
* name is its whole path, the way Robot itself prints it.
|
|
7
|
+
*/
|
|
8
|
+
export function robotCases(root) {
|
|
9
|
+
const cases = [];
|
|
10
|
+
const walk = (element, path, source) => {
|
|
11
|
+
for (const child of element.children) {
|
|
12
|
+
if (child.name === 'suite') {
|
|
13
|
+
walk(child, [...path, child.attrs.name ?? ''], child.attrs.source ?? source);
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
if (child.name !== 'test')
|
|
17
|
+
continue;
|
|
18
|
+
const status = child.children.find((c) => c.name === 'status');
|
|
19
|
+
const failed = (status?.attrs.status ?? '').toUpperCase() === 'FAIL';
|
|
20
|
+
const start = status?.attrs.starttime ?? status?.attrs.start ?? '';
|
|
21
|
+
const end = status?.attrs.endtime ?? status?.attrs.end ?? '';
|
|
22
|
+
// The last keyword that failed says where it went wrong, which the message alone does not.
|
|
23
|
+
const failedKeyword = descendants(child, 'kw').find((kw) => (kw.children.find((c) => c.name === 'status')?.attrs.status ?? '').toUpperCase() === 'FAIL');
|
|
24
|
+
const where = failedKeyword?.attrs.name ? `Keyword: ${failedKeyword.attrs.name}` : '';
|
|
25
|
+
cases.push({
|
|
26
|
+
testId: [...path, child.attrs.name ?? ''].filter(Boolean).join(' > '),
|
|
27
|
+
file: source,
|
|
28
|
+
duration: robotDuration(start, end),
|
|
29
|
+
attempts: [{ failed, message: failed ? text(status) || 'the test failed' : '', stack: failed ? where : '' }],
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
walk(root, [], root.attrs.source ?? '');
|
|
34
|
+
return cases;
|
|
35
|
+
}
|
|
36
|
+
/** Robot's timestamp as an ISO one, or '' when it is not the shape Robot writes. */
|
|
37
|
+
export function robotTime(stamp) {
|
|
38
|
+
const at = robotStamp(stamp);
|
|
39
|
+
return at > 0 ? new Date(at).toISOString() : '';
|
|
40
|
+
}
|
|
41
|
+
/** Robot's own timestamp shape: 20260923 14:05:01.123 */
|
|
42
|
+
function robotStamp(s) {
|
|
43
|
+
const m = /^(\d{4})(\d{2})(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{1,3})$/.exec(s.trim());
|
|
44
|
+
return m ? Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]), Number(m[5]), Number(m[6]), Number(m[7])) : 0;
|
|
45
|
+
}
|
|
46
|
+
function robotDuration(start, end) {
|
|
47
|
+
const from = robotStamp(start);
|
|
48
|
+
const to = robotStamp(end);
|
|
49
|
+
return from > 0 && to >= from ? to - from : 0;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* JMeter's .jtl in XML form: one sample per request, s="false" when it failed. A load test has
|
|
53
|
+
* thousands of samples, so they are grouped by label - nobody wants ten thousand records, they want
|
|
54
|
+
* "the checkout request failed 412 times, mostly 503".
|
|
55
|
+
*/
|
|
56
|
+
export function jmeterCases(root) {
|
|
57
|
+
const samples = [...descendants(root, 'httpSample'), ...descendants(root, 'sample')];
|
|
58
|
+
const byLabel = new Map();
|
|
59
|
+
for (const sample of samples) {
|
|
60
|
+
const label = sample.attrs.lb ?? 'sample';
|
|
61
|
+
const found = byLabel.get(label) ?? { failed: 0, total: 0, time: 0, codes: new Map(), messages: new Set() };
|
|
62
|
+
found.total++;
|
|
63
|
+
found.time += Number(sample.attrs.t ?? 0);
|
|
64
|
+
if ((sample.attrs.s ?? 'true').toLowerCase() === 'false') {
|
|
65
|
+
found.failed++;
|
|
66
|
+
const code = `${sample.attrs.rc ?? ''} ${sample.attrs.rm ?? ''}`.trim();
|
|
67
|
+
if (code)
|
|
68
|
+
found.codes.set(code, (found.codes.get(code) ?? 0) + 1);
|
|
69
|
+
const assertion = descendants(sample, 'failureMessage')[0];
|
|
70
|
+
if (assertion)
|
|
71
|
+
found.messages.add(text(assertion).slice(0, 200));
|
|
72
|
+
}
|
|
73
|
+
byLabel.set(label, found);
|
|
74
|
+
}
|
|
75
|
+
return [...byLabel].map(([label, s]) => {
|
|
76
|
+
const worst = [...s.codes].sort((a, b) => b[1] - a[1])[0];
|
|
77
|
+
// A 2xx that still counts as failed is an assertion failing, not the server: "mostly 200" would
|
|
78
|
+
// read as nonsense to whoever opens the report.
|
|
79
|
+
const why = worst ? (/^2\d\d/.test(worst[0]) ? `, an assertion failed (${worst[1]} times, response ${worst[0]})` : `, mostly ${worst[0]} (${worst[1]} times)`) : '';
|
|
80
|
+
const message = s.failed === 0 ? '' : `${s.failed} of ${s.total} requests failed${why}${s.messages.size > 0 ? `: ${[...s.messages][0]}` : ''}`;
|
|
81
|
+
return { testId: `JMeter > ${label}`, file: '', duration: s.time, attempts: [{ failed: s.failed > 0, message, stack: [...s.messages].join('\n').slice(0, 2000) }] };
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* k6 writes a summary as JSON (`k6 run --summary-export=summary.json`). A performance run does not
|
|
86
|
+
* fail test by test: it fails when a threshold is crossed, and that is what is imported - one
|
|
87
|
+
* record per crossed threshold, saying by how much.
|
|
88
|
+
*/
|
|
89
|
+
export function k6Cases(json) {
|
|
90
|
+
const report = json;
|
|
91
|
+
if (!report?.metrics)
|
|
92
|
+
return [];
|
|
93
|
+
const cases = [];
|
|
94
|
+
for (const [metric, body] of Object.entries(report.metrics)) {
|
|
95
|
+
const thresholds = body.thresholds;
|
|
96
|
+
if (!thresholds)
|
|
97
|
+
continue;
|
|
98
|
+
for (const [expression, result] of Object.entries(thresholds)) {
|
|
99
|
+
// k6 writes either a boolean "ok" or an object with it; a crossed threshold is ok === false.
|
|
100
|
+
const ok = typeof result === 'boolean' ? result : Boolean(result?.ok);
|
|
101
|
+
const measured = describeMetric((body.values ?? body));
|
|
102
|
+
cases.push({
|
|
103
|
+
testId: `k6 > ${metric} > ${expression}`,
|
|
104
|
+
file: '',
|
|
105
|
+
duration: 0,
|
|
106
|
+
attempts: [{ failed: !ok, message: ok ? '' : `the threshold "${expression}" on ${metric} was crossed${measured ? `: ${measured}` : ''}`, stack: measured }],
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return cases;
|
|
111
|
+
}
|
|
112
|
+
/** The numbers k6 reports for a metric, in one readable line. */
|
|
113
|
+
function describeMetric(body) {
|
|
114
|
+
const wanted = ['avg', 'min', 'med', 'max', 'p(90)', 'p(95)', 'p(99)', 'rate', 'count', 'value'];
|
|
115
|
+
return wanted
|
|
116
|
+
.filter((key) => typeof body[key] === 'number')
|
|
117
|
+
.map((key) => `${key}=${round(body[key])}`)
|
|
118
|
+
.join(' ');
|
|
119
|
+
}
|
|
120
|
+
const round = (n) => (Number.isInteger(n) ? String(n) : n.toFixed(2));
|
|
121
|
+
/** Whether a parsed JSON file looks like a k6 summary, so nothing else is read as one. */
|
|
122
|
+
export function looksLikeK6(json) {
|
|
123
|
+
const report = json;
|
|
124
|
+
if (!report?.metrics || typeof report.metrics !== 'object')
|
|
125
|
+
return false;
|
|
126
|
+
return ['http_req_duration', 'iterations', 'vus', 'checks', 'http_reqs'].some((m) => m in report.metrics);
|
|
127
|
+
}
|