tewip-ai 0.52.0 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/access/access-command.js +2 -1
- package/dist/engine/access/checks.js +26 -5
- package/dist/engine/access/element.js +14 -1
- package/dist/engine/access/markup.js +22 -1
- package/dist/engine/quality/source-text.js +29 -0
- package/dist/engine/schema/schema-command.js +28 -3
- package/dist/engine/vouch/find-tests.js +14 -2
- package/dist/engine/vouch/swallowed.js +92 -0
- package/dist/engine/vouch/text.js +45 -0
- package/dist/engine/vouch/vouch-command.js +2 -1
- package/dist/engine/vouch/vouch.js +3 -94
- package/package.json +1 -1
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// It never says a page is accessible. It says what was checked.
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { normalise } from '../quality/source-text.js';
|
|
12
13
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
13
14
|
import { sourceFiles } from '../secure/secrets.js';
|
|
14
15
|
import { JS_LIKE, MARKUP, tags } from './markup.js';
|
|
@@ -50,7 +51,7 @@ export function findBarriers(root, only) {
|
|
|
50
51
|
for (const path of files) {
|
|
51
52
|
let source;
|
|
52
53
|
try {
|
|
53
|
-
source = readFileSync(path, 'utf8');
|
|
54
|
+
source = normalise(readFileSync(path, 'utf8'));
|
|
54
55
|
}
|
|
55
56
|
catch {
|
|
56
57
|
continue;
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
// div that only answers to a mouse. The ones that need a rendered page (contrast, focus order,
|
|
7
7
|
// whether alt text is any good) are not attempted here and are listed in NOT_CHECKED instead.
|
|
8
8
|
import { inside, labelledBy } from './markup.js';
|
|
9
|
-
import { ARIA, INTERACTIVE, UNLABELLED_TYPES, at, clickHandler, decorative, hiddenFromEveryone, keyHandler, show, tabIndexOf } from './element.js';
|
|
9
|
+
import { ARIA, INTERACTIVE, UNLABELLED_TYPES, at, clickHandler, decorative, hiddenFromEveryone, keyHandler, notRendered, show, tabIndexOf } from './element.js';
|
|
10
10
|
/** Checks that look at one element at a time. */
|
|
11
|
-
function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent = new Set()) {
|
|
11
|
+
function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent = new Set(), boundLabels = new Set()) {
|
|
12
12
|
const out = [];
|
|
13
13
|
const place = at(t, file);
|
|
14
14
|
const name = labelledBy(t);
|
|
@@ -44,7 +44,8 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
|
|
|
44
44
|
// and `<Select onOpenChange={...}>` was reported as an unnamed combo box; what that component
|
|
45
45
|
// renders, and whether it labels itself, is not in this file and cannot be read from it.
|
|
46
46
|
if (nameRule && !t.component && !hiddenFromEveryone(t) && (t.name !== 'input' || !UNLABELLED_TYPES.has(t.attrs.type ?? 'text'))) {
|
|
47
|
-
const
|
|
47
|
+
const bound = ['asp-for', 'formControlName', 'v-model', 'data-valmsg-for'].some((key) => typeof t.attrs[key] === 'string' && boundLabels.has(`${key}=${t.attrs[key]}`));
|
|
48
|
+
const linked = bound || (typeof t.attrs.id === 'string' && labelFor.has(t.attrs.id));
|
|
48
49
|
if (!name.named && !linked && openLabels === 0) {
|
|
49
50
|
if (name.unknown || t.dynamic.has('id') || t.dynamic.has('style') || (typeof t.attrs.id === 'string' && namedByComponent.has(t.attrs.id))) {
|
|
50
51
|
out.push({ rule: nameRule, confidence: 'ask-a-person', ...place,
|
|
@@ -73,7 +74,11 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
|
|
|
73
74
|
why: `tabindex="${tabindex}" takes this element out of the document order and puts it before everything that has none`,
|
|
74
75
|
next: 'use tabindex="0" and let the source order decide, or move the element in the markup' });
|
|
75
76
|
}
|
|
76
|
-
|
|
77
|
+
// `hidden` takes the element out of the tab order as well as out of the page, so aria-hidden on it
|
|
78
|
+
// hides nothing that was reachable. GOV.UK's service-navigation toggle is written
|
|
79
|
+
// `<button ... hidden aria-hidden="true">` and uncovered by JavaScript when the menu is enabled -
|
|
80
|
+
// progressive enhancement done properly, reported as a barrier.
|
|
81
|
+
if (t.attrs['aria-hidden'] === 'true' && !notRendered(t) && (INTERACTIVE.has(t.name) || (tabindex !== null && tabindex >= 0))) {
|
|
77
82
|
out.push({ rule: 'aria-hidden-focusable', confidence: 'barrier', ...place,
|
|
78
83
|
why: 'hidden from screen readers but still reachable by the tab key',
|
|
79
84
|
next: 'also remove it from the tab order with tabindex="-1", or stop hiding it' });
|
|
@@ -121,6 +126,22 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
|
|
|
121
126
|
export function checkFile(source, all, file, isDocument = false) {
|
|
122
127
|
const out = [];
|
|
123
128
|
const labelFor = new Set(all.filter((t) => t.name === 'label' && t.attrs.for).map((t) => t.attrs.for));
|
|
129
|
+
// A framework can bind a label to a field without either of them spelling `for` or `id`. ASP.NET
|
|
130
|
+
// Core Tag Helpers do it with the model property: `asp-for="Name"` on the <label> renders
|
|
131
|
+
// `for="Name"`, and the same on the <input> renders `id="Name"`. bitwarden/server's admin views
|
|
132
|
+
// are written this way throughout, and 111 correctly labelled fields were called unlabelled.
|
|
133
|
+
// The same shape as PatternFly's FormGroup in Phase 43: the label is real, it is just not here.
|
|
134
|
+
const BINDS = ['asp-for', 'formControlName', 'v-model', 'data-valmsg-for'];
|
|
135
|
+
const boundLabels = new Set();
|
|
136
|
+
for (const t of all) {
|
|
137
|
+
if (t.name !== 'label')
|
|
138
|
+
continue;
|
|
139
|
+
for (const key of BINDS) {
|
|
140
|
+
const value = t.attrs[key];
|
|
141
|
+
if (typeof value === 'string' && value !== '')
|
|
142
|
+
boundLabels.add(`${key}=${value}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
124
145
|
// Ids that a *component* names in one of its props. A design system labels a field by being told
|
|
125
146
|
// which field it is for - PatternFly's `<FormGroup label={...} fieldId="x">` renders `<label for="x">`
|
|
126
147
|
// around its children - and that label exists only in the component's own source, not here.
|
|
@@ -153,7 +174,7 @@ export function checkFile(source, all, file, isDocument = false) {
|
|
|
153
174
|
open.length = from;
|
|
154
175
|
continue;
|
|
155
176
|
}
|
|
156
|
-
out.push(...perElement(t, file, labelFor, Math.max(openLabels - (t.name === 'label' ? 1 : 0), 0), open.some((o) => o.keys), namedByComponent));
|
|
177
|
+
out.push(...perElement(t, file, labelFor, Math.max(openLabels - (t.name === 'label' ? 1 : 0), 0), open.some((o) => o.keys), namedByComponent, boundLabels));
|
|
157
178
|
if (!t.selfClosing)
|
|
158
179
|
open.push({ name: t.name, keys: keyHandler(t) });
|
|
159
180
|
const id = t.attrs.id;
|
|
@@ -58,9 +58,22 @@ export const UNLABELLED_TYPES = new Set(['hidden', 'submit', 'reset', 'button',
|
|
|
58
58
|
* When the style is a JSX expression its text is not kept, so whether it hides the element cannot be
|
|
59
59
|
* read. That is not a reason to call it a certain barrier - it is a reason to say so.
|
|
60
60
|
*/
|
|
61
|
-
|
|
61
|
+
/**
|
|
62
|
+
* Not rendered at all, and therefore not in the tab order either.
|
|
63
|
+
*
|
|
64
|
+
* Separate from `hiddenFromEveryone` because `aria-hidden` hides an element from a screen reader
|
|
65
|
+
* while leaving it perfectly focusable - that gap is a real barrier and has its own rule. The
|
|
66
|
+
* `hidden` attribute and `display:none` close the gap: nothing can reach the element by any route.
|
|
67
|
+
*/
|
|
68
|
+
export function notRendered(t) {
|
|
62
69
|
if ('hidden' in t.attrs && t.attrs.hidden !== 'false')
|
|
63
70
|
return true;
|
|
71
|
+
const style = typeof t.attrs.style === 'string' ? t.attrs.style : '';
|
|
72
|
+
return /display\s*:\s*none|visibility\s*:\s*hidden/i.test(style);
|
|
73
|
+
}
|
|
74
|
+
export function hiddenFromEveryone(t) {
|
|
75
|
+
if (notRendered(t))
|
|
76
|
+
return true;
|
|
64
77
|
if (t.attrs['aria-hidden'] === 'true')
|
|
65
78
|
return true;
|
|
66
79
|
const style = typeof t.attrs.style === 'string' ? t.attrs.style : '';
|
|
@@ -31,6 +31,27 @@ const TAG = /<(\/?)([A-Za-z][A-Za-z0-9._:-]*)(?=[\s/>])((?:[^>"'{]|"[^"]*"|'[^']
|
|
|
31
31
|
const ATTRIBUTE = /([@:#\[A-Za-z_][-A-Za-z0-9_:.@\]]*)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|\{((?:[^{}]|\{[^{}]*\})*)\}|([^\s"'<>`=]+)))?/g;
|
|
32
32
|
/** `{...props}`, `{...rest}` - the thing that makes every other judgement about the element unsafe. */
|
|
33
33
|
const SPREAD = /\{\s*\.\.\./;
|
|
34
|
+
/**
|
|
35
|
+
* A template language's version of a spread: an expression standing where an attribute *name*
|
|
36
|
+
* would go, rather than inside a value.
|
|
37
|
+
*
|
|
38
|
+
* GOV.UK's own input component is written this way - every attribute, `id` included, comes out of
|
|
39
|
+
* one macro call:
|
|
40
|
+
*
|
|
41
|
+
* <input
|
|
42
|
+
* {{- govukAttributes({ class: classNames, id: id, name: params.name, ... }) }}
|
|
43
|
+
*
|
|
44
|
+
* so after the expressions are blanked the element reads as a bare `<input>` with nothing on it,
|
|
45
|
+
* and the scanner called the UK government's accessible-by-obligation text input a certain
|
|
46
|
+
* barrier. It is not: it is an element this file cannot answer for, which is a different verdict.
|
|
47
|
+
*
|
|
48
|
+
* Quoted values are removed first, because `id="{{ params.id }}"` is a dynamic attribute - which
|
|
49
|
+
* is already handled, and separately - rather than attributes arriving from somewhere else.
|
|
50
|
+
*/
|
|
51
|
+
function templateSpread(blob) {
|
|
52
|
+
const withoutValues = blob.replace(/"[^"]*"|'[^']*'/g, '""');
|
|
53
|
+
return /\{\{|\{%|<%|#{/.test(withoutValues);
|
|
54
|
+
}
|
|
34
55
|
/**
|
|
35
56
|
* Comments, and the bodies of script and style, hold things shaped like tags that are not tags.
|
|
36
57
|
*
|
|
@@ -73,7 +94,7 @@ export const JS_LIKE = /\.(?:[cm]?[jt]sx?|vue|svelte|astro)$/i;
|
|
|
73
94
|
function readAttributes(blob) {
|
|
74
95
|
const attrs = {};
|
|
75
96
|
const dynamic = new Set();
|
|
76
|
-
const spread = SPREAD.test(blob);
|
|
97
|
+
const spread = SPREAD.test(blob) || templateSpread(blob);
|
|
77
98
|
// A spread's contents would otherwise be read as attribute names: `{...props}` becomes `props`.
|
|
78
99
|
const readable = blob.replace(/\{\s*\.\.\.(?:[^{}]|\{[^{}]*\})*\}/g, ' ');
|
|
79
100
|
for (const m of readable.matchAll(ATTRIBUTE)) {
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Reading somebody else's file, the same way on every machine.
|
|
2
|
+
//
|
|
3
|
+
// Found by the corpus harness on its first run in CI, Phase 44: the same pinned commit of Keycloak
|
|
4
|
+
// produced 0 swallows-failure findings on Windows and 3 on Linux, and OpenMRS differed by seven
|
|
5
|
+
// crash-only tests. Git for Windows checks source out with CRLF, so the text Tewip judged was not
|
|
6
|
+
// the text the project wrote, and a scanner whose answer depends on which machine ran it is not
|
|
7
|
+
// giving an answer at all - it is giving an opinion with a hostname attached.
|
|
8
|
+
//
|
|
9
|
+
// The carriage returns go at the boundary rather than in each rule, because the alternative is
|
|
10
|
+
// remembering `\r?` in every pattern anybody writes afterwards, and one forgotten `\r?` is this bug
|
|
11
|
+
// again. Line numbers are unaffected: a CRLF file has exactly as many newlines as an LF one.
|
|
12
|
+
//
|
|
13
|
+
// A BOM goes too. It is invisible, it sits before the first character of the file, and it stops a
|
|
14
|
+
// pattern anchored at the start of the text from matching the first line.
|
|
15
|
+
import { readFileSync } from 'node:fs';
|
|
16
|
+
/** The file's text with line endings normalised, or '' when it cannot be read. */
|
|
17
|
+
export function sourceText(file) {
|
|
18
|
+
try {
|
|
19
|
+
return normalise(readFileSync(file, 'utf8'));
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return '';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/** Normalised, for text that has already been read. */
|
|
26
|
+
export function normalise(text) {
|
|
27
|
+
const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
28
|
+
return withoutBom.includes(String.fromCharCode(13)) ? withoutBom.split(String.fromCharCode(13) + String.fromCharCode(10)).join(String.fromCharCode(10)) : withoutBom;
|
|
29
|
+
}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
// such. A mention in a comment counts as a mention; Tewip says where it looked and lets the reader judge.
|
|
10
10
|
import { spawnSync } from 'node:child_process';
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { normalise } from '../quality/source-text.js';
|
|
12
13
|
import { dirname, resolve } from 'node:path';
|
|
13
14
|
import { changeSummary } from '../quality/grounding.js';
|
|
14
15
|
import { dialectOf, forwardOnly, irreversible, readFramework } from './frameworks.js';
|
|
@@ -38,8 +39,32 @@ export function schemaFiles(cmd) {
|
|
|
38
39
|
*/
|
|
39
40
|
function createdHere(text) {
|
|
40
41
|
const names = new Set();
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
// The frameworks say "create this table" in their own words, and the suppression is worth nothing
|
|
43
|
+
// in a dialect it cannot read. Only SQL was read, so bitwarden/server's 456 EF Core migrations -
|
|
44
|
+
// which do exactly what Prisma does, a CreateTable followed by a CreateIndex for every relation -
|
|
45
|
+
// produced 647 "blocks writes" findings about tables that had not existed a statement earlier.
|
|
46
|
+
// EF Core puts the name on the line after the call, so these have to reach across a newline.
|
|
47
|
+
const FRAMEWORK_CREATES = [
|
|
48
|
+
/migrationBuilder\s*\.\s*CreateTable\s*\([\s\S]{0,80}?name\s*:\s*["']([\w$.]+)["']/gi,
|
|
49
|
+
/queryInterface\s*\.\s*createTable\s*\(\s*["']([\w$.]+)["']/gi,
|
|
50
|
+
/knex\s*\.\s*schema\s*\.\s*createTable\s*\(\s*["']([\w$.]+)["']/gi,
|
|
51
|
+
/Schema\s*::\s*create\s*\(\s*["']([\w$.]+)["']/gi,
|
|
52
|
+
/\bcreate_table\s+[:'"]([\w$.]+)/gi,
|
|
53
|
+
/\bop\s*\.\s*create_table\s*\(\s*["']([\w$.]+)["']/gi,
|
|
54
|
+
/<createTable[^>]*tableName\s*=\s*["']([\w$.]+)["']/gi,
|
|
55
|
+
];
|
|
56
|
+
for (const pattern of FRAMEWORK_CREATES)
|
|
57
|
+
for (const m of text.matchAll(pattern))
|
|
58
|
+
names.add(m[1].toLowerCase().replace(/^.*\./, ''));
|
|
59
|
+
// Every dialect's way of quoting a name, because the suppression is useless in the one that does
|
|
60
|
+
// not match. This read PostgreSQL's "quoted" form and plain names, and not SQL Server's
|
|
61
|
+
// [dbo].[Grant] - which is standard T-SQL and what bitwarden/server is written in throughout. No
|
|
62
|
+
// table was ever recognised as freshly created there, so every index built on a table created two
|
|
63
|
+
// statements earlier was reported as blocking writes: 714 findings, nearly all of them free.
|
|
64
|
+
for (const m of text.matchAll(/\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w$.\"\[\]`]+)/gi)) {
|
|
65
|
+
// `[dbo].[Grant]` and `"public"."grant"` and `grant` all have to arrive here as the same word,
|
|
66
|
+
// because that is the form the findings report a table in.
|
|
67
|
+
names.add(m[1].replace(/[\"\[\]`]/g, '').toLowerCase().replace(/^.*\./, ''));
|
|
43
68
|
}
|
|
44
69
|
return names;
|
|
45
70
|
}
|
|
@@ -54,7 +79,7 @@ export function droppingWhatIsNew(findings, text) {
|
|
|
54
79
|
export function readMigration(root, file) {
|
|
55
80
|
let text = '';
|
|
56
81
|
try {
|
|
57
|
-
text = readFileSync(resolve(root, file), 'utf8');
|
|
82
|
+
text = normalise(readFileSync(resolve(root, file), 'utf8'));
|
|
58
83
|
}
|
|
59
84
|
catch {
|
|
60
85
|
return { file, dialect: 'unknown', findings: [], unread: 'the file could not be read', irreversible: false };
|
|
@@ -239,7 +239,14 @@ function bracedTests(language, text) {
|
|
|
239
239
|
// An annotation disables a test outright. A `t.Skip()` may be conditional - cobra's
|
|
240
240
|
// TestDeadcodeElimination skips on Windows and runs everywhere else - so it counts only when it
|
|
241
241
|
// is unconditional, at the top level of the body.
|
|
242
|
-
|
|
242
|
+
// xUnit - the most used .NET framework by a distance - has no separate attribute for this. It
|
|
243
|
+
// skips by giving the test attribute a reason: `[Fact(Skip = "...")]`, `[Theory(Skip = ...)]`.
|
|
244
|
+
// Only `[Ignore]` was read, which is NUnit and MSTest, so bitwarden/server's **43 skipped tests
|
|
245
|
+
// were reported as 1**. A team told their suite is sound while 43 of its tests never run is
|
|
246
|
+
// worse off than a team told nothing at all.
|
|
247
|
+
const skipped = /@(Disabled|Ignore)\b|\[\s*Ignore\b/.test(around) ||
|
|
248
|
+
/\[\s*\w*(?:Fact|Theory)\s*\([^)]*\bSkip\s*=/.test(around) ||
|
|
249
|
+
skipsUnconditionally(body);
|
|
243
250
|
found.push({ name, line: lineAt(text, at), body, skipped, callback: '' });
|
|
244
251
|
}
|
|
245
252
|
return found;
|
|
@@ -253,7 +260,12 @@ function skipsUnconditionally(body) {
|
|
|
253
260
|
let depth = 0;
|
|
254
261
|
for (const line of body.split(String.fromCharCode(10))) {
|
|
255
262
|
const text = line.trim();
|
|
256
|
-
|
|
263
|
+
// `t.Skip(...)` in Go: a short receiver, and a whole statement with nothing chained onto it.
|
|
264
|
+
// `\w+\.Skip(` matched any identifier at all, so C#'s LINQ - `collections.Skip(1).ToList()`,
|
|
265
|
+
// which is how .NET pages a sequence - read as a test turning itself off. bitwarden/server has
|
|
266
|
+
// one, and it was the only never-runs Tewip reported there until xUnit's `[Fact(Skip = ...)]`
|
|
267
|
+
// was understood; the real number was 43 and the one it had was wrong.
|
|
268
|
+
if (depth === 0 && /^\w{1,3}\s*\.\s*Skip(f|Now)?\s*\([^)]*\)\s*;?\s*$/.test(text))
|
|
257
269
|
return true;
|
|
258
270
|
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
259
271
|
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Whether a test's own verdict is caught and thrown away.
|
|
2
|
+
//
|
|
3
|
+
// Split out of vouch.ts in Phase 44. It is the rule that has been wrong most often and most
|
|
4
|
+
// expensively - 166 false accusations in Phase 43, two more in Phase 44 - so it is worth having
|
|
5
|
+
// where it can be read on its own, with the reasoning that corrected it attached.
|
|
6
|
+
import { checksSomething } from './assertions.js';
|
|
7
|
+
import { balanced, withoutComments } from './text.js';
|
|
8
|
+
/**
|
|
9
|
+
* On the JVM, the only catches that can swallow a failing assertion.
|
|
10
|
+
*
|
|
11
|
+
* `Assert.fail()` and every JUnit matcher throw `AssertionError`, which extends `Error`, which
|
|
12
|
+
* extends `Throwable` - and `Error` is **not** an `Exception`. So `catch (Exception e) { }` does not
|
|
13
|
+
* catch a failing assertion, and neither does `catch (NotFoundException e) { }`. Only these do.
|
|
14
|
+
*
|
|
15
|
+
* This is not a detail. It is the difference between the standard Java way of writing "I expect this
|
|
16
|
+
* to throw" -
|
|
17
|
+
*
|
|
18
|
+
* try { parse(bad); Assert.fail(); } catch (VerificationException expected) { }
|
|
19
|
+
*
|
|
20
|
+
* - which is a correct test that fails properly when `parse` stops throwing, and a test that really
|
|
21
|
+
* does discard its own verdict. Tewip could not tell them apart and called **166 correct Keycloak
|
|
22
|
+
* tests worthless**, every one of them this idiom. Telling a developer their working test is
|
|
23
|
+
* worthless is the one failure this project cannot afford, so the caught type is now read.
|
|
24
|
+
*
|
|
25
|
+
* C# is deliberately not here: NUnit, xUnit and MSTest all throw assertion failures that derive from
|
|
26
|
+
* `Exception`, so `catch (Exception)` really does swallow them there. Same shape, opposite answer.
|
|
27
|
+
*/
|
|
28
|
+
const JVM_SWALLOWS = /^(?:java\.lang\.)?(?:Throwable|Error|AssertionError)$/;
|
|
29
|
+
/**
|
|
30
|
+
* The `try` block that belongs to a `catch`, so the question can be asked about the right code.
|
|
31
|
+
*
|
|
32
|
+
* Walks back from the `catch` over the `}` that closes the try and finds its `{` by counting.
|
|
33
|
+
*/
|
|
34
|
+
function tryBlockBefore(body, catchAt) {
|
|
35
|
+
const close = body.lastIndexOf('}', catchAt);
|
|
36
|
+
if (close === -1)
|
|
37
|
+
return '';
|
|
38
|
+
let depth = 0;
|
|
39
|
+
for (let i = close; i >= 0; i -= 1) {
|
|
40
|
+
if (body[i] === '}')
|
|
41
|
+
depth += 1;
|
|
42
|
+
else if (body[i] === '{') {
|
|
43
|
+
depth -= 1;
|
|
44
|
+
if (depth === 0)
|
|
45
|
+
return body.slice(i + 1, close);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return '';
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Everything inside a `try` whose `catch` discards what it caught. An assertion failure is an
|
|
52
|
+
* exception, so a test shaped this way reports success however the code behaves - and a `catch` that
|
|
53
|
+
* asserts, re-throws or fails the test is the legitimate version, which this leaves alone.
|
|
54
|
+
*/
|
|
55
|
+
export function swallowsFailure(body, checks, language) {
|
|
56
|
+
if (!checks)
|
|
57
|
+
return ''; // nothing to swallow; judged as no-assertion instead
|
|
58
|
+
for (const hit of body.matchAll(/catch\s*(\([^)]*\))?\s*\{/g)) {
|
|
59
|
+
// On the JVM the caught type decides it, and nearly every type leaves the assertion alone.
|
|
60
|
+
if (language === 'jvm') {
|
|
61
|
+
const caught = /\(\s*(?:final\s+)?([\w.]+)/.exec(hit[1] ?? '');
|
|
62
|
+
// A multi-catch - `catch (A | B e)` - swallows only if one of its types does.
|
|
63
|
+
const types = (hit[1] ?? '').replace(/[()]/g, '').split('|').map((t) => t.trim().split(/\s+/)[0]);
|
|
64
|
+
if (!types.some((t) => JVM_SWALLOWS.test(t)) && !(caught && JVM_SWALLOWS.test(caught[1])))
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
// The catch has to be able to swallow *this test's verdict*, and it can only do that when the
|
|
68
|
+
// assertion is inside its own try. Keycloak's admin-client tests open with
|
|
69
|
+
//
|
|
70
|
+
// try { await delByName({ name }); } catch { /* ignore */ }
|
|
71
|
+
//
|
|
72
|
+
// to make sure the scope does not already exist, and then assert afterwards, outside it. The
|
|
73
|
+
// rule asked "does this test assert anywhere, and is there an empty catch anywhere" and so
|
|
74
|
+
// called three correct tests worthless. A try around setup is not a swallowed verdict.
|
|
75
|
+
const guarded = tryBlockBefore(body, hit.index ?? 0);
|
|
76
|
+
if (guarded !== '' && !checksSomething(guarded))
|
|
77
|
+
continue;
|
|
78
|
+
const open = body.indexOf('{', hit.index ?? 0);
|
|
79
|
+
const handler = balanced(body, open);
|
|
80
|
+
if (handler === null)
|
|
81
|
+
continue;
|
|
82
|
+
const meaningful = withoutComments(handler).trim();
|
|
83
|
+
const rescues = checksSomething(handler) || /\b(throw|raise|rethrow|fail|t\s*\.\s*(Error|Fatal)|pytest\s*\.\s*fail)\b/.test(handler);
|
|
84
|
+
if (rescues)
|
|
85
|
+
continue;
|
|
86
|
+
// A handler that only logs, or does nothing at all, discards the failure.
|
|
87
|
+
if (meaningful === '' || /^(console\s*\.\s*\w+\s*\([^;]*\)\s*;?|print\s*\([^;]*\)\s*;?|pass|;)$/.test(meaningful.replace(/\s+/g, ' ').trim())) {
|
|
88
|
+
return meaningful === '' ? 'catch { }' : meaningful.slice(0, 120);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return '';
|
|
92
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Small text operations the vouch rules share.
|
|
2
|
+
//
|
|
3
|
+
// Split out in Phase 44 so that `swallowed.ts` could use them without importing the module that
|
|
4
|
+
// imports it. Nothing here knows what a test is; they are string surgery and nothing more.
|
|
5
|
+
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
|
6
|
+
export function withoutComments(body) {
|
|
7
|
+
return body
|
|
8
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
9
|
+
.split(String.fromCharCode(10))
|
|
10
|
+
.map((line) => line.replace(/(^|\s)\/\/.*$/, '').replace(/(^|\s)#(?!\{).*$/, ''))
|
|
11
|
+
.join(String.fromCharCode(10));
|
|
12
|
+
}
|
|
13
|
+
export const stripLines = (body, lines) => body
|
|
14
|
+
.split(String.fromCharCode(10))
|
|
15
|
+
.filter((line) => !lines.some((drop) => line.trim() === drop.trim()))
|
|
16
|
+
.join(String.fromCharCode(10));
|
|
17
|
+
/** The first line that does something, for the report to quote. */
|
|
18
|
+
export function firstRealLine(body) {
|
|
19
|
+
const lines = body
|
|
20
|
+
.split(String.fromCharCode(10))
|
|
21
|
+
.map((l) => l.trim())
|
|
22
|
+
.filter((l) => l !== '' && l !== '{' && l !== '}');
|
|
23
|
+
// A comment is not the evidence. Saleor's tests are written `# given` / `# when / then`, so the
|
|
24
|
+
// quoted evidence for 166 real findings was the single character "#", which tells a reader
|
|
25
|
+
// nothing about the test it is accusing. The line that does something is the evidence.
|
|
26
|
+
// Nor is a line with nothing in it but punctuation: a lone quote closing a docstring, a stray
|
|
27
|
+
// bracket. Evidence has to be something a reader can recognise as the test doing work.
|
|
28
|
+
const code = lines.find((l) => !/^(#|\/\/|\/\*|\*|"""|'''|--)/.test(l) && /[A-Za-z0-9]/.test(l));
|
|
29
|
+
return (code ?? lines[0] ?? '').slice(0, 120);
|
|
30
|
+
}
|
|
31
|
+
export const balanced = (text, open) => {
|
|
32
|
+
if (open === -1)
|
|
33
|
+
return null;
|
|
34
|
+
let depth = 0;
|
|
35
|
+
for (let i = open; i < text.length; i += 1) {
|
|
36
|
+
if (text[i] === '{')
|
|
37
|
+
depth += 1;
|
|
38
|
+
else if (text[i] === '}') {
|
|
39
|
+
depth -= 1;
|
|
40
|
+
if (depth === 0)
|
|
41
|
+
return text.slice(open + 1, i);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// Tewip would not judge - and it ends with what it refused, because that is what makes the rest of
|
|
6
6
|
// it believable.
|
|
7
7
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
8
|
+
import { normalise } from '../quality/source-text.js';
|
|
8
9
|
import { join, relative } from 'node:path';
|
|
9
10
|
import { CANNOT_FAIL, vouchFile } from './vouch.js';
|
|
10
11
|
const SKIP = new Set(['node_modules', '.git', 'dist', 'build', 'coverage', 'vendor', 'target', '.venv', '__pycache__', '.tewip']);
|
|
@@ -57,7 +58,7 @@ export function vouchAll(root, paths) {
|
|
|
57
58
|
for (const file of testFiles(root, paths)) {
|
|
58
59
|
let text;
|
|
59
60
|
try {
|
|
60
|
-
text = readFileSync(file, 'utf8');
|
|
61
|
+
text = normalise(readFileSync(file, 'utf8'));
|
|
61
62
|
}
|
|
62
63
|
catch {
|
|
63
64
|
continue;
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
// their time and their trust, and a reviewer who is wrong once is ignored afterwards. So a body whose
|
|
10
10
|
// checking happens somewhere this cannot follow is reported as undecided, by name, and never as
|
|
11
11
|
// vacuous.
|
|
12
|
+
import { balanced, firstRealLine, stripLines, withoutComments } from './text.js';
|
|
13
|
+
import { swallowsFailure } from './swallowed.js';
|
|
12
14
|
import { checksSomething, tautologies, unfinishedExpectations } from './assertions.js';
|
|
13
15
|
import { languageOf, looksLikeTests, testsIn, textRanges } from './tests-in.js';
|
|
14
16
|
/** Everything a verdict other than `checks` means, in the words the report uses. */
|
|
@@ -129,59 +131,6 @@ function withoutText(body) {
|
|
|
129
131
|
}
|
|
130
132
|
return out;
|
|
131
133
|
}
|
|
132
|
-
/**
|
|
133
|
-
* On the JVM, the only catches that can swallow a failing assertion.
|
|
134
|
-
*
|
|
135
|
-
* `Assert.fail()` and every JUnit matcher throw `AssertionError`, which extends `Error`, which
|
|
136
|
-
* extends `Throwable` - and `Error` is **not** an `Exception`. So `catch (Exception e) { }` does not
|
|
137
|
-
* catch a failing assertion, and neither does `catch (NotFoundException e) { }`. Only these do.
|
|
138
|
-
*
|
|
139
|
-
* This is not a detail. It is the difference between the standard Java way of writing "I expect this
|
|
140
|
-
* to throw" -
|
|
141
|
-
*
|
|
142
|
-
* try { parse(bad); Assert.fail(); } catch (VerificationException expected) { }
|
|
143
|
-
*
|
|
144
|
-
* - which is a correct test that fails properly when `parse` stops throwing, and a test that really
|
|
145
|
-
* does discard its own verdict. Tewip could not tell them apart and called **166 correct Keycloak
|
|
146
|
-
* tests worthless**, every one of them this idiom. Telling a developer their working test is
|
|
147
|
-
* worthless is the one failure this project cannot afford, so the caught type is now read.
|
|
148
|
-
*
|
|
149
|
-
* C# is deliberately not here: NUnit, xUnit and MSTest all throw assertion failures that derive from
|
|
150
|
-
* `Exception`, so `catch (Exception)` really does swallow them there. Same shape, opposite answer.
|
|
151
|
-
*/
|
|
152
|
-
const JVM_SWALLOWS = /^(?:java\.lang\.)?(?:Throwable|Error|AssertionError)$/;
|
|
153
|
-
/**
|
|
154
|
-
* Everything inside a `try` whose `catch` discards what it caught. An assertion failure is an
|
|
155
|
-
* exception, so a test shaped this way reports success however the code behaves - and a `catch` that
|
|
156
|
-
* asserts, re-throws or fails the test is the legitimate version, which this leaves alone.
|
|
157
|
-
*/
|
|
158
|
-
function swallowsFailure(body, checks, language) {
|
|
159
|
-
if (!checks)
|
|
160
|
-
return ''; // nothing to swallow; judged as no-assertion instead
|
|
161
|
-
for (const hit of body.matchAll(/catch\s*(\([^)]*\))?\s*\{/g)) {
|
|
162
|
-
// On the JVM the caught type decides it, and nearly every type leaves the assertion alone.
|
|
163
|
-
if (language === 'jvm') {
|
|
164
|
-
const caught = /\(\s*(?:final\s+)?([\w.]+)/.exec(hit[1] ?? '');
|
|
165
|
-
// A multi-catch - `catch (A | B e)` - swallows only if one of its types does.
|
|
166
|
-
const types = (hit[1] ?? '').replace(/[()]/g, '').split('|').map((t) => t.trim().split(/\s+/)[0]);
|
|
167
|
-
if (!types.some((t) => JVM_SWALLOWS.test(t)) && !(caught && JVM_SWALLOWS.test(caught[1])))
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
const open = body.indexOf('{', hit.index ?? 0);
|
|
171
|
-
const handler = balanced(body, open);
|
|
172
|
-
if (handler === null)
|
|
173
|
-
continue;
|
|
174
|
-
const meaningful = withoutComments(handler).trim();
|
|
175
|
-
const rescues = checksSomething(handler) || /\b(throw|raise|rethrow|fail|t\s*\.\s*(Error|Fatal)|pytest\s*\.\s*fail)\b/.test(handler);
|
|
176
|
-
if (rescues)
|
|
177
|
-
continue;
|
|
178
|
-
// A handler that only logs, or does nothing at all, discards the failure.
|
|
179
|
-
if (meaningful === '' || /^(console\s*\.\s*\w+\s*\([^;]*\)\s*;?|print\s*\([^;]*\)\s*;?|pass|;)$/.test(meaningful.replace(/\s+/g, ' ').trim())) {
|
|
180
|
-
return meaningful === '' ? 'catch { }' : meaningful.slice(0, 120);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
return '';
|
|
184
|
-
}
|
|
185
134
|
/** Functions defined in this file that assert, so a test calling one really is checking something. */
|
|
186
135
|
function helpersThatCheck(text, language) {
|
|
187
136
|
const names = new Set();
|
|
@@ -216,44 +165,4 @@ function indentedFrom(text, from) {
|
|
|
216
165
|
}
|
|
217
166
|
/** Names that are calls but not helpers: language and framework machinery. */
|
|
218
167
|
const LANGUAGE_NOISE = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'await', 'typeof', 'new', 'require', 'import', 'print', 'console', 'log', 'describe', 'beforeEach', 'afterEach', 'it', 'test', 'range', 'len', 'str', 'int', 'super', 'Error', 'Promise', 'Array', 'Object', 'String', 'Number', 'Boolean', 'Date', 'JSON', 'Math', 'Map', 'Set']);
|
|
219
|
-
|
|
220
|
-
if (open === -1)
|
|
221
|
-
return null;
|
|
222
|
-
let depth = 0;
|
|
223
|
-
for (let i = open; i < text.length; i += 1) {
|
|
224
|
-
if (text[i] === '{')
|
|
225
|
-
depth += 1;
|
|
226
|
-
else if (text[i] === '}') {
|
|
227
|
-
depth -= 1;
|
|
228
|
-
if (depth === 0)
|
|
229
|
-
return text.slice(open + 1, i);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return null;
|
|
233
|
-
};
|
|
234
|
-
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
|
235
|
-
function withoutComments(body) {
|
|
236
|
-
return body
|
|
237
|
-
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
238
|
-
.split(String.fromCharCode(10))
|
|
239
|
-
.map((line) => line.replace(/(^|\s)\/\/.*$/, '').replace(/(^|\s)#(?!\{).*$/, ''))
|
|
240
|
-
.join(String.fromCharCode(10));
|
|
241
|
-
}
|
|
242
|
-
const stripLines = (body, lines) => body
|
|
243
|
-
.split(String.fromCharCode(10))
|
|
244
|
-
.filter((line) => !lines.some((drop) => line.trim() === drop.trim()))
|
|
245
|
-
.join(String.fromCharCode(10));
|
|
246
|
-
/** The first line that does something, for the report to quote. */
|
|
247
|
-
function firstRealLine(body) {
|
|
248
|
-
const lines = body
|
|
249
|
-
.split(String.fromCharCode(10))
|
|
250
|
-
.map((l) => l.trim())
|
|
251
|
-
.filter((l) => l !== '' && l !== '{' && l !== '}');
|
|
252
|
-
// A comment is not the evidence. Saleor's tests are written `# given` / `# when / then`, so the
|
|
253
|
-
// quoted evidence for 166 real findings was the single character "#", which tells a reader
|
|
254
|
-
// nothing about the test it is accusing. The line that does something is the evidence.
|
|
255
|
-
// Nor is a line with nothing in it but punctuation: a lone quote closing a docstring, a stray
|
|
256
|
-
// bracket. Evidence has to be something a reader can recognise as the test doing work.
|
|
257
|
-
const code = lines.find((l) => !/^(#|\/\/|\/\*|\*|"""|'''|--)/.test(l) && /[A-Za-z0-9]/.test(l));
|
|
258
|
-
return (code ?? lines[0] ?? '').slice(0, 120);
|
|
259
|
-
}
|
|
168
|
+
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tewip-ai",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.54.0",
|
|
4
4
|
"mcpName": "io.github.hmamut39/tewip",
|
|
5
5
|
"description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
|
|
6
6
|
"keywords": [
|