tewip-ai 0.51.0 → 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/access/access-command.js +2 -1
- package/dist/engine/quality/source-text.js +29 -0
- package/dist/engine/schema/migration-files.js +79 -0
- package/dist/engine/schema/schema-command.js +4 -13
- package/dist/engine/vouch/assertions.js +7 -0
- package/dist/engine/vouch/swallowed.js +92 -0
- package/dist/engine/vouch/text.js +45 -0
- package/dist/engine/vouch/vouch-command.js +2 -1
- package/dist/engine/vouch/vouch.js +3 -94
- package/package.json +4 -2
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// It never says a page is accessible. It says what was checked.
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { normalise } from '../quality/source-text.js';
|
|
12
13
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
13
14
|
import { sourceFiles } from '../secure/secrets.js';
|
|
14
15
|
import { JS_LIKE, MARKUP, tags } from './markup.js';
|
|
@@ -50,7 +51,7 @@ export function findBarriers(root, only) {
|
|
|
50
51
|
for (const path of files) {
|
|
51
52
|
let source;
|
|
52
53
|
try {
|
|
53
|
-
source = readFileSync(path, 'utf8');
|
|
54
|
+
source = normalise(readFileSync(path, 'utf8'));
|
|
54
55
|
}
|
|
55
56
|
catch {
|
|
56
57
|
continue;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Reading somebody else's file, the same way on every machine.
|
|
2
|
+
//
|
|
3
|
+
// Found by the corpus harness on its first run in CI, Phase 44: the same pinned commit of Keycloak
|
|
4
|
+
// produced 0 swallows-failure findings on Windows and 3 on Linux, and OpenMRS differed by seven
|
|
5
|
+
// crash-only tests. Git for Windows checks source out with CRLF, so the text Tewip judged was not
|
|
6
|
+
// the text the project wrote, and a scanner whose answer depends on which machine ran it is not
|
|
7
|
+
// giving an answer at all - it is giving an opinion with a hostname attached.
|
|
8
|
+
//
|
|
9
|
+
// The carriage returns go at the boundary rather than in each rule, because the alternative is
|
|
10
|
+
// remembering `\r?` in every pattern anybody writes afterwards, and one forgotten `\r?` is this bug
|
|
11
|
+
// again. Line numbers are unaffected: a CRLF file has exactly as many newlines as an LF one.
|
|
12
|
+
//
|
|
13
|
+
// A BOM goes too. It is invisible, it sits before the first character of the file, and it stops a
|
|
14
|
+
// pattern anchored at the start of the text from matching the first line.
|
|
15
|
+
import { readFileSync } from 'node:fs';
|
|
16
|
+
/** The file's text with line endings normalised, or '' when it cannot be read. */
|
|
17
|
+
export function sourceText(file) {
|
|
18
|
+
try {
|
|
19
|
+
return normalise(readFileSync(file, 'utf8'));
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return '';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/** Normalised, for text that has already been read. */
|
|
26
|
+
export function normalise(text) {
|
|
27
|
+
const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
28
|
+
return withoutBom.includes(String.fromCharCode(13)) ? withoutBom.split(String.fromCharCode(13) + String.fromCharCode(10)).join(String.fromCharCode(10)) : withoutBom;
|
|
29
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Which files in a project are migrations at all.
|
|
2
|
+
//
|
|
3
|
+
// Split out of schema-command.ts in Phase 44, when the corpus harness needed to ask the question
|
|
4
|
+
// across a whole checkout rather than across a diff and that file went past the 300-line rule.
|
|
5
|
+
// Keeping it in one place is the point: the alternative was a copy of these patterns in a script,
|
|
6
|
+
// and two definitions of "is this a migration" drift the moment one of them is fixed.
|
|
7
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
8
|
+
import { join, relative } from 'node:path';
|
|
9
|
+
// A directory called `migration/` says what is probably in it, not what every file in it is. This
|
|
10
|
+
// matched *anything* underneath one, so Keycloak's `docs/guides/migration/index.adoc` and its 113
|
|
11
|
+
// `migration/migrators/MigrateTo12_0_0.java` classes - application code that moves data between
|
|
12
|
+
// Keycloak versions, nothing to do with a schema file - were selected as migrations and then listed
|
|
13
|
+
// as ones Tewip could not read. 205 files chosen in order to read 89. A "could not read" list that
|
|
14
|
+
// is mostly documentation teaches the reader to skip the list.
|
|
15
|
+
//
|
|
16
|
+
// So the directory decides *where*, and the extension decides *whether it could hold DDL at all*.
|
|
17
|
+
const MIGRATION_DIR = /(^|\/)(migrations?|migration|db\/migrate|prisma\/migrations|alembic|flyway|changelogs?|sql)\//i;
|
|
18
|
+
const COULD_HOLD_DDL = /\.(sql|py|rb|js|mjs|cjs|ts|php|cs|xml|ya?ml|json|go|exs?)$/i;
|
|
19
|
+
// Flyway's Java migrations are real, and they are named the way Flyway requires: `V2__add_index.java`
|
|
20
|
+
// or `R__rebuild_views.java`. Keycloak's `MigrateTo12_0_0.java` is not one. Taking `.java` on the
|
|
21
|
+
// strength of its directory alone is what filled that list with source files.
|
|
22
|
+
const FLYWAY_JAVA = /(^|\/)[VR][\w.]*__[\w.-]+\.(java|kt|scala)$/;
|
|
23
|
+
export const MIGRATION = {
|
|
24
|
+
test: (path) => MIGRATION_DIR.test(path) && (COULD_HOLD_DDL.test(path) || FLYWAY_JAVA.test(path)),
|
|
25
|
+
};
|
|
26
|
+
// A changelog is often not in a directory that says so. OpenMRS keeps 44 Liquibase changelogs in
|
|
27
|
+
// `api/src/main/resources/` named `liquibase-*.xml`, and the path filter saw none of them - so a
|
|
28
|
+
// real healthcare system with 44 migration files was told it had none. The name counts too.
|
|
29
|
+
//
|
|
30
|
+
// And a changelog is named for what it is, but hardly ever *only* that. Phase 42 taught this filter
|
|
31
|
+
// to read `liquibase-update-to-latest.xml` and it still missed Keycloak's 83, every one called
|
|
32
|
+
// `jpa-changelog-<version>.xml`: the word had to *begin* the name. Prefixes are the norm once a
|
|
33
|
+
// project has more than one database. Letting a release-notes `CHANGELOG.xml` through costs nothing,
|
|
34
|
+
// because `dialectOf` still has to find `<databaseChangeLog` in it before it is read as a migration.
|
|
35
|
+
export const NAMED_MIGRATION = /(^|\/)[\w.-]*(liquibase|changelog)[\w.-]*\.(xml|ya?ml|json)$/i;
|
|
36
|
+
export const SCHEMA_FILE = /\.(sql)$|(^|\/)(migrations?|migration|db\/migrate)\/.*\.(py|rb|js|ts|php|cs)$/i;
|
|
37
|
+
/**
|
|
38
|
+
* Every migration in a whole repository, by the same filters a diff is read through.
|
|
39
|
+
*
|
|
40
|
+
* `schemaFiles` answers "which of these changed files are migrations", which is the question a pull
|
|
41
|
+
* request asks. "What migrations does this project have at all" is a different question, and the
|
|
42
|
+
* corpus harness needs it to re-derive a verified number across a pinned checkout.
|
|
43
|
+
*/
|
|
44
|
+
export function migrationsUnder(root, limit = 20_000) {
|
|
45
|
+
const skip = new Set(['node_modules', '.git', 'dist', 'build', 'target', 'vendor', '__pycache__', '.venv', 'venv', '.next', '.cache']);
|
|
46
|
+
const found = [];
|
|
47
|
+
const walk = (where, depth) => {
|
|
48
|
+
if (depth > 12 || found.length >= limit)
|
|
49
|
+
return;
|
|
50
|
+
let entries;
|
|
51
|
+
try {
|
|
52
|
+
entries = readdirSync(where);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
for (const entry of entries) {
|
|
58
|
+
if (skip.has(entry) || entry.startsWith('.'))
|
|
59
|
+
continue;
|
|
60
|
+
const full = join(where, entry);
|
|
61
|
+
let info;
|
|
62
|
+
try {
|
|
63
|
+
info = statSync(full);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (info.isDirectory()) {
|
|
69
|
+
walk(full, depth + 1);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const rel = relative(root, full).replaceAll(String.fromCharCode(92), '/');
|
|
73
|
+
if (SCHEMA_FILE.test(rel) || MIGRATION.test(rel) || NAMED_MIGRATION.test(rel))
|
|
74
|
+
found.push(rel);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
walk(root, 0);
|
|
78
|
+
return found.sort();
|
|
79
|
+
}
|
|
@@ -9,22 +9,13 @@
|
|
|
9
9
|
// such. A mention in a comment counts as a mention; Tewip says where it looked and lets the reader judge.
|
|
10
10
|
import { spawnSync } from 'node:child_process';
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { normalise } from '../quality/source-text.js';
|
|
12
13
|
import { dirname, resolve } from 'node:path';
|
|
13
14
|
import { changeSummary } from '../quality/grounding.js';
|
|
14
15
|
import { dialectOf, forwardOnly, irreversible, readFramework } from './frameworks.js';
|
|
15
16
|
import { readSql } from './statements.js';
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
// `api/src/main/resources/` named `liquibase-*.xml`, and the path filter saw none of them - so a
|
|
19
|
-
// real healthcare system with 44 migration files was told it had none. The name counts too.
|
|
20
|
-
// A changelog is named for what it is, but hardly ever *only* that. Phase 42 taught this filter to
|
|
21
|
-
// read `liquibase-update-to-latest.xml` and it still missed Keycloak's 83 files, every one of them
|
|
22
|
-
// called `jpa-changelog-<version>.xml`: the name was required to *begin* with the word. Prefixes are
|
|
23
|
-
// the norm - a project with more than one database names each changelog after its module. So the
|
|
24
|
-
// word may sit anywhere in the name. Letting a release-notes `CHANGELOG.xml` through costs nothing,
|
|
25
|
-
// because `dialectOf` still has to find `<databaseChangeLog` in it before it is read as a migration.
|
|
26
|
-
const NAMED_MIGRATION = /(^|\/)[\w.-]*(liquibase|changelog)[\w.-]*\.(xml|ya?ml|json)$/i;
|
|
27
|
-
const SCHEMA_FILE = /\.(sql)$|(^|\/)(migrations?|migration|db\/migrate)\/.*\.(py|rb|js|ts|php|cs)$/i;
|
|
17
|
+
import { MIGRATION, NAMED_MIGRATION, SCHEMA_FILE, migrationsUnder } from './migration-files.js';
|
|
18
|
+
export { migrationsUnder };
|
|
28
19
|
/** The schema files in this change. */
|
|
29
20
|
export function schemaFiles(cmd) {
|
|
30
21
|
const named = cmd.files !== undefined && cmd.files.length > 0;
|
|
@@ -64,7 +55,7 @@ export function droppingWhatIsNew(findings, text) {
|
|
|
64
55
|
export function readMigration(root, file) {
|
|
65
56
|
let text = '';
|
|
66
57
|
try {
|
|
67
|
-
text = readFileSync(resolve(root, file), 'utf8');
|
|
58
|
+
text = normalise(readFileSync(resolve(root, file), 'utf8'));
|
|
68
59
|
}
|
|
69
60
|
catch {
|
|
70
61
|
return { file, dialect: 'unknown', findings: [], unread: 'the file could not be read', irreversible: false };
|
|
@@ -30,6 +30,13 @@ const CHECKS = [
|
|
|
30
30
|
/\bpytest\s*\.\s*(raises|warns|approx|fail)\b/,
|
|
31
31
|
/\bnp\s*\.\s*testing\s*\.\s*assert/,
|
|
32
32
|
/\bassert_(frame|series|index)_equal\b/,
|
|
33
|
+
// A bare `fail(...)`, which is how JUnit, TestNG and Go spell "this must not be reached" once the
|
|
34
|
+
// assertion is statically imported. `Assert.fail(...)` was already read by the pattern below;
|
|
35
|
+
// `fail(...)` on its own was not, so OpenMRS's RefByUuidTest - whose only real check is a `fail()`
|
|
36
|
+
// in the catch, beside a deliberate `assertTrue(true)` in the try - was called a tautology that
|
|
37
|
+
// "no change to the code can falsify". Make `getRefByUuid` throw and it falsifies immediately.
|
|
38
|
+
// The leading class keeps `shouldFail(` and `canFail(` out, and the paren must come next.
|
|
39
|
+
/(^|[^\w.])(?:Assert(?:ions)?\s*\.\s*)?fail\s*\(/,
|
|
33
40
|
// Java, Kotlin, C#, Scala.
|
|
34
41
|
/\bAssert(ions)?\s*\./,
|
|
35
42
|
/\bassertThat\s*\(/,
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Whether a test's own verdict is caught and thrown away.
|
|
2
|
+
//
|
|
3
|
+
// Split out of vouch.ts in Phase 44. It is the rule that has been wrong most often and most
|
|
4
|
+
// expensively - 166 false accusations in Phase 43, two more in Phase 44 - so it is worth having
|
|
5
|
+
// where it can be read on its own, with the reasoning that corrected it attached.
|
|
6
|
+
import { checksSomething } from './assertions.js';
|
|
7
|
+
import { balanced, withoutComments } from './text.js';
|
|
8
|
+
/**
|
|
9
|
+
* On the JVM, the only catches that can swallow a failing assertion.
|
|
10
|
+
*
|
|
11
|
+
* `Assert.fail()` and every JUnit matcher throw `AssertionError`, which extends `Error`, which
|
|
12
|
+
* extends `Throwable` - and `Error` is **not** an `Exception`. So `catch (Exception e) { }` does not
|
|
13
|
+
* catch a failing assertion, and neither does `catch (NotFoundException e) { }`. Only these do.
|
|
14
|
+
*
|
|
15
|
+
* This is not a detail. It is the difference between the standard Java way of writing "I expect this
|
|
16
|
+
* to throw" -
|
|
17
|
+
*
|
|
18
|
+
* try { parse(bad); Assert.fail(); } catch (VerificationException expected) { }
|
|
19
|
+
*
|
|
20
|
+
* - which is a correct test that fails properly when `parse` stops throwing, and a test that really
|
|
21
|
+
* does discard its own verdict. Tewip could not tell them apart and called **166 correct Keycloak
|
|
22
|
+
* tests worthless**, every one of them this idiom. Telling a developer their working test is
|
|
23
|
+
* worthless is the one failure this project cannot afford, so the caught type is now read.
|
|
24
|
+
*
|
|
25
|
+
* C# is deliberately not here: NUnit, xUnit and MSTest all throw assertion failures that derive from
|
|
26
|
+
* `Exception`, so `catch (Exception)` really does swallow them there. Same shape, opposite answer.
|
|
27
|
+
*/
|
|
28
|
+
const JVM_SWALLOWS = /^(?:java\.lang\.)?(?:Throwable|Error|AssertionError)$/;
|
|
29
|
+
/**
|
|
30
|
+
* The `try` block that belongs to a `catch`, so the question can be asked about the right code.
|
|
31
|
+
*
|
|
32
|
+
* Walks back from the `catch` over the `}` that closes the try and finds its `{` by counting.
|
|
33
|
+
*/
|
|
34
|
+
function tryBlockBefore(body, catchAt) {
|
|
35
|
+
const close = body.lastIndexOf('}', catchAt);
|
|
36
|
+
if (close === -1)
|
|
37
|
+
return '';
|
|
38
|
+
let depth = 0;
|
|
39
|
+
for (let i = close; i >= 0; i -= 1) {
|
|
40
|
+
if (body[i] === '}')
|
|
41
|
+
depth += 1;
|
|
42
|
+
else if (body[i] === '{') {
|
|
43
|
+
depth -= 1;
|
|
44
|
+
if (depth === 0)
|
|
45
|
+
return body.slice(i + 1, close);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return '';
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Everything inside a `try` whose `catch` discards what it caught. An assertion failure is an
|
|
52
|
+
* exception, so a test shaped this way reports success however the code behaves - and a `catch` that
|
|
53
|
+
* asserts, re-throws or fails the test is the legitimate version, which this leaves alone.
|
|
54
|
+
*/
|
|
55
|
+
export function swallowsFailure(body, checks, language) {
|
|
56
|
+
if (!checks)
|
|
57
|
+
return ''; // nothing to swallow; judged as no-assertion instead
|
|
58
|
+
for (const hit of body.matchAll(/catch\s*(\([^)]*\))?\s*\{/g)) {
|
|
59
|
+
// On the JVM the caught type decides it, and nearly every type leaves the assertion alone.
|
|
60
|
+
if (language === 'jvm') {
|
|
61
|
+
const caught = /\(\s*(?:final\s+)?([\w.]+)/.exec(hit[1] ?? '');
|
|
62
|
+
// A multi-catch - `catch (A | B e)` - swallows only if one of its types does.
|
|
63
|
+
const types = (hit[1] ?? '').replace(/[()]/g, '').split('|').map((t) => t.trim().split(/\s+/)[0]);
|
|
64
|
+
if (!types.some((t) => JVM_SWALLOWS.test(t)) && !(caught && JVM_SWALLOWS.test(caught[1])))
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
// The catch has to be able to swallow *this test's verdict*, and it can only do that when the
|
|
68
|
+
// assertion is inside its own try. Keycloak's admin-client tests open with
|
|
69
|
+
//
|
|
70
|
+
// try { await delByName({ name }); } catch { /* ignore */ }
|
|
71
|
+
//
|
|
72
|
+
// to make sure the scope does not already exist, and then assert afterwards, outside it. The
|
|
73
|
+
// rule asked "does this test assert anywhere, and is there an empty catch anywhere" and so
|
|
74
|
+
// called three correct tests worthless. A try around setup is not a swallowed verdict.
|
|
75
|
+
const guarded = tryBlockBefore(body, hit.index ?? 0);
|
|
76
|
+
if (guarded !== '' && !checksSomething(guarded))
|
|
77
|
+
continue;
|
|
78
|
+
const open = body.indexOf('{', hit.index ?? 0);
|
|
79
|
+
const handler = balanced(body, open);
|
|
80
|
+
if (handler === null)
|
|
81
|
+
continue;
|
|
82
|
+
const meaningful = withoutComments(handler).trim();
|
|
83
|
+
const rescues = checksSomething(handler) || /\b(throw|raise|rethrow|fail|t\s*\.\s*(Error|Fatal)|pytest\s*\.\s*fail)\b/.test(handler);
|
|
84
|
+
if (rescues)
|
|
85
|
+
continue;
|
|
86
|
+
// A handler that only logs, or does nothing at all, discards the failure.
|
|
87
|
+
if (meaningful === '' || /^(console\s*\.\s*\w+\s*\([^;]*\)\s*;?|print\s*\([^;]*\)\s*;?|pass|;)$/.test(meaningful.replace(/\s+/g, ' ').trim())) {
|
|
88
|
+
return meaningful === '' ? 'catch { }' : meaningful.slice(0, 120);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return '';
|
|
92
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Small text operations the vouch rules share.
|
|
2
|
+
//
|
|
3
|
+
// Split out in Phase 44 so that `swallowed.ts` could use them without importing the module that
|
|
4
|
+
// imports it. Nothing here knows what a test is; they are string surgery and nothing more.
|
|
5
|
+
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
|
6
|
+
export function withoutComments(body) {
|
|
7
|
+
return body
|
|
8
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
9
|
+
.split(String.fromCharCode(10))
|
|
10
|
+
.map((line) => line.replace(/(^|\s)\/\/.*$/, '').replace(/(^|\s)#(?!\{).*$/, ''))
|
|
11
|
+
.join(String.fromCharCode(10));
|
|
12
|
+
}
|
|
13
|
+
export const stripLines = (body, lines) => body
|
|
14
|
+
.split(String.fromCharCode(10))
|
|
15
|
+
.filter((line) => !lines.some((drop) => line.trim() === drop.trim()))
|
|
16
|
+
.join(String.fromCharCode(10));
|
|
17
|
+
/** The first line that does something, for the report to quote. */
|
|
18
|
+
export function firstRealLine(body) {
|
|
19
|
+
const lines = body
|
|
20
|
+
.split(String.fromCharCode(10))
|
|
21
|
+
.map((l) => l.trim())
|
|
22
|
+
.filter((l) => l !== '' && l !== '{' && l !== '}');
|
|
23
|
+
// A comment is not the evidence. Saleor's tests are written `# given` / `# when / then`, so the
|
|
24
|
+
// quoted evidence for 166 real findings was the single character "#", which tells a reader
|
|
25
|
+
// nothing about the test it is accusing. The line that does something is the evidence.
|
|
26
|
+
// Nor is a line with nothing in it but punctuation: a lone quote closing a docstring, a stray
|
|
27
|
+
// bracket. Evidence has to be something a reader can recognise as the test doing work.
|
|
28
|
+
const code = lines.find((l) => !/^(#|\/\/|\/\*|\*|"""|'''|--)/.test(l) && /[A-Za-z0-9]/.test(l));
|
|
29
|
+
return (code ?? lines[0] ?? '').slice(0, 120);
|
|
30
|
+
}
|
|
31
|
+
export const balanced = (text, open) => {
|
|
32
|
+
if (open === -1)
|
|
33
|
+
return null;
|
|
34
|
+
let depth = 0;
|
|
35
|
+
for (let i = open; i < text.length; i += 1) {
|
|
36
|
+
if (text[i] === '{')
|
|
37
|
+
depth += 1;
|
|
38
|
+
else if (text[i] === '}') {
|
|
39
|
+
depth -= 1;
|
|
40
|
+
if (depth === 0)
|
|
41
|
+
return text.slice(open + 1, i);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// Tewip would not judge - and it ends with what it refused, because that is what makes the rest of
|
|
6
6
|
// it believable.
|
|
7
7
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
8
|
+
import { normalise } from '../quality/source-text.js';
|
|
8
9
|
import { join, relative } from 'node:path';
|
|
9
10
|
import { CANNOT_FAIL, vouchFile } from './vouch.js';
|
|
10
11
|
const SKIP = new Set(['node_modules', '.git', 'dist', 'build', 'coverage', 'vendor', 'target', '.venv', '__pycache__', '.tewip']);
|
|
@@ -57,7 +58,7 @@ export function vouchAll(root, paths) {
|
|
|
57
58
|
for (const file of testFiles(root, paths)) {
|
|
58
59
|
let text;
|
|
59
60
|
try {
|
|
60
|
-
text = readFileSync(file, 'utf8');
|
|
61
|
+
text = normalise(readFileSync(file, 'utf8'));
|
|
61
62
|
}
|
|
62
63
|
catch {
|
|
63
64
|
continue;
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
// their time and their trust, and a reviewer who is wrong once is ignored afterwards. So a body whose
|
|
10
10
|
// checking happens somewhere this cannot follow is reported as undecided, by name, and never as
|
|
11
11
|
// vacuous.
|
|
12
|
+
import { balanced, firstRealLine, stripLines, withoutComments } from './text.js';
|
|
13
|
+
import { swallowsFailure } from './swallowed.js';
|
|
12
14
|
import { checksSomething, tautologies, unfinishedExpectations } from './assertions.js';
|
|
13
15
|
import { languageOf, looksLikeTests, testsIn, textRanges } from './tests-in.js';
|
|
14
16
|
/** Everything a verdict other than `checks` means, in the words the report uses. */
|
|
@@ -129,59 +131,6 @@ function withoutText(body) {
|
|
|
129
131
|
}
|
|
130
132
|
return out;
|
|
131
133
|
}
|
|
132
|
-
/**
|
|
133
|
-
* On the JVM, the only catches that can swallow a failing assertion.
|
|
134
|
-
*
|
|
135
|
-
* `Assert.fail()` and every JUnit matcher throw `AssertionError`, which extends `Error`, which
|
|
136
|
-
* extends `Throwable` - and `Error` is **not** an `Exception`. So `catch (Exception e) { }` does not
|
|
137
|
-
* catch a failing assertion, and neither does `catch (NotFoundException e) { }`. Only these do.
|
|
138
|
-
*
|
|
139
|
-
* This is not a detail. It is the difference between the standard Java way of writing "I expect this
|
|
140
|
-
* to throw" -
|
|
141
|
-
*
|
|
142
|
-
* try { parse(bad); Assert.fail(); } catch (VerificationException expected) { }
|
|
143
|
-
*
|
|
144
|
-
* - which is a correct test that fails properly when `parse` stops throwing, and a test that really
|
|
145
|
-
* does discard its own verdict. Tewip could not tell them apart and called **166 correct Keycloak
|
|
146
|
-
* tests worthless**, every one of them this idiom. Telling a developer their working test is
|
|
147
|
-
* worthless is the one failure this project cannot afford, so the caught type is now read.
|
|
148
|
-
*
|
|
149
|
-
* C# is deliberately not here: NUnit, xUnit and MSTest all throw assertion failures that derive from
|
|
150
|
-
* `Exception`, so `catch (Exception)` really does swallow them there. Same shape, opposite answer.
|
|
151
|
-
*/
|
|
152
|
-
const JVM_SWALLOWS = /^(?:java\.lang\.)?(?:Throwable|Error|AssertionError)$/;
|
|
153
|
-
/**
|
|
154
|
-
* Everything inside a `try` whose `catch` discards what it caught. An assertion failure is an
|
|
155
|
-
* exception, so a test shaped this way reports success however the code behaves - and a `catch` that
|
|
156
|
-
* asserts, re-throws or fails the test is the legitimate version, which this leaves alone.
|
|
157
|
-
*/
|
|
158
|
-
function swallowsFailure(body, checks, language) {
|
|
159
|
-
if (!checks)
|
|
160
|
-
return ''; // nothing to swallow; judged as no-assertion instead
|
|
161
|
-
for (const hit of body.matchAll(/catch\s*(\([^)]*\))?\s*\{/g)) {
|
|
162
|
-
// On the JVM the caught type decides it, and nearly every type leaves the assertion alone.
|
|
163
|
-
if (language === 'jvm') {
|
|
164
|
-
const caught = /\(\s*(?:final\s+)?([\w.]+)/.exec(hit[1] ?? '');
|
|
165
|
-
// A multi-catch - `catch (A | B e)` - swallows only if one of its types does.
|
|
166
|
-
const types = (hit[1] ?? '').replace(/[()]/g, '').split('|').map((t) => t.trim().split(/\s+/)[0]);
|
|
167
|
-
if (!types.some((t) => JVM_SWALLOWS.test(t)) && !(caught && JVM_SWALLOWS.test(caught[1])))
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
const open = body.indexOf('{', hit.index ?? 0);
|
|
171
|
-
const handler = balanced(body, open);
|
|
172
|
-
if (handler === null)
|
|
173
|
-
continue;
|
|
174
|
-
const meaningful = withoutComments(handler).trim();
|
|
175
|
-
const rescues = checksSomething(handler) || /\b(throw|raise|rethrow|fail|t\s*\.\s*(Error|Fatal)|pytest\s*\.\s*fail)\b/.test(handler);
|
|
176
|
-
if (rescues)
|
|
177
|
-
continue;
|
|
178
|
-
// A handler that only logs, or does nothing at all, discards the failure.
|
|
179
|
-
if (meaningful === '' || /^(console\s*\.\s*\w+\s*\([^;]*\)\s*;?|print\s*\([^;]*\)\s*;?|pass|;)$/.test(meaningful.replace(/\s+/g, ' ').trim())) {
|
|
180
|
-
return meaningful === '' ? 'catch { }' : meaningful.slice(0, 120);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
return '';
|
|
184
|
-
}
|
|
185
134
|
/** Functions defined in this file that assert, so a test calling one really is checking something. */
|
|
186
135
|
function helpersThatCheck(text, language) {
|
|
187
136
|
const names = new Set();
|
|
@@ -216,44 +165,4 @@ function indentedFrom(text, from) {
|
|
|
216
165
|
}
|
|
217
166
|
/** Names that are calls but not helpers: language and framework machinery. */
|
|
218
167
|
const LANGUAGE_NOISE = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'await', 'typeof', 'new', 'require', 'import', 'print', 'console', 'log', 'describe', 'beforeEach', 'afterEach', 'it', 'test', 'range', 'len', 'str', 'int', 'super', 'Error', 'Promise', 'Array', 'Object', 'String', 'Number', 'Boolean', 'Date', 'JSON', 'Math', 'Map', 'Set']);
|
|
219
|
-
|
|
220
|
-
if (open === -1)
|
|
221
|
-
return null;
|
|
222
|
-
let depth = 0;
|
|
223
|
-
for (let i = open; i < text.length; i += 1) {
|
|
224
|
-
if (text[i] === '{')
|
|
225
|
-
depth += 1;
|
|
226
|
-
else if (text[i] === '}') {
|
|
227
|
-
depth -= 1;
|
|
228
|
-
if (depth === 0)
|
|
229
|
-
return text.slice(open + 1, i);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return null;
|
|
233
|
-
};
|
|
234
|
-
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
|
235
|
-
function withoutComments(body) {
|
|
236
|
-
return body
|
|
237
|
-
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
238
|
-
.split(String.fromCharCode(10))
|
|
239
|
-
.map((line) => line.replace(/(^|\s)\/\/.*$/, '').replace(/(^|\s)#(?!\{).*$/, ''))
|
|
240
|
-
.join(String.fromCharCode(10));
|
|
241
|
-
}
|
|
242
|
-
const stripLines = (body, lines) => body
|
|
243
|
-
.split(String.fromCharCode(10))
|
|
244
|
-
.filter((line) => !lines.some((drop) => line.trim() === drop.trim()))
|
|
245
|
-
.join(String.fromCharCode(10));
|
|
246
|
-
/** The first line that does something, for the report to quote. */
|
|
247
|
-
function firstRealLine(body) {
|
|
248
|
-
const lines = body
|
|
249
|
-
.split(String.fromCharCode(10))
|
|
250
|
-
.map((l) => l.trim())
|
|
251
|
-
.filter((l) => l !== '' && l !== '{' && l !== '}');
|
|
252
|
-
// A comment is not the evidence. Saleor's tests are written `# given` / `# when / then`, so the
|
|
253
|
-
// quoted evidence for 166 real findings was the single character "#", which tells a reader
|
|
254
|
-
// nothing about the test it is accusing. The line that does something is the evidence.
|
|
255
|
-
// Nor is a line with nothing in it but punctuation: a lone quote closing a docstring, a stray
|
|
256
|
-
// bracket. Evidence has to be something a reader can recognise as the test doing work.
|
|
257
|
-
const code = lines.find((l) => !/^(#|\/\/|\/\*|\*|"""|'''|--)/.test(l) && /[A-Za-z0-9]/.test(l));
|
|
258
|
-
return (code ?? lines[0] ?? '').slice(0, 120);
|
|
259
|
-
}
|
|
168
|
+
/** Comments cannot assert, and a commented-out check is the classic way a test stops checking. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tewip-ai",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.53.0",
|
|
4
4
|
"mcpName": "io.github.hmamut39/tewip",
|
|
5
5
|
"description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
|
|
6
6
|
"keywords": [
|
|
@@ -63,7 +63,9 @@
|
|
|
63
63
|
"typecheck": "tsc --noEmit",
|
|
64
64
|
"build": "tsc -p tsconfig.build.json",
|
|
65
65
|
"prepack": "npm run build",
|
|
66
|
-
"tewip": "node cli/tewip.ts"
|
|
66
|
+
"tewip": "node cli/tewip.ts",
|
|
67
|
+
"version": "node scripts/sync-version.ts && git add server.json",
|
|
68
|
+
"corpus": "node scripts/corpus.ts"
|
|
67
69
|
},
|
|
68
70
|
"license": "Apache-2.0",
|
|
69
71
|
"devDependencies": {
|