tewip-ai 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/cli/help.js +3 -2
- package/dist/engine/access/markup.js +1 -1
- package/dist/engine/vouch/assertions.js +28 -3
- package/dist/engine/vouch/find-tests.js +274 -0
- package/dist/engine/vouch/tests-in.js +38 -159
- package/dist/engine/vouch/vouch-command.js +4 -1
- package/dist/engine/vouch/vouch.js +11 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -701,6 +701,10 @@ tewip access src/components/Nav.tsx # or just one
|
|
|
701
701
|
tewip access --strict # fails the build on a barrier that stops somebody outright
|
|
702
702
|
```
|
|
703
703
|
|
|
704
|
+
Reads **HTML, JSX, TSX, Vue, Svelte, Astro, ERB, PHP** and the server-rendered template languages —
|
|
705
|
+
**Blade, Twig, Jinja, Razor, Handlebars, Liquid, Nunjucks, EJS** — which between them are most of the
|
|
706
|
+
web's actual HTML.
|
|
707
|
+
|
|
704
708
|
Accessibility is the one quality question that is also a legal one. The European Accessibility Act has
|
|
705
709
|
been enforceable since June 2025, and it lands on the people who already own testing.
|
|
706
710
|
|
|
@@ -995,6 +999,9 @@ tewip vouch tests/ # or a file, or the whole project
|
|
|
995
999
|
tewip vouch tests/ --strict # exits non-zero, for a merge gate
|
|
996
1000
|
```
|
|
997
1001
|
|
|
1002
|
+
Reads **JavaScript, TypeScript, Python, Java, Kotlin, Scala, Go, C#, Ruby, PHP and Rust** — and says
|
|
1003
|
+
so, rather than clearing a file it cannot read.
|
|
1004
|
+
|
|
998
1005
|
Most tests in a pull request are now written by an agent, and the question that decides whether they
|
|
999
1006
|
are worth anything goes unasked: **could this test ever fail?** Coverage counts a test that checks
|
|
1000
1007
|
nothing. CI goes green on it. A reviewer skims something plausible and approves it.
|
package/dist/cli/help.js
CHANGED
|
@@ -185,8 +185,9 @@ Usage:
|
|
|
185
185
|
that check nothing, compare a value with itself, leave an
|
|
186
186
|
expectation unfinished, swallow their own failure, or never run.
|
|
187
187
|
Where the checking is in a helper it cannot follow it says so
|
|
188
|
-
instead of accusing.
|
|
189
|
-
|
|
188
|
+
instead of accusing. Reads JavaScript, TypeScript, Python, Java,
|
|
189
|
+
Kotlin, Scala, Go, C#, Ruby, PHP and Rust. --strict exits
|
|
190
|
+
non-zero on a finding, for a gate. No model, no cost.
|
|
190
191
|
tewip trace [--only "<ticket or title>"] [--out trace.md]
|
|
191
192
|
Is this requirement covered, and is it passing right now? Joins
|
|
192
193
|
the plans Tewip wrote to the tests that cover them and the runs
|
|
@@ -171,4 +171,4 @@ export function labelledBy(tag) {
|
|
|
171
171
|
return { named: false, unknown: false };
|
|
172
172
|
}
|
|
173
173
|
/** Files whose markup this scanner reads. */
|
|
174
|
-
export const MARKUP = /\.(?:html?|jsx|tsx|vue|svelte|astro|erb|php)$/i;
|
|
174
|
+
export const MARKUP = /\.(?:html?|jsx|tsx|vue|svelte|astro|erb|php|twig|jinja2?|j2|cshtml|razor|hbs|handlebars|mustache|ejs|liquid|njk|blade\.php)$/i;
|
|
@@ -38,10 +38,35 @@ const CHECKS = [
|
|
|
38
38
|
// Go.
|
|
39
39
|
/\bt\s*\.\s*(Error|Errorf|Fatal|Fatalf|Fail|FailNow)\b/,
|
|
40
40
|
/\b(require|assert)\s*\.\s*[A-Z]\w+\s*\(/,
|
|
41
|
-
// Ruby
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
// Ruby. The snake_case family is the whole of minitest and most of Rails's own suite, and
|
|
42
|
+
// `assert_equal "x", y` has no parentheses - so the generic `assert` patterns above, which all
|
|
43
|
+
// want a `.` or a `(`, matched none of it. Without this, 121 of the 154 real tests in Rails's
|
|
44
|
+
// activesupport string_ext_test.rb were reported as comparing no value. That is the Phase 8
|
|
45
|
+
// lesson for the fourth time: a missing entry here does not produce a smaller answer, it
|
|
46
|
+
// produces a confident wrong one.
|
|
47
|
+
/\b(assert|refute)_\w+/,
|
|
48
|
+
/\bassert\s+[^=\n]/,
|
|
49
|
+
/\bexpect\s*\(.*\)\s*\.\s*(to|not_to|to_not)\b/,
|
|
50
|
+
// RSpec's block form, which is how every raise_error, change and output matcher is written:
|
|
51
|
+
// `expect { thing }.to raise_error(...)`. Missing it reported rspec-core's own specs - the
|
|
52
|
+
// reference implementation - as containing tests that cannot fail.
|
|
53
|
+
/\bexpect\s*\{[\s\S]*?\}\s*\.\s*(to|not_to|to_not)\b/,
|
|
54
|
+
/\bis_expected\s*\.\s*to\b/,
|
|
55
|
+
/\.\s*(must_equal|must_be|wont_equal|wont_be)\b/,
|
|
56
|
+
// PHP: PHPUnit on the instance or statically, and Pest's expectation chain.
|
|
57
|
+
//
|
|
58
|
+
// No `\b` before the `$`. A word boundary needs a word character on one side, `$` is not one,
|
|
59
|
+
// and the character before it in real code is a space - so `/\b\$this/` matched nothing, ever.
|
|
60
|
+
// This pattern was on the list from the start and had never once fired: 88 of the 128 real tests
|
|
61
|
+
// in laravel/framework's SupportStrTest came back as "Tewip would not judge" because the
|
|
62
|
+
// `assertSame(...)` in them was being read as an unknown helper that might assert.
|
|
63
|
+
/\$this\s*->\s*assert/,
|
|
64
|
+
/\b(?:self|static|Assert)\s*::\s*assert/,
|
|
65
|
+
/\bexpect\s*\(.*\)\s*->\s*\w+/,
|
|
66
|
+
// Rust. A #[should_panic] test asserts by panicking, so it can fail without an assert in it.
|
|
44
67
|
/\bassert(_eq|_ne|_matches)?!\s*\(/,
|
|
68
|
+
/\bdebug_assert(_eq|_ne)?!\s*\(/,
|
|
69
|
+
/#\s*\[\s*should_panic/,
|
|
45
70
|
// Robot Framework and other keyword styles: a line that reads as a check.
|
|
46
71
|
/^\s*Should\s+(Be|Not|Contain|Match)\b/im,
|
|
47
72
|
];
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
// Phase 22 and after: finding the tests inside a file, in each language's own shape.
|
|
2
|
+
//
|
|
3
|
+
// Split out of tests-in.ts, which passed this project's 300-line rule when Ruby, PHP and Rust were
|
|
4
|
+
// added. tests-in.ts keeps the questions every language shares - which language is this, does it
|
|
5
|
+
// look like tests at all, which parts of it are text rather than code - and this file holds the
|
|
6
|
+
// four readers: callbacks (JavaScript), indentation (Python), `end` (Ruby), and braces (the rest).
|
|
7
|
+
import { braced, isText, lineAt, textRanges } from './tests-in.js';
|
|
8
|
+
/**
|
|
9
|
+
* `it('...', async () => { ... })` and its many spellings, including `test.each`, `it.skip` and a
|
|
10
|
+
* plain `function ()` callback. The body is taken by matching braces from the callback's opening one.
|
|
11
|
+
*/
|
|
12
|
+
function jsTests(text) {
|
|
13
|
+
const found = [];
|
|
14
|
+
const ranges = textRanges(text);
|
|
15
|
+
const declaration = /\b(it|test|specify)\s*(\.\s*(skip|only|todo|failing|concurrent|each|for)\s*(\([^)]*\)|`[^`]*`)?\s*)*\(\s*(['"`])((?:\\.|(?!\5)[^\\])*)\5/g;
|
|
16
|
+
for (const hit of text.matchAll(declaration)) {
|
|
17
|
+
const at = hit.index ?? 0;
|
|
18
|
+
if (isText(ranges, at))
|
|
19
|
+
continue;
|
|
20
|
+
const open = text.indexOf('{', at + hit[0].length);
|
|
21
|
+
if (open === -1)
|
|
22
|
+
continue;
|
|
23
|
+
const body = braced(text, open);
|
|
24
|
+
if (body === null)
|
|
25
|
+
continue;
|
|
26
|
+
const signature = text.slice(at + hit[0].length, open);
|
|
27
|
+
found.push({
|
|
28
|
+
name: hit[6],
|
|
29
|
+
line: lineAt(text, at),
|
|
30
|
+
body,
|
|
31
|
+
skipped: /\.\s*(skip|todo|failing)\b/.test(hit[0]) || /\bxit\s*\(/.test(hit[0]),
|
|
32
|
+
callback: /\(\s*([A-Za-z_$][\w$]*)\s*\)\s*(=>)?\s*$/.exec(signature.trim())?.[1] ?? '',
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
// `xit(...)` and `xtest(...)`, which the pattern above deliberately does not match.
|
|
36
|
+
for (const hit of text.matchAll(/\bx(it|test)\s*\(\s*(['"`])((?:\\.|(?!\2)[^\\])*)\2/g)) {
|
|
37
|
+
const at = hit.index ?? 0;
|
|
38
|
+
if (isText(ranges, at))
|
|
39
|
+
continue;
|
|
40
|
+
const open = text.indexOf('{', at + hit[0].length);
|
|
41
|
+
const body = open === -1 ? null : braced(text, open);
|
|
42
|
+
found.push({ name: hit[3], line: lineAt(text, at), body: body ?? '', skipped: true, callback: '' });
|
|
43
|
+
}
|
|
44
|
+
return found.sort((a, b) => a.line - b.line);
|
|
45
|
+
}
|
|
46
|
+
/** Python has no braces: a test's body is everything indented further than its `def`. */
|
|
47
|
+
function pythonTests(text) {
|
|
48
|
+
const lines = text.split(String.fromCharCode(10));
|
|
49
|
+
const ranges = textRanges(text);
|
|
50
|
+
const offsets = [];
|
|
51
|
+
let seen = 0;
|
|
52
|
+
for (const line of lines) {
|
|
53
|
+
offsets.push(seen);
|
|
54
|
+
seen += line.length + 1;
|
|
55
|
+
}
|
|
56
|
+
const found = [];
|
|
57
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
58
|
+
const hit = /^(\s*)(?:async\s+)?def\s+(test_\w+|test)\s*\(/.exec(lines[i]);
|
|
59
|
+
if (!hit)
|
|
60
|
+
continue;
|
|
61
|
+
if (isText(ranges, offsets[i] + hit[1].length + 1))
|
|
62
|
+
continue;
|
|
63
|
+
const indent = hit[1].length;
|
|
64
|
+
// Nested in another def? Then it is a helper or, in Click's tests, a CLI command that happens to
|
|
65
|
+
// be named `test`. Only a module-level def, or a method of a test class, is a test.
|
|
66
|
+
if (nestedInFunction(lines, i, indent))
|
|
67
|
+
continue;
|
|
68
|
+
if (hit[2] === 'test' && indent > 0)
|
|
69
|
+
continue;
|
|
70
|
+
// A def whose parameters run over several lines ends at the line that closes them, and the body
|
|
71
|
+
// starts after that. Without this, the parameter list was read as the body and the real body was
|
|
72
|
+
// thrown away: psf/requests has several, and each came back as a test that compares no value.
|
|
73
|
+
let signatureEnds = i;
|
|
74
|
+
let open = 0;
|
|
75
|
+
for (let j = i; j < lines.length && j < i + 20; j += 1) {
|
|
76
|
+
open += (lines[j].match(/\(/g) ?? []).length - (lines[j].match(/\)/g) ?? []).length;
|
|
77
|
+
signatureEnds = j;
|
|
78
|
+
if (open <= 0)
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
const body = [];
|
|
82
|
+
for (let j = signatureEnds + 1; j < lines.length; j += 1) {
|
|
83
|
+
const line = lines[j];
|
|
84
|
+
if (line.trim() === '') {
|
|
85
|
+
body.push(line);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const width = line.length - line.trimStart().length;
|
|
89
|
+
if (width <= indent)
|
|
90
|
+
break;
|
|
91
|
+
body.push(line);
|
|
92
|
+
}
|
|
93
|
+
// Decorators sit above the def, so skipping is read upwards.
|
|
94
|
+
const above = lines.slice(Math.max(0, i - 4), i).join(String.fromCharCode(10));
|
|
95
|
+
// A conditional skip runs somewhere - `skipif` on Windows registry tests in psf/requests runs
|
|
96
|
+
// on Windows - so only an unconditional skip means the test never runs. `xfail` runs too.
|
|
97
|
+
const skipped = /@\s*(pytest\s*\.\s*mark\s*\.\s*skip|unittest\s*\.\s*skip)\b/.test(above) && !/skipif|skipUnless|xfail/.test(above);
|
|
98
|
+
found.push({ name: hit[2], line: i + 1, body: body.join(String.fromCharCode(10)), skipped, callback: '' });
|
|
99
|
+
}
|
|
100
|
+
return found;
|
|
101
|
+
}
|
|
102
|
+
/** Is the def on line `i` nested inside another def, rather than inside a class or at the top? */
|
|
103
|
+
function nestedInFunction(lines, i, indent) {
|
|
104
|
+
for (let j = i - 1; j >= 0; j -= 1) {
|
|
105
|
+
const line = lines[j];
|
|
106
|
+
if (line.trim() === '')
|
|
107
|
+
continue;
|
|
108
|
+
const width = line.length - line.trimStart().length;
|
|
109
|
+
if (width >= indent)
|
|
110
|
+
continue;
|
|
111
|
+
return /^\s*(async\s+)?def\s/.test(line);
|
|
112
|
+
}
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Ruby, in both of the two shapes that cover nearly all of it: minitest's `def test_name ... end`
|
|
117
|
+
* and RSpec's `it "name" do ... end`. Rails's own activesupport tests are the first; rspec-core's
|
|
118
|
+
* own specs are the second.
|
|
119
|
+
*
|
|
120
|
+
* The body runs to the `end` that lines up with the opener's indentation. Counting `do`, `def`,
|
|
121
|
+
* `if`, `case` and the rest against `end` would be the thorough way and is a poor trade: Ruby has
|
|
122
|
+
* one-line `if` modifiers, `end` inside strings, and heredocs, so a counter gets it wrong on real
|
|
123
|
+
* code in ways that are hard to see. Matching the indentation is right for conventionally
|
|
124
|
+
* formatted Ruby, which is nearly all Ruby, and when it cannot find the closing `end` the test is
|
|
125
|
+
* skipped rather than judged on a body that was guessed at.
|
|
126
|
+
*/
|
|
127
|
+
function rubyTests(text) {
|
|
128
|
+
/** Each accepted test with the line its `end` sits on, so a nested one can be recognised. */
|
|
129
|
+
const found = [];
|
|
130
|
+
const lines = text.split(String.fromCharCode(10));
|
|
131
|
+
const ranges = textRanges(text);
|
|
132
|
+
const offsets = [];
|
|
133
|
+
let seen = 0;
|
|
134
|
+
for (const line of lines) {
|
|
135
|
+
offsets.push(seen);
|
|
136
|
+
seen += line.length + 1;
|
|
137
|
+
}
|
|
138
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
139
|
+
// def test_foo minitest
|
|
140
|
+
// it "does a thing" do RSpec
|
|
141
|
+
// test "does a thing" do Rails and minitest's spec syntax
|
|
142
|
+
// specify { ... } is deliberately not here: a one-line specify has no body to read.
|
|
143
|
+
const hit = /^(\s*)(?:(?:x|f)?(it|test|scenario|example)\s*[("']|def\s+(test_\w+))/.exec(lines[i]);
|
|
144
|
+
if (!hit)
|
|
145
|
+
continue;
|
|
146
|
+
if (isText(ranges, offsets[i] + hit[1].length))
|
|
147
|
+
continue;
|
|
148
|
+
const indent = hit[1].length;
|
|
149
|
+
let close = -1;
|
|
150
|
+
for (let j = i + 1; j < lines.length; j += 1) {
|
|
151
|
+
const line = lines[j];
|
|
152
|
+
if (line.trim() === '')
|
|
153
|
+
continue;
|
|
154
|
+
if (/^\s*end\b/.test(line) && line.length - line.trimStart().length === indent) {
|
|
155
|
+
close = j;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (close === -1)
|
|
160
|
+
continue;
|
|
161
|
+
// An `it` inside another `it` is not a test. rspec-core's own specs define one as a fixture -
|
|
162
|
+
// `describe_successfully do it "..." do sequence << :example end end` - and the assertion that
|
|
163
|
+
// matters is in the enclosing test. Python's finder skips nested defs for the same reason.
|
|
164
|
+
if (found.some((earlier) => i > earlier.line - 1 && i < earlier.endsAt))
|
|
165
|
+
continue;
|
|
166
|
+
const body = lines.slice(i + 1, close).join(String.fromCharCode(10));
|
|
167
|
+
const named = hit[3] ?? /^\s*(?:x|f)?(?:it|test|scenario|example)\s*[("']\s*["']?([^"')]*)/.exec(lines[i])?.[1]?.trim();
|
|
168
|
+
found.push({
|
|
169
|
+
endsAt: close,
|
|
170
|
+
name: (named ?? '').trim() === '' ? 'unnamed' : named,
|
|
171
|
+
line: i + 1,
|
|
172
|
+
body,
|
|
173
|
+
// `xit` turns a test off. A bare `skip` does too - but only at the top level of the body.
|
|
174
|
+
// rspec-core's own specs pass a block containing `skip "because"` to a helper, as the
|
|
175
|
+
// fixture whose behaviour the test is checking, and the test around it asserts plenty.
|
|
176
|
+
// Reading that as "this test never runs" condemned the reference implementation.
|
|
177
|
+
skipped: /^\s*(?:x(?:it|specify|example)|skip\b)/.test(lines[i]) || skipsAtTopLevel(body),
|
|
178
|
+
callback: '',
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
return found;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* How each language says "this is a test".
|
|
185
|
+
*
|
|
186
|
+
* Rust and PHP join the braced finder rather than getting one of their own, because both mark a
|
|
187
|
+
* test the way the JVM languages do: an attribute or a naming convention, then a braced body. Only
|
|
188
|
+
* the marker differs.
|
|
189
|
+
*/
|
|
190
|
+
const DECLARES = {
|
|
191
|
+
// Rust marks a test with an attribute. `#[bench]` is left out for the same reason Go's Benchmark
|
|
192
|
+
// is: it measures rather than checks, and calling it assertion-free is true and useless.
|
|
193
|
+
rust: /#\s*\[\s*(?:test|tokio::test|async_std::test|rstest)\s*\]/g,
|
|
194
|
+
// PHPUnit accepts three markers and real suites use all of them: the testFoo naming convention
|
|
195
|
+
// (laravel/framework does this), the #[Test] attribute of PHPUnit 10, and the @test docblock.
|
|
196
|
+
php: /(?:#\s*\[\s*Test[^\]]*\]|\*\s*@test\b|(?:public|protected|private)?\s*function\s+test\w*\s*\()/g,
|
|
197
|
+
};
|
|
198
|
+
/** JUnit, Kotlin, Scala, Go, C#, PHP and Rust: an annotation or a naming convention, then a braced body. */
|
|
199
|
+
function bracedTests(language, text) {
|
|
200
|
+
const found = [];
|
|
201
|
+
const ranges = textRanges(text, language);
|
|
202
|
+
const declaration = DECLARES[language] ??
|
|
203
|
+
(language === 'go'
|
|
204
|
+
? // Only a Test function. A Benchmark asserts nothing by design, an Example is checked against
|
|
205
|
+
// its own Output comment, and a Fuzz target is not a test - all three were condemned as
|
|
206
|
+
// "checks nothing" in gorilla/mux before this narrowed.
|
|
207
|
+
/\bfunc\s+(Test)(\w+)\s*\(\s*\w+\s+\*testing\.[TB]\s*\)\s*\{/g
|
|
208
|
+
: /(@Test\b[^\n]*|\[\s*(?:Test|Fact|Theory)[^\]]*\]|@(?:ParameterizedTest|RepeatedTest)\b)/g);
|
|
209
|
+
for (const hit of text.matchAll(declaration)) {
|
|
210
|
+
const at = hit.index ?? 0;
|
|
211
|
+
if (isText(ranges, at))
|
|
212
|
+
continue;
|
|
213
|
+
// Where the body starts. Go's marker already includes the brace. Rust and PHP put a signature
|
|
214
|
+
// after the marker, and in PHP the marker may itself *be* the signature, so searching from the
|
|
215
|
+
// end of the match covers both.
|
|
216
|
+
const open = language === 'go'
|
|
217
|
+
? text.indexOf('{', at)
|
|
218
|
+
: language === 'rust' || language === 'php'
|
|
219
|
+
? text.indexOf('{', at + hit[0].length)
|
|
220
|
+
: text.indexOf('{', text.indexOf('(', at + hit[0].length));
|
|
221
|
+
if (open === -1)
|
|
222
|
+
continue;
|
|
223
|
+
const body = braced(text, open);
|
|
224
|
+
if (body === null)
|
|
225
|
+
continue;
|
|
226
|
+
const signature = text.slice(at, open);
|
|
227
|
+
// `void foo() throws IOException` still names foo. Without allowing the throws clause, every
|
|
228
|
+
// checked-exception test in gson was reported as 'unnamed'.
|
|
229
|
+
const name = language === 'go'
|
|
230
|
+
? hit[1] + hit[2]
|
|
231
|
+
: language === 'rust'
|
|
232
|
+
? // ripgrep generates whole families of tests from a macro, where the name in the source
|
|
233
|
+
// is literally `$name`. Reporting that is more honest than inventing one.
|
|
234
|
+
/\bfn\s+(\$?\w+)/.exec(text.slice(at, open))?.[1] ?? 'unnamed'
|
|
235
|
+
: language === 'php'
|
|
236
|
+
? /\bfunction\s+(\w+)/.exec(text.slice(at, open))?.[1] ?? 'unnamed'
|
|
237
|
+
: (/([\w]+)\s*\([^)]*\)\s*(throws\s+[\w.,\s]+)?$/.exec(signature.trim())?.[1] ?? 'unnamed');
|
|
238
|
+
const around = text.slice(Math.max(0, at - 200), open);
|
|
239
|
+
// An annotation disables a test outright. A `t.Skip()` may be conditional - cobra's
|
|
240
|
+
// TestDeadcodeElimination skips on Windows and runs everywhere else - so it counts only when it
|
|
241
|
+
// is unconditional, at the top level of the body.
|
|
242
|
+
const skipped = /@(Disabled|Ignore)\b|\[\s*Ignore\b/.test(around) || skipsUnconditionally(body);
|
|
243
|
+
found.push({ name, line: lineAt(text, at), body, skipped, callback: '' });
|
|
244
|
+
}
|
|
245
|
+
return found;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* A `t.Skip()` that runs whatever happens: at the top level of the body, not inside an `if`. Go
|
|
249
|
+
* tests commonly skip on one platform and run on the others, and calling those "never runs" is
|
|
250
|
+
* wrong.
|
|
251
|
+
*/
|
|
252
|
+
function skipsUnconditionally(body) {
|
|
253
|
+
let depth = 0;
|
|
254
|
+
for (const line of body.split(String.fromCharCode(10))) {
|
|
255
|
+
const text = line.trim();
|
|
256
|
+
if (depth === 0 && /^\w+\s*\.\s*Skip(Now)?\s*\(/.test(text))
|
|
257
|
+
return true;
|
|
258
|
+
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
259
|
+
}
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* A Ruby `skip` that turns this test off, rather than one sitting inside a block the test is
|
|
264
|
+
* handing to something else. Only a call at the body's own indentation counts: anything deeper is
|
|
265
|
+
* inside a block, and in rspec-core's specs that block is the fixture under test.
|
|
266
|
+
*/
|
|
267
|
+
function skipsAtTopLevel(body) {
|
|
268
|
+
const lines = body.split(String.fromCharCode(10)).filter((l) => l.trim() !== '');
|
|
269
|
+
if (lines.length === 0)
|
|
270
|
+
return false;
|
|
271
|
+
const base = Math.min(...lines.map((l) => l.length - l.trimStart().length));
|
|
272
|
+
return lines.some((l) => l.length - l.trimStart().length === base && /^\s*(?:skip|pending)\b/.test(l));
|
|
273
|
+
}
|
|
274
|
+
export { jsTests, pythonTests, rubyTests, bracedTests };
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
// Finding the individual tests in a file, and their bodies, without a parser per language.
|
|
2
|
+
//
|
|
3
|
+
// A parser for each of six languages is not the job, and would be the whole job. What is needed is
|
|
4
|
+
// narrower: where each test starts, what text belongs to it, and whether it was marked as skipped.
|
|
5
|
+
// Everything after this reads that text. Where the shape is not recognised, the file is reported as
|
|
6
|
+
// unread rather than judged - a file Tewip cannot see into is not a file full of worthless tests.
|
|
7
|
+
import { bracedTests, jsTests, pythonTests, rubyTests } from './find-tests.js';
|
|
1
8
|
/** Which language this file is, by extension. Unknown extensions are not guessed at. */
|
|
2
9
|
export function languageOf(file) {
|
|
3
10
|
const lower = file.toLowerCase();
|
|
@@ -11,12 +18,27 @@ export function languageOf(file) {
|
|
|
11
18
|
return 'go';
|
|
12
19
|
if (/\.(cs|fs)$/.test(lower))
|
|
13
20
|
return 'csharp';
|
|
21
|
+
// Phase 36. The assertion vocabulary already knew how these three check things - `expect(x).to`,
|
|
22
|
+
// `$this->assert`, `assert_eq!` were all on the list - so the only reason a Ruby, PHP or Rust
|
|
23
|
+
// file was never judged is that it never got this far. Triage has claimed all three for months.
|
|
24
|
+
if (lower.endsWith('.rb'))
|
|
25
|
+
return 'ruby';
|
|
26
|
+
if (lower.endsWith('.php'))
|
|
27
|
+
return 'php';
|
|
28
|
+
if (lower.endsWith('.rs'))
|
|
29
|
+
return 'rust';
|
|
14
30
|
return null;
|
|
15
31
|
}
|
|
16
32
|
/** Does this file look like tests at all? Vouching for a source file is a category error. */
|
|
17
33
|
export function looksLikeTests(file, text) {
|
|
18
34
|
const named = /(^|[\\/])(tests?|spec|specs|__tests__|e2e|it|integration)[\\/]|[._-](test|tests|spec|specs|it)\.[a-z]+$|(^|[\\/])test_[^\\/]+\.py$|_test\.(go|py|rb)$|Tests?\.(java|kt|cs|scala)$/i.test(file);
|
|
19
|
-
const declares = /\b(it|test|describe|Scenario)\s*\(|^\s*def\s+test_|@Test\b|\[(Test|Fact|Theory)\]|\bfunc\s+Test[A-Z]/m.test(text)
|
|
35
|
+
const declares = /\b(it|test|describe|Scenario)\s*\(|^\s*def\s+test_|@Test\b|\[(Test|Fact|Theory)\]|\bfunc\s+Test[A-Z]/m.test(text) ||
|
|
36
|
+
// Ruby, PHP and Rust, added in Phase 36. PHPUnit names a test `testSomething`, so the `test\s*\(`
|
|
37
|
+
// above never matched it, and Rust marks one with an attribute no pattern here knew. Both came
|
|
38
|
+
// back as "0 test(s) read in 0 file(s)", which reads like a clean answer and is not one.
|
|
39
|
+
/#\s*\[\s*(?:test|tokio::test|rstest)\s*\]/.test(text) ||
|
|
40
|
+
/\bfunction\s+test\w*\s*\(|#\s*\[\s*Test\b|\*\s*@test\b/.test(text) ||
|
|
41
|
+
/^\s*(?:RSpec\.)?(?:describe|context)\s+|^\s*(?:x|f)?it\s+["'].*\bdo\b/m.test(text);
|
|
20
42
|
return named || declares;
|
|
21
43
|
}
|
|
22
44
|
/**
|
|
@@ -26,7 +48,9 @@ export function looksLikeTests(file, text) {
|
|
|
26
48
|
* a checker ends up telling somebody that a string in their fixture checks nothing. Found by
|
|
27
49
|
* pointing this at Tewip's own test suite, where it produced 27 confident false accusations.
|
|
28
50
|
*/
|
|
29
|
-
|
|
51
|
+
/** A Rust char literal: `'a'`, `'\n'`, `'\''`. Anything else beginning with a quote is a lifetime. */
|
|
52
|
+
const RUST_CHAR = new RegExp(`^'(?:${String.raw `\\`}.|[^'${String.raw `\\`}])'`);
|
|
53
|
+
export function textRanges(text, language) {
|
|
30
54
|
const ranges = [];
|
|
31
55
|
const TRIPLES = ['"'.repeat(3), "'".repeat(3)];
|
|
32
56
|
for (let i = 0; i < text.length; i += 1) {
|
|
@@ -47,6 +71,14 @@ export function textRanges(text) {
|
|
|
47
71
|
ranges.push([i, end]);
|
|
48
72
|
i = end;
|
|
49
73
|
}
|
|
74
|
+
else if (ch === "'" && language === 'rust' && !RUST_CHAR.test(text.slice(i, i + 4))) {
|
|
75
|
+
// A lone quote in Rust is a lifetime - `&'a str`, `Cow<'_, str>` - and not the start of
|
|
76
|
+
// anything. Treating it as a quote made the detector swallow the rest of the file, so all
|
|
77
|
+
// six of ripgrep's #[test] attributes were discarded as being inside a string and the file
|
|
78
|
+
// was reported as having no tests at all. A char literal is still text, so it is matched
|
|
79
|
+
// first: a char closes within three characters and a lifetime never closes.
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
50
82
|
else if (ch === '"' || ch === "'" || ch === '`') {
|
|
51
83
|
// A Python docstring runs to the next triple quote, not to the next single one.
|
|
52
84
|
const triple = TRIPLES.find((t) => text.startsWith(t, i));
|
|
@@ -58,171 +90,18 @@ export function textRanges(text) {
|
|
|
58
90
|
return ranges;
|
|
59
91
|
}
|
|
60
92
|
/** Is this offset inside text rather than code? */
|
|
61
|
-
const isText = (ranges, at) => ranges.some(([from, to]) => at > from && at < to);
|
|
93
|
+
export const isText = (ranges, at) => ranges.some(([from, to]) => at > from && at < to);
|
|
62
94
|
export function testsIn(language, text) {
|
|
63
95
|
if (language === 'python')
|
|
64
96
|
return pythonTests(text);
|
|
65
97
|
if (language === 'js')
|
|
66
98
|
return jsTests(text);
|
|
99
|
+
if (language === 'ruby')
|
|
100
|
+
return rubyTests(text);
|
|
67
101
|
return bracedTests(language, text);
|
|
68
102
|
}
|
|
69
103
|
/** The line number of an offset, 1-based. */
|
|
70
|
-
const lineAt = (text, index) => text.slice(0, index).split(String.fromCharCode(10)).length;
|
|
71
|
-
/**
|
|
72
|
-
* `it('...', async () => { ... })` and its many spellings, including `test.each`, `it.skip` and a
|
|
73
|
-
* plain `function ()` callback. The body is taken by matching braces from the callback's opening one.
|
|
74
|
-
*/
|
|
75
|
-
function jsTests(text) {
|
|
76
|
-
const found = [];
|
|
77
|
-
const ranges = textRanges(text);
|
|
78
|
-
const declaration = /\b(it|test|specify)\s*(\.\s*(skip|only|todo|failing|concurrent|each|for)\s*(\([^)]*\)|`[^`]*`)?\s*)*\(\s*(['"`])((?:\\.|(?!\5)[^\\])*)\5/g;
|
|
79
|
-
for (const hit of text.matchAll(declaration)) {
|
|
80
|
-
const at = hit.index ?? 0;
|
|
81
|
-
if (isText(ranges, at))
|
|
82
|
-
continue;
|
|
83
|
-
const open = text.indexOf('{', at + hit[0].length);
|
|
84
|
-
if (open === -1)
|
|
85
|
-
continue;
|
|
86
|
-
const body = braced(text, open);
|
|
87
|
-
if (body === null)
|
|
88
|
-
continue;
|
|
89
|
-
const signature = text.slice(at + hit[0].length, open);
|
|
90
|
-
found.push({
|
|
91
|
-
name: hit[6],
|
|
92
|
-
line: lineAt(text, at),
|
|
93
|
-
body,
|
|
94
|
-
skipped: /\.\s*(skip|todo|failing)\b/.test(hit[0]) || /\bxit\s*\(/.test(hit[0]),
|
|
95
|
-
callback: /\(\s*([A-Za-z_$][\w$]*)\s*\)\s*(=>)?\s*$/.exec(signature.trim())?.[1] ?? '',
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
// `xit(...)` and `xtest(...)`, which the pattern above deliberately does not match.
|
|
99
|
-
for (const hit of text.matchAll(/\bx(it|test)\s*\(\s*(['"`])((?:\\.|(?!\2)[^\\])*)\2/g)) {
|
|
100
|
-
const at = hit.index ?? 0;
|
|
101
|
-
if (isText(ranges, at))
|
|
102
|
-
continue;
|
|
103
|
-
const open = text.indexOf('{', at + hit[0].length);
|
|
104
|
-
const body = open === -1 ? null : braced(text, open);
|
|
105
|
-
found.push({ name: hit[3], line: lineAt(text, at), body: body ?? '', skipped: true, callback: '' });
|
|
106
|
-
}
|
|
107
|
-
return found.sort((a, b) => a.line - b.line);
|
|
108
|
-
}
|
|
109
|
-
/** Python has no braces: a test's body is everything indented further than its `def`. */
|
|
110
|
-
function pythonTests(text) {
|
|
111
|
-
const lines = text.split(String.fromCharCode(10));
|
|
112
|
-
const ranges = textRanges(text);
|
|
113
|
-
const offsets = [];
|
|
114
|
-
let seen = 0;
|
|
115
|
-
for (const line of lines) {
|
|
116
|
-
offsets.push(seen);
|
|
117
|
-
seen += line.length + 1;
|
|
118
|
-
}
|
|
119
|
-
const found = [];
|
|
120
|
-
for (let i = 0; i < lines.length; i += 1) {
|
|
121
|
-
const hit = /^(\s*)(?:async\s+)?def\s+(test_\w+|test)\s*\(/.exec(lines[i]);
|
|
122
|
-
if (!hit)
|
|
123
|
-
continue;
|
|
124
|
-
if (isText(ranges, offsets[i] + hit[1].length + 1))
|
|
125
|
-
continue;
|
|
126
|
-
const indent = hit[1].length;
|
|
127
|
-
// Nested in another def? Then it is a helper or, in Click's tests, a CLI command that happens to
|
|
128
|
-
// be named `test`. Only a module-level def, or a method of a test class, is a test.
|
|
129
|
-
if (nestedInFunction(lines, i, indent))
|
|
130
|
-
continue;
|
|
131
|
-
if (hit[2] === 'test' && indent > 0)
|
|
132
|
-
continue;
|
|
133
|
-
// A def whose parameters run over several lines ends at the line that closes them, and the body
|
|
134
|
-
// starts after that. Without this, the parameter list was read as the body and the real body was
|
|
135
|
-
// thrown away: psf/requests has several, and each came back as a test that compares no value.
|
|
136
|
-
let signatureEnds = i;
|
|
137
|
-
let open = 0;
|
|
138
|
-
for (let j = i; j < lines.length && j < i + 20; j += 1) {
|
|
139
|
-
open += (lines[j].match(/\(/g) ?? []).length - (lines[j].match(/\)/g) ?? []).length;
|
|
140
|
-
signatureEnds = j;
|
|
141
|
-
if (open <= 0)
|
|
142
|
-
break;
|
|
143
|
-
}
|
|
144
|
-
const body = [];
|
|
145
|
-
for (let j = signatureEnds + 1; j < lines.length; j += 1) {
|
|
146
|
-
const line = lines[j];
|
|
147
|
-
if (line.trim() === '') {
|
|
148
|
-
body.push(line);
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
const width = line.length - line.trimStart().length;
|
|
152
|
-
if (width <= indent)
|
|
153
|
-
break;
|
|
154
|
-
body.push(line);
|
|
155
|
-
}
|
|
156
|
-
// Decorators sit above the def, so skipping is read upwards.
|
|
157
|
-
const above = lines.slice(Math.max(0, i - 4), i).join(String.fromCharCode(10));
|
|
158
|
-
// A conditional skip runs somewhere - `skipif` on Windows registry tests in psf/requests runs
|
|
159
|
-
// on Windows - so only an unconditional skip means the test never runs. `xfail` runs too.
|
|
160
|
-
const skipped = /@\s*(pytest\s*\.\s*mark\s*\.\s*skip|unittest\s*\.\s*skip)\b/.test(above) && !/skipif|skipUnless|xfail/.test(above);
|
|
161
|
-
found.push({ name: hit[2], line: i + 1, body: body.join(String.fromCharCode(10)), skipped, callback: '' });
|
|
162
|
-
}
|
|
163
|
-
return found;
|
|
164
|
-
}
|
|
165
|
-
/** Is the def on line `i` nested inside another def, rather than inside a class or at the top? */
|
|
166
|
-
function nestedInFunction(lines, i, indent) {
|
|
167
|
-
for (let j = i - 1; j >= 0; j -= 1) {
|
|
168
|
-
const line = lines[j];
|
|
169
|
-
if (line.trim() === '')
|
|
170
|
-
continue;
|
|
171
|
-
const width = line.length - line.trimStart().length;
|
|
172
|
-
if (width >= indent)
|
|
173
|
-
continue;
|
|
174
|
-
return /^\s*(async\s+)?def\s/.test(line);
|
|
175
|
-
}
|
|
176
|
-
return false;
|
|
177
|
-
}
|
|
178
|
-
/** JUnit, Kotlin, Scala, Go and C#: an annotation or a naming convention, then a braced body. */
|
|
179
|
-
function bracedTests(language, text) {
|
|
180
|
-
const found = [];
|
|
181
|
-
const ranges = textRanges(text);
|
|
182
|
-
const declaration = language === 'go'
|
|
183
|
-
? // Only a Test function. A Benchmark asserts nothing by design, an Example is checked against
|
|
184
|
-
// its own Output comment, and a Fuzz target is not a test - all three were condemned as
|
|
185
|
-
// "checks nothing" in gorilla/mux before this narrowed.
|
|
186
|
-
/\bfunc\s+(Test)(\w+)\s*\(\s*\w+\s+\*testing\.[TB]\s*\)\s*\{/g
|
|
187
|
-
: /(@Test\b[^\n]*|\[\s*(?:Test|Fact|Theory)[^\]]*\]|@(?:ParameterizedTest|RepeatedTest)\b)/g;
|
|
188
|
-
for (const hit of text.matchAll(declaration)) {
|
|
189
|
-
const at = hit.index ?? 0;
|
|
190
|
-
if (isText(ranges, at))
|
|
191
|
-
continue;
|
|
192
|
-
const open = language === 'go' ? text.indexOf('{', at) : text.indexOf('{', text.indexOf('(', at + hit[0].length));
|
|
193
|
-
if (open === -1)
|
|
194
|
-
continue;
|
|
195
|
-
const body = braced(text, open);
|
|
196
|
-
if (body === null)
|
|
197
|
-
continue;
|
|
198
|
-
const signature = text.slice(at, open);
|
|
199
|
-
// `void foo() throws IOException` still names foo. Without allowing the throws clause, every
|
|
200
|
-
// checked-exception test in gson was reported as 'unnamed'.
|
|
201
|
-
const name = language === 'go' ? hit[1] + hit[2] : /([\w]+)\s*\([^)]*\)\s*(throws\s+[\w.,\s]+)?$/.exec(signature.trim())?.[1] ?? 'unnamed';
|
|
202
|
-
const around = text.slice(Math.max(0, at - 200), open);
|
|
203
|
-
// An annotation disables a test outright. A `t.Skip()` may be conditional - cobra's
|
|
204
|
-
// TestDeadcodeElimination skips on Windows and runs everywhere else - so it counts only when it
|
|
205
|
-
// is unconditional, at the top level of the body.
|
|
206
|
-
const skipped = /@(Disabled|Ignore)\b|\[\s*Ignore\b/.test(around) || skipsUnconditionally(body);
|
|
207
|
-
found.push({ name, line: lineAt(text, at), body, skipped, callback: '' });
|
|
208
|
-
}
|
|
209
|
-
return found;
|
|
210
|
-
}
|
|
211
|
-
/**
|
|
212
|
-
* A `t.Skip()` that runs whatever happens: at the top level of the body, not inside an `if`. Go
|
|
213
|
-
* tests commonly skip on one platform and run on the others, and calling those "never runs" is
|
|
214
|
-
* wrong.
|
|
215
|
-
*/
|
|
216
|
-
function skipsUnconditionally(body) {
|
|
217
|
-
let depth = 0;
|
|
218
|
-
for (const line of body.split(String.fromCharCode(10))) {
|
|
219
|
-
const text = line.trim();
|
|
220
|
-
if (depth === 0 && /^\w+\s*\.\s*Skip(Now)?\s*\(/.test(text))
|
|
221
|
-
return true;
|
|
222
|
-
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
223
|
-
}
|
|
224
|
-
return false;
|
|
225
|
-
}
|
|
104
|
+
export const lineAt = (text, index) => text.slice(0, index).split(String.fromCharCode(10)).length;
|
|
226
105
|
/**
|
|
227
106
|
* The text between the brace at `open` and the one that closes it, skipping over strings, template
|
|
228
107
|
* literals, comments and regular expressions - all of which can hold a brace that closes nothing.
|
|
@@ -34,7 +34,10 @@ export function testFiles(root, paths) {
|
|
|
34
34
|
}
|
|
35
35
|
if (stat.isDirectory())
|
|
36
36
|
walk(full, depth + 1);
|
|
37
|
-
|
|
37
|
+
// rb, php and rs added in Phase 36. They were missing here as well as in languageOf, so even
|
|
38
|
+
// a file named outright came back "0 test(s) read in 0 file(s)" - which reads like a clean
|
|
39
|
+
// answer and is not one.
|
|
40
|
+
else if (/\.(js|jsx|mjs|cjs|ts|tsx|mts|cts|py|java|kt|kts|scala|groovy|go|cs|fs|rb|php|rs)$/i.test(entry))
|
|
38
41
|
found.push(full);
|
|
39
42
|
}
|
|
40
43
|
};
|
|
@@ -95,6 +95,17 @@ function judge(file, test, helpers) {
|
|
|
95
95
|
const unknown = called.find((name) => /^(check|verify|ensure|validate|assert|expect|should|confirm)/i.test(name));
|
|
96
96
|
if (unknown)
|
|
97
97
|
return { ...at, verdict: 'cannot-tell', evidence: `${unknown}(…)`, why: `${unknown}() may assert, and it is not in this file, so Tewip will not call this either way` };
|
|
98
|
+
// A body whose whole content is one call to something defined in another file. The check is very
|
|
99
|
+
// likely inside it, and Tewip cannot follow it there, so "it compares no value" would be a
|
|
100
|
+
// confident wrong answer. Rails's own `test_constantize` is exactly this: one line calling
|
|
101
|
+
// `run_constantize_tests_on`, which lives in a shared helper and asserts plenty.
|
|
102
|
+
const statements = body
|
|
103
|
+
.split(String.fromCharCode(10))
|
|
104
|
+
.map((l) => l.trim())
|
|
105
|
+
.filter((l) => l !== '' && !l.startsWith('#') && !l.startsWith('//'));
|
|
106
|
+
if (statements.length === 1 && /^[A-Za-z_][\w.:]*\s*[({]/.test(statements[0])) {
|
|
107
|
+
return { ...at, verdict: 'cannot-tell', evidence: statements[0].slice(0, 80), why: 'its whole body is one call to something defined elsewhere, which may assert; Tewip will not call this either way' };
|
|
108
|
+
}
|
|
98
109
|
if (unfinished.length > 0)
|
|
99
110
|
return { ...at, verdict: 'unfinished-expectation', evidence: unfinished[0], why: MEANS['unfinished-expectation'] };
|
|
100
111
|
if (tautology.length > 0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tewip-ai",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.44.0",
|
|
4
4
|
"mcpName": "io.github.hmamut39/tewip",
|
|
5
5
|
"description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
|
|
6
6
|
"keywords": [
|