tewip-ai 0.53.0 → 0.55.0

package/README.md

@@ -1067,9 +1067,18 @@ that never run, and tests that compare no value at all. No model, no cost.
 It is also an MCP tool, so a coding agent can ask about the tests it just wrote before claiming a
 change is covered.
 
-**Measured on 5,448 tests in 8 public projects across 5 languages** (express, axios, chalk, requests,
-click, gorilla/mux, cobra, gson): 19 tests were called unable to fail, and **all 19 were correct** when
-read against the files - one unconditional skip in psf/requests, eighteen `@Ignore` methods in gson.
+**Measured on 41,523 tests in six public projects** - Keycloak, OpenMRS, bitwarden/server, Saleor,
+GOV.UK Frontend and Firefly III - in Java, C#, Python, PHP and TypeScript. Every accusation was read
+against its source by hand, and the counts are kept in [`corpus/expected.json`](corpus/expected.json)
+against a pinned commit, re-derived by `npm run corpus` on every release.
+
+**It has been badly wrong once, and that is worth saying plainly.** An earlier version of this
+paragraph reported 19 of 19 correct on 5,448 tests. Then Tewip was pointed at Keycloak and **called
+166 correct tests worthless** - every one of them the standard Java way of writing "I expect this to
+throw", which `Assert.fail()` makes fail properly because `AssertionError` is not an `Exception`.
+The rule had never read the caught type. 918 unit tests passed before that and after it, which is
+why the corpus exists: the thing that matters is what Tewip says about other people's code, and
+nothing was measuring it.
 
 Two things it refuses to do, which is why the list is worth reading:
 

package/dist/engine/access/checks.js

@@ -6,9 +6,9 @@
 // div that only answers to a mouse. The ones that need a rendered page (contrast, focus order,
 // whether alt text is any good) are not attempted here and are listed in NOT_CHECKED instead.
 import { inside, labelledBy } from './markup.js';
-import { ARIA, INTERACTIVE, UNLABELLED_TYPES, at, clickHandler, decorative, hiddenFromEveryone, keyHandler, show, tabIndexOf } from './element.js';
+import { ARIA, INTERACTIVE, UNLABELLED_TYPES, at, clickHandler, decorative, hiddenFromEveryone, keyHandler, notRendered, show, tabIndexOf } from './element.js';
 /** Checks that look at one element at a time. */
-function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent = new Set()) {
+function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent = new Set(), boundLabels = new Set()) {
     const out = [];
     const place = at(t, file);
     const name = labelledBy(t);
@@ -44,7 +44,8 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
     // and `<Select onOpenChange={...}>` was reported as an unnamed combo box; what that component
     // renders, and whether it labels itself, is not in this file and cannot be read from it.
     if (nameRule && !t.component && !hiddenFromEveryone(t) && (t.name !== 'input' || !UNLABELLED_TYPES.has(t.attrs.type ?? 'text'))) {
-        const linked = typeof t.attrs.id === 'string' && labelFor.has(t.attrs.id);
+        const bound = ['asp-for', 'formControlName', 'v-model', 'data-valmsg-for'].some((key) => typeof t.attrs[key] === 'string' && boundLabels.has(`${key}=${t.attrs[key]}`));
+        const linked = bound || (typeof t.attrs.id === 'string' && labelFor.has(t.attrs.id));
         if (!name.named && !linked && openLabels === 0) {
             if (name.unknown || t.dynamic.has('id') || t.dynamic.has('style') || (typeof t.attrs.id === 'string' && namedByComponent.has(t.attrs.id))) {
                 out.push({ rule: nameRule, confidence: 'ask-a-person', ...place,
@@ -73,7 +74,11 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
             why: `tabindex="${tabindex}" takes this element out of the document order and puts it before everything that has none`,
             next: 'use tabindex="0" and let the source order decide, or move the element in the markup' });
     }
-    if (t.attrs['aria-hidden'] === 'true' && (INTERACTIVE.has(t.name) || (tabindex !== null && tabindex >= 0))) {
+    // `hidden` takes the element out of the tab order as well as out of the page, so aria-hidden on it
+    // hides nothing that was reachable. GOV.UK's service-navigation toggle is written
+    // `<button ... hidden aria-hidden="true">` and uncovered by JavaScript when the menu is enabled -
+    // progressive enhancement done properly, reported as a barrier.
+    if (t.attrs['aria-hidden'] === 'true' && !notRendered(t) && (INTERACTIVE.has(t.name) || (tabindex !== null && tabindex >= 0))) {
         out.push({ rule: 'aria-hidden-focusable', confidence: 'barrier', ...place,
             why: 'hidden from screen readers but still reachable by the tab key',
             next: 'also remove it from the tab order with tabindex="-1", or stop hiding it' });
@@ -121,6 +126,22 @@ function perElement(t, file, labelFor, openLabels, ancestorKey, namedByComponent
 export function checkFile(source, all, file, isDocument = false) {
     const out = [];
     const labelFor = new Set(all.filter((t) => t.name === 'label' && t.attrs.for).map((t) => t.attrs.for));
+    // A framework can bind a label to a field without either of them spelling `for` or `id`. ASP.NET
+    // Core Tag Helpers do it with the model property: `asp-for="Name"` on the <label> renders
+    // `for="Name"`, and the same on the <input> renders `id="Name"`. bitwarden/server's admin views
+    // are written this way throughout, and 111 correctly labelled fields were called unlabelled.
+    // The same shape as PatternFly's FormGroup in Phase 43: the label is real, it is just not here.
+    const BINDS = ['asp-for', 'formControlName', 'v-model', 'data-valmsg-for'];
+    const boundLabels = new Set();
+    for (const t of all) {
+        if (t.name !== 'label')
+            continue;
+        for (const key of BINDS) {
+            const value = t.attrs[key];
+            if (typeof value === 'string' && value !== '')
+                boundLabels.add(`${key}=${value}`);
+        }
+    }
     // Ids that a *component* names in one of its props. A design system labels a field by being told
     // which field it is for - PatternFly's `<FormGroup label={...} fieldId="x">` renders `<label for="x">`
     // around its children - and that label exists only in the component's own source, not here.
@@ -153,7 +174,7 @@ export function checkFile(source, all, file, isDocument = false) {
                 open.length = from;
             continue;
         }
-        out.push(...perElement(t, file, labelFor, Math.max(openLabels - (t.name === 'label' ? 1 : 0), 0), open.some((o) => o.keys), namedByComponent));
+        out.push(...perElement(t, file, labelFor, Math.max(openLabels - (t.name === 'label' ? 1 : 0), 0), open.some((o) => o.keys), namedByComponent, boundLabels));
         if (!t.selfClosing)
             open.push({ name: t.name, keys: keyHandler(t) });
         const id = t.attrs.id;

package/dist/engine/access/element.js

@@ -58,9 +58,22 @@ export const UNLABELLED_TYPES = new Set(['hidden', 'submit', 'reset', 'button',
  * When the style is a JSX expression its text is not kept, so whether it hides the element cannot be
  * read. That is not a reason to call it a certain barrier - it is a reason to say so.
  */
-export function hiddenFromEveryone(t) {
+/**
+ * Not rendered at all, and therefore not in the tab order either.
+ *
+ * Separate from `hiddenFromEveryone` because `aria-hidden` hides an element from a screen reader
+ * while leaving it perfectly focusable - that gap is a real barrier and has its own rule. The
+ * `hidden` attribute and `display:none` close the gap: nothing can reach the element by any route.
+ */
+export function notRendered(t) {
     if ('hidden' in t.attrs && t.attrs.hidden !== 'false')
         return true;
+    const style = typeof t.attrs.style === 'string' ? t.attrs.style : '';
+    return /display\s*:\s*none|visibility\s*:\s*hidden/i.test(style);
+}
+export function hiddenFromEveryone(t) {
+    if (notRendered(t))
+        return true;
     if (t.attrs['aria-hidden'] === 'true')
         return true;
     const style = typeof t.attrs.style === 'string' ? t.attrs.style : '';

package/dist/engine/access/markup.js

@@ -31,6 +31,27 @@ const TAG = /<(\/?)([A-Za-z][A-Za-z0-9._:-]*)(?=[\s/>])((?:[^>"'{]|"[^"]*"|'[^']
 const ATTRIBUTE = /([@:#\[A-Za-z_][-A-Za-z0-9_:.@\]]*)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|\{((?:[^{}]|\{[^{}]*\})*)\}|([^\s"'<>`=]+)))?/g;
 /** `{...props}`, `{...rest}` - the thing that makes every other judgement about the element unsafe. */
 const SPREAD = /\{\s*\.\.\./;
+/**
+ * A template language's version of a spread: an expression standing where an attribute *name*
+ * would go, rather than inside a value.
+ *
+ * GOV.UK's own input component is written this way - every attribute, `id` included, comes out of
+ * one macro call:
+ *
+ *     <input
+ *       {{- govukAttributes({ class: classNames, id: id, name: params.name, ... }) }}
+ *
+ * so after the expressions are blanked the element reads as a bare `<input>` with nothing on it,
+ * and the scanner called the UK government's accessible-by-obligation text input a certain
+ * barrier. It is not: it is an element this file cannot answer for, which is a different verdict.
+ *
+ * Quoted values are removed first, because `id="{{ params.id }}"` is a dynamic attribute - which
+ * is already handled, and separately - rather than attributes arriving from somewhere else.
+ */
+function templateSpread(blob) {
+    const withoutValues = blob.replace(/"[^"]*"|'[^']*'/g, '""');
+    return /\{\{|\{%|<%|#{/.test(withoutValues);
+}
 /**
  * Comments, and the bodies of script and style, hold things shaped like tags that are not tags.
  *
@@ -73,7 +94,7 @@ export const JS_LIKE = /\.(?:[cm]?[jt]sx?|vue|svelte|astro)$/i;
 function readAttributes(blob) {
     const attrs = {};
     const dynamic = new Set();
-    const spread = SPREAD.test(blob);
+    const spread = SPREAD.test(blob) || templateSpread(blob);
     // A spread's contents would otherwise be read as attribute names: `{...props}` becomes `props`.
     const readable = blob.replace(/\{\s*\.\.\.(?:[^{}]|\{[^{}]*\})*\}/g, ' ');
     for (const m of readable.matchAll(ATTRIBUTE)) {

package/dist/engine/schema/schema-command.js

@@ -39,8 +39,32 @@ export function schemaFiles(cmd) {
  */
 function createdHere(text) {
     const names = new Set();
-    for (const m of text.matchAll(/\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?"?([\w$.]+)"?/gi)) {
-        names.add(m[1].toLowerCase().replace(/^.*\./, ''));
+    // The frameworks say "create this table" in their own words, and the suppression is worth nothing
+    // in a dialect it cannot read. Only SQL was read, so bitwarden/server's 456 EF Core migrations -
+    // which do exactly what Prisma does, a CreateTable followed by a CreateIndex for every relation -
+    // produced 647 "blocks writes" findings about tables that had not existed a statement earlier.
+    // EF Core puts the name on the line after the call, so these have to reach across a newline.
+    const FRAMEWORK_CREATES = [
+        /migrationBuilder\s*\.\s*CreateTable\s*\([\s\S]{0,80}?name\s*:\s*["']([\w$.]+)["']/gi,
+        /queryInterface\s*\.\s*createTable\s*\(\s*["']([\w$.]+)["']/gi,
+        /knex\s*\.\s*schema\s*\.\s*createTable\s*\(\s*["']([\w$.]+)["']/gi,
+        /Schema\s*::\s*create\s*\(\s*["']([\w$.]+)["']/gi,
+        /\bcreate_table\s+[:'"]([\w$.]+)/gi,
+        /\bop\s*\.\s*create_table\s*\(\s*["']([\w$.]+)["']/gi,
+        /<createTable[^>]*tableName\s*=\s*["']([\w$.]+)["']/gi,
+    ];
+    for (const pattern of FRAMEWORK_CREATES)
+        for (const m of text.matchAll(pattern))
+            names.add(m[1].toLowerCase().replace(/^.*\./, ''));
+    // Every dialect's way of quoting a name, because the suppression is useless in the one that does
+    // not match. This read PostgreSQL's "quoted" form and plain names, and not SQL Server's
+    // [dbo].[Grant] - which is standard T-SQL and what bitwarden/server is written in throughout. No
+    // table was ever recognised as freshly created there, so every index built on a table created two
+    // statements earlier was reported as blocking writes: 714 findings, nearly all of them free.
+    for (const m of text.matchAll(/\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w$.\"\[\]`]+)/gi)) {
+        // `[dbo].[Grant]` and `"public"."grant"` and `grant` all have to arrive here as the same word,
+        // because that is the form the findings report a table in.
+        names.add(m[1].replace(/[\"\[\]`]/g, '').toLowerCase().replace(/^.*\./, ''));
     }
     return names;
 }

package/dist/engine/verdict/verdict.js

@@ -25,11 +25,33 @@ export function decide(evidence) {
     if (evidence.changedSource.length === 0) {
         checks.push({ question: 'Is the changed code tested?', standing: 'not-applicable', answer: 'this change touches no source file', decidedBy: 'the diff' });
     }
+    else if (evidence.covering.length === 0 && evidence.changedTests.length > 0) {
+        // Nothing *imports* the changed file, and the same change ships tests. Coverage here is followed
+        // through imports, which is a fact rather than a name match - and which cannot see a test that
+        // reaches the code through a framework's entry point. A Saleor test posts a GraphQL query and
+        // asserts the error it gets back; the resolver chain in between is never imported by name.
+        //
+        // Found in Phase 46 by running this verdict over six real merged Saleor pull requests. Four came
+        // back **Not ready**, and all four shipped a test for exactly the change they were making. A gate
+        // that says "not ready" to a correct pull request is uninstalled within a week, and it is the
+        // same overclaim as a false accusation, pointed at the change instead of at a test.
+        //
+        // So: cannot say. Which is this project's whole point, and this check had no way to say it.
+        checks.push({
+            question: 'Is the changed code tested?',
+            standing: 'not-checked',
+            answer: `no test imports or reaches ${evidence.changedSource.slice(0, 3).join(', ')}, but this change also ` +
+                `edits ${evidence.changedTests.length} test file(s). Coverage is followed through imports, which cannot see a test ` +
+                'that reaches the code through a GraphQL resolver, an HTTP route or any other framework entry point - ' +
+                'so whether those tests cover this is a question the imports cannot answer',
+            decidedBy: 'imports followed through the source, which is a fact rather than a name match',
+        });
+    }
     else if (evidence.covering.length === 0) {
         checks.push({
             question: 'Is the changed code tested?',
             standing: 'not-met',
-            answer: `no test imports or reaches ${evidence.changedSource.slice(0, 3).join(', ')}, so nothing here could notice a mistake in it`,
+            answer: `no test imports or reaches ${evidence.changedSource.slice(0, 3).join(', ')}, and this change adds no test either, so nothing here could notice a mistake in it`,
             decidedBy: 'imports followed through the source, which is a fact rather than a name match',
         });
     }
@@ -50,7 +72,7 @@ export function decide(evidence) {
             question: 'Can the tests in this change fail?',
             standing: 'not-met',
             answer: `${evidence.cannotFail.length} of them cannot: ${evidence.cannotFail.slice(0, 2).map((t) => `${t.name} (${t.file}:${t.line})`).join(', ')}`,
-            decidedBy: 'tewip vouch, 19 of 19 correct on 5,448 tests in 8 public projects',
+            decidedBy: 'tewip vouch, measured on 41,523 real tests in the six public projects in corpus/expected.json',
         });
     }
     else {
@@ -120,7 +142,13 @@ export function decide(evidence) {
         headline: broken.length > 0
             ? `Not ready: ${broken[0].answer}`
             : verdict === 'cannot-say'
-                ? `Cannot say: ${essential.filter((c) => c.standing === 'not-checked').map((c) => c.question.toLowerCase().replace(/\?$/, '')).join(' and ')} could not be checked, so this is not an approval`
+                ? // Quoted, because the questions are questions: lowercasing them and joining with "and"
+                    // produced "is anything failing and is the changed code tested could not be checked",
+                    // which is the headline of the verdict this whole tool is named for.
+                    `Cannot say: ${essential.filter((c) => c.standing === 'not-checked').length === 1 ? 'one question that matters' : 'two questions that matter'} here went unanswered - ${essential
+                        .filter((c) => c.standing === 'not-checked')
+                        .map((c) => `"${c.question}"`)
+                        .join(' and ')} - so this is not an approval`
                 : `Ready as far as ${checks.filter((c) => c.standing === 'met').length} check(s) can show${unknown.length > 0 ? `, with ${unknown.length} not checked` : ''}`,
         checks,
         notChecked: [

package/dist/engine/vouch/find-tests.js

@@ -239,7 +239,14 @@ function bracedTests(language, text) {
         // An annotation disables a test outright. A `t.Skip()` may be conditional - cobra's
         // TestDeadcodeElimination skips on Windows and runs everywhere else - so it counts only when it
         // is unconditional, at the top level of the body.
-        const skipped = /@(Disabled|Ignore)\b|\[\s*Ignore\b/.test(around) || skipsUnconditionally(body);
+        // xUnit - the most used .NET framework by a distance - has no separate attribute for this. It
+        // skips by giving the test attribute a reason: `[Fact(Skip = "...")]`, `[Theory(Skip = ...)]`.
+        // Only `[Ignore]` was read, which is NUnit and MSTest, so bitwarden/server's **43 skipped tests
+        // were reported as 1**. A team told their suite is sound while 43 of its tests never run is
+        // worse off than a team told nothing at all.
+        const skipped = /@(Disabled|Ignore)\b|\[\s*Ignore\b/.test(around) ||
+            /\[\s*\w*(?:Fact|Theory)\s*\([^)]*\bSkip\s*=/.test(around) ||
+            skipsUnconditionally(body);
         found.push({ name, line: lineAt(text, at), body, skipped, callback: '' });
     }
     return found;
@@ -253,7 +260,12 @@ function skipsUnconditionally(body) {
     let depth = 0;
     for (const line of body.split(String.fromCharCode(10))) {
         const text = line.trim();
-        if (depth === 0 && /^\w+\s*\.\s*Skip(Now)?\s*\(/.test(text))
+        // `t.Skip(...)` in Go: a short receiver, and a whole statement with nothing chained onto it.
+        // `\w+\.Skip(` matched any identifier at all, so C#'s LINQ - `collections.Skip(1).ToList()`,
+        // which is how .NET pages a sequence - read as a test turning itself off. bitwarden/server has
+        // one, and it was the only never-runs Tewip reported there until xUnit's `[Fact(Skip = ...)]`
+        // was understood; the real number was 43 and the one it had was wrong.
+        if (depth === 0 && /^\w{1,3}\s*\.\s*Skip(f|Now)?\s*\([^)]*\)\s*;?\s*$/.test(text))
             return true;
         depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.53.0",
+  "version": "0.55.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [