tewip-ai 0.15.1 → 0.16.1

package/README.md

@@ -14,7 +14,7 @@ It works for every stack: Node.js (node:test, Jest, Vitest, Mocha), Python (pyte
 
 Formerly *triage-agent*: the repository moved to `hmamut39/tewip`, and GitHub redirects the old address. See [ROADMAP.md](ROADMAP.md) for the plan.
 
-**Status: phases 1 to 4 are built, Phase 5 (auto-fix) is in progress, and phases 6 and 7 (test plans and verified test writing) are built.** Those phases are collection, rule-based diagnosis, LLM diagnosis with a real-data eval, multi-role views, and planning and writing tests in the project's own framework. It runs live in GitHub CI and answers questions from AI assistants through MCP. On fresh real Gutenberg CI failures, with the pipeline frozen before labelling, accuracy is 93.4%; the caveats are in [eval/README.md](eval/README.md).
+**Status: phases 1 to 14 are built; the two auto-fix phases (5 and 8) are the ones still in progress.** Tewip collects failures from any framework, diagnoses them with rules first and a model only for what rules cannot settle, and turns that into work for each role: a CI comment and merge gate, test plans and verified tests, a flake dossier, a PR-risk review, security scanning, tickets in Jira, Rally, GitHub Issues, Azure Boards or Linear, checks generated from a Figma design, specifications written for people, and - opt in, and only when proved - fixes to test or application code. It runs live in GitHub CI and answers questions from AI assistants through MCP. On fresh real Gutenberg CI failures, with the pipeline frozen before labelling, accuracy is 93.4%; across six real datasets it ranges from 86.1% to 99.7%, and the caveats are in [eval/README.md](eval/README.md).
 
 ## Use it in your project
 

package/dist/engine/autofix/data-site.js

@@ -0,0 +1,58 @@
+/** The line is checking the value rather than creating anything with it. */
+const ASSERTING = /\bexpect\s*\(|\.\s*should\s*\(|\bassert(?:Equals?|That|ion)?\b|\btoBe\w*\s*\(|\bto(?:Have|Contain)\w*\s*\(/;
+/**
+ * The line is making or sending something: this is data, and making it unique is the fix.
+ * The object-property form matters most and was missing at first - real test data is written
+ * `name: "Secondi"` and `employee_name="Nguyen Van A"` far more often than it is assigned to its
+ * own variable, so without it four real creation sites were misread as assertions.
+ */
+const CREATING = /\b(?:const|let|var)\s+[\w$]+\s*=|[\w$]+\s*[:=]\s*['"`]|\b(?:create|insert|add|register|signup|signUp|post|put|save|seed|make|new)\w*\s*\(|\.\s*(?:fill|type|values|send)\s*\(/;
+/**
+ * The value is naming an element, not creating a record. `getByRole('button', { name: 'Submit' })`
+ * reads exactly like an object property to the pattern above, and rewriting it would point the test
+ * at an element that does not exist - breaking the test rather than fixing the collision. A unit
+ * test caught this before any real data did.
+ */
+const A_SELECTOR = /\bgetBy\w+\s*\(|\bfindBy\w+\s*\(|\blocator\s*\(|\bcy\.get\s*\(|querySelector|\bBy\.\w+\s*\(|\bselector\b/i;
+/**
+ * Picks the one occurrence it is safe to make unique, given the whole file.
+ */
+export function chooseDataSite(source, sites, value) {
+    const asserting = sites.filter((s) => ASSERTING.test(lineOf(source, s)));
+    const creating = sites.filter((s) => {
+        const line = lineOf(source, s);
+        return CREATING.test(line) && !ASSERTING.test(line) && !A_SELECTOR.test(line);
+    });
+    if (creating.length === 0) {
+        return {
+            site: null,
+            reason: 'value-only-asserted',
+            why: sites.length === 1
+                ? `"${value}" appears only in ${lineOf(source, sites[0]).trim().slice(0, 60)}, where the test is checking it rather than creating anything. Making that unique would change what the test asserts instead of fixing the collision.`
+                : `"${value}" is written ${sites.length} times in the test and none of them creates anything: they are assertions or arguments Tewip cannot tell apart. A person should decide which one is the data.`,
+        };
+    }
+    if (creating.length > 1) {
+        return {
+            site: null,
+            reason: 'value-ambiguous',
+            why: `"${value}" is created in ${creating.length} places (lines ${creating.map((s) => s.line).join(', ')}). Making only one unique would break the link between them, so this needs a person.`,
+        };
+    }
+    if (asserting.length > 0) {
+        // The commonest real shape of all: `name: "Secondi"` on one line and
+        // `expect(body.data.name).toBe("Secondi")` a few lines down. Making the data unique without
+        // rewriting the assertion turns a flaky test into a failing one, and rewriting both is a bigger
+        // change than this fix is allowed to make.
+        return {
+            site: null,
+            reason: 'value-also-asserted',
+            why: `"${value}" is created on line ${creating[0].line} and also checked on line ${asserting[0].line}. Making the data unique would leave the assertion looking for the old value, so the two have to change together - which is a person's call, not this fix's.`,
+        };
+    }
+    return {
+        site: creating[0],
+        why: sites.length === 1 ? `"${value}" is written once in the test, where the record is created` : `"${value}" is written ${sites.length} times; line ${creating[0].line} is the one that creates the record`,
+    };
+}
+const lineOf = (source, site) => source.split('\n')[site.line - 1] ?? '';

package/dist/engine/autofix/fix-data.js

@@ -8,6 +8,7 @@
 // Same sandbox as the other fixes: test files only, never application code, and the file goes back
 // exactly as it was unless a candidate is accepted.
 import { relative, resolve, sep } from 'node:path';
+import { chooseDataSite } from './data-site.js';
 import { applyDataFix, collision, dataFixCandidates, findLiteral, languageOf } from './data.js';
 /** Two passes in a row: the second run is what proves the value is really unique. */
 export const REQUIRED_PASSES = 2;
@@ -34,7 +35,13 @@ export async function attemptDataFix({ record, diagnosis, action, specFile }, de
     const sites = findLiteral(source, found.value);
     if (sites.length === 0)
         return stop('value-not-in-test', `"${found.value}" is not written in ${specFile}: it comes from a fixture or a factory, which a person should make unique`);
-    const site = sites[0];
+    // Not simply the first occurrence: a colliding value is often also a value the test checks, and
+    // rewriting that would change what the test asserts while leaving the collision in place.
+    const choice = chooseDataSite(source, sites, found.value);
+    if (!choice.site)
+        return stop(choice.reason, `refused: ${choice.why}`);
+    const site = choice.site;
+    result.log.push(choice.why);
     result.line = site.line;
     result.original = site.text;
     const candidates = dataFixCandidates(found.value, languageOf(specFile));

package/dist/engine/autofix/locate.js

@@ -69,10 +69,17 @@ export function hasAssertionAfter(source, line) {
     for (let i = line - 1; i < lines.length; i++) {
         if (i > line - 1 && /^\s*test(\.\w+)?\(/.test(lines[i]))
             return false;
-        if (/\bexpect\s*\(/.test(lines[i]))
+        if (ASSERTION.test(lines[i]))
             return true;
         if (/^\}\);?\s*$/.test(lines[i]))
             return false;
     }
     return false;
 }
+/**
+ * What counts as checking something. `expect(` alone missed a whole framework: Cypress asserts with
+ * `cy.get(...).should('be.visible')` and writes `expect(` only inside `.then()`. Six real Cypress
+ * fixed-wait removals were refused as "nothing is asserted after the wait" when the very next line
+ * was an assertion, which is how this was found. Python's bare `assert` counts for the same reason.
+ */
+const ASSERTION = /\bexpect\s*\(|\.should\s*\(|\bassert\s+|\bassertThat\s*\(/;

package/dist/engine/autofix/wait-candidates.js

@@ -0,0 +1,61 @@
+// What to put in place of a fixed wait, and - more important - when there is nothing safe to put.
+//
+// Split out of waits.ts after the first measurement of this fix against real commits: 60 commits
+// from 47 public repositories in which a developer removed one fixed wait from a test. Two things
+// that measurement showed:
+//
+// 1. Tewip offered a second option in 0 of 43 proposals. The explicit-wait candidate existed but
+//    almost never fired, because it only recognised a Playwright action written in one shape. So in
+//    practice Tewip could only ever say "delete the wait", and disagreed with the developer in 27
+//    cases where they waited for something instead.
+// 2. More serious: deleting a fixed wait is only safe when the *next* step waits by itself.
+//    `await expect(locator).toBeVisible()` retries until it passes, so the fixed wait before it is
+//    pure delay. `const url = page.url()`, `.count()`, `.textContent()` and `if (await x.isEnabled())`
+//    read a value once and never retry - and those were common in the real commits. Deleting the
+//    wait in front of one of those removes the only thing making the read reliable, and because a
+//    race passes by luck, even three passing runs can fail to catch it.
+//
+// So a bare deletion is now offered only in front of a step that retries on its own. Everywhere
+// else Tewip must name something to wait for, or refuse.
+/** A step that waits by itself, so a fixed wait in front of it only adds time. */
+export function nextStepRetries(line, framework) {
+    if (framework === 'cypress') {
+        // Every cy.* query retries until it resolves, except cy.wait itself.
+        return /\bcy\.(?!wait\b)\w+\s*\(/.test(line);
+    }
+    if (framework !== 'playwright')
+        return false;
+    // `await expect(page.getByRole(...)).toBeVisible()` retries; `expect(page.url())` and
+    // `expect(await x.count())` read a value once and do not.
+    const assertion = /\bexpect\s*\(\s*([^)]*)/.exec(line);
+    if (assertion && !/\bawait\b/.test(assertion[1]) && LOCATOR_EXPRESSION.test(assertion[1]))
+        return true;
+    // A step that is itself a wait, or a navigation, already blocks until the page is ready: a fixed
+    // wait in front of `await this.cancelButton().waitFor({ state: 'visible' })` is pure delay, and
+    // that exact line appeared in the real commits.
+    if (SELF_WAITING.test(line))
+        return true;
+    // An action on a locator waits for that element to be actionable first.
+    return ACTION_ON_LOCATOR.test(line);
+}
+const SELF_WAITING = /\.\s*waitFor(?:Selector|URL|LoadState|Function|Event|Response|Request|Navigation)?\s*\(|\.\s*goto\s*\(/;
+const LOCATOR_EXPRESSION = /(?:page|frame|component|[\w$]+)\s*\.\s*(?:getBy\w+|locator|frameLocator)\s*\(|^\s*[\w$]+\s*$/;
+const ACTION_ON_LOCATOR = /(?:getBy\w+\([^;]*?\)|locator\([^;]*?\))(?:\s*\.\s*\w+\([^;]*?\))*?\s*\.\s*(?:click|fill|press|check|uncheck|selectOption|hover|type|tap|dragTo|setInputFiles|focus)\s*\(/;
+/**
+ * The thing worth waiting for instead of the guess: whatever the next step acts on or reads.
+ * Recognising only `locator.click()` was why the explicit-wait option never appeared in practice;
+ * real code just as often assigns the locator first, asserts on it, or reads from it.
+ */
+export function thingToWaitFor(line, framework) {
+    if (framework === 'cypress') {
+        const m = /\bcy\.get\s*\(\s*([^)]+?)\s*\)/.exec(line);
+        return m ? `cy.get(${m[1]})` : null;
+    }
+    const chain = /((?:page|frame|component|[\w$]+)\s*\.\s*(?:getBy\w+|locator|frameLocator)\s*\([^;]*?\)(?:\s*\.\s*(?:first|last|nth|filter|getBy\w+|locator)\s*\([^;]*?\))*)/.exec(line);
+    if (!chain)
+        return null;
+    const expression = chain[1].trim();
+    // `await page.locator('x').count()` and `const n = page.locator('x')` both point at the same
+    // element; the expression itself is what to wait for either way.
+    return expression.startsWith('.') ? `page${expression}` : expression;
+}

package/dist/engine/autofix/waits.js

@@ -6,17 +6,30 @@
 // time, so one passing run proves the fix. A race passes by luck, so one passing run proves nothing.
 // A wait fix is therefore only accepted after several passes in a row, and the whole change is
 // deleting or replacing a line whose only job was to wait.
+// What may follow the wait on its own line and still leave it a pure wait: a semicolon, and a
+// comment. Real code nearly always carries one - `cy.wait(500); // eslint-disable-line` is written
+// precisely *because* the line is a fixed wait, and `waitForTimeout(300); // allow debounce` says
+// what the guess was for. Requiring the line to end after the call made Tewip refuse 15 of 60 real
+// fixed-wait removals harvested from public repositories, which is how this was found.
+const TRAILING = String.raw `\s*;?\s*(?://.*)?$`;
+const TRAILING_PY = String.raw `\s*(?:#.*)?$`;
+/** Digits, allowing JavaScript's numeric separators: `3_000` is as common as `3000`. */
+const MS = String.raw `[\d_]+`;
+import { nextStepRetries, thingToWaitFor } from './wait-candidates.js';
 const FIXED_WAIT = [
     // Playwright and Puppeteer
-    /^(?<indent>\s*)(?:await\s+)?[\w.]*page\.waitForTimeout\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?[\w.]*page\.waitForTimeout\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // Cypress
-    /^(?<indent>\s*)cy\.wait\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)cy\.wait\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // WebdriverIO and anything with a sleep helper
-    /^(?<indent>\s*)(?:await\s+)?(?:browser\.pause|sleep|delay)\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?(?:browser\.pause|sleep|delay)\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
+    // The hand-rolled sleep, which is what people write when no helper exists:
+    // `await new Promise(resolve => setTimeout(resolve, 500))`.
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?new\s+Promise\(\s*\(?\s*(?<arg>\w+)\s*\)?\s*=>\s*setTimeout\(\s*\k<arg>\s*,\s*(?<ms>${MS})\s*\)\s*\)${TRAILING}`),
     // Java and Kotlin
-    /^(?<indent>\s*)Thread\.sleep\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)Thread\.sleep\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // Python
-    /^(?<indent>\s*)time\.sleep\(\s*(?<seconds>[\d.]+)\s*\)\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)time\.sleep\(\s*(?<seconds>[\d._]+)\s*\)${TRAILING_PY}`),
 ];
 /** Every fixed wait in the file, in order. */
 export function findFixedWaits(source) {
@@ -26,7 +39,8 @@ export function findFixedWaits(source) {
             const m = line.match(pattern);
             if (!m?.groups)
                 continue;
-            const ms = m.groups.seconds ? Math.round(Number(m.groups.seconds) * 1000) : Number(m.groups.ms);
+            const digits = (text) => Number(text.replace(/_/g, ''));
+            const ms = m.groups.seconds ? Math.round(digits(m.groups.seconds) * 1000) : digits(m.groups.ms);
             out.push({ line: i + 1, text: line.trim(), ms, indent: m.groups.indent ?? '' });
             break;
         }
@@ -39,36 +53,33 @@ export function waitNearest(sites, failing) {
         return sites[0];
     return [...sites].sort((a, b) => Math.abs(a.line - failing) - Math.abs(b.line - failing))[0];
 }
-/** The Playwright locator a line acts on, when it has one: the thing worth waiting for instead. */
-function locatorIn(line) {
-    const m = line.match(/((?:page|frame|component)?\.?(?:getBy\w+\([^;]*?\)|locator\([^;]*?\))(?:\.\w+\([^;]*?\))*?)\.(?:click|fill|press|check|selectOption|hover|type|textContent|innerText|isVisible)\(/);
-    return m ? m[1].replace(/^\./, 'page.') : null;
-}
 /**
- * What to put in place of the fixed wait, best first.
+ * What to put in place of the fixed wait, best first - and nothing at all when nothing is safe.
  *
- * Removing it is first on purpose: Playwright and Cypress already wait for an element to be
- * actionable before clicking or asserting, so a fixed wait in front of such a step adds nothing but
- * time, and deleting it is the smallest possible change. Where the next line does act on an element,
- * waiting for that element is offered too, for suites whose style is an explicit wait.
+ * Deleting it comes first only where the next step waits by itself: Playwright and Cypress wait for
+ * an element before acting on it, and a web-first assertion retries until it passes, so a fixed wait
+ * in front of one of those is pure delay. In front of a step that reads a value once -
+ * `page.url()`, `.count()`, `if (await x.isEnabled())` - the fixed wait may be the only thing making
+ * the read reliable, so deletion is not offered there at all and Tewip must name something to wait
+ * for or refuse. Measured against 60 real fixed-wait removals in 47 public repositories.
  */
 export function waitFixCandidates(source, site, framework) {
     const lines = source.split('\n');
     const next = lines.slice(site.line, site.line + 3).find((l) => l.trim() && !l.trim().startsWith('//'));
-    const autoWaits = framework === 'playwright' || framework === 'cypress';
+    if (!next)
+        return [];
     const candidates = [];
-    if (autoWaits && next) {
+    if (nextStepRetries(next, framework)) {
         candidates.push({
             replacement: '',
-            why: `${framework === 'playwright' ? 'Playwright' : 'Cypress'} already waits for the element in the next step to be ready, so the fixed ${site.ms} ms wait only added time.`,
+            why: `${framework === 'cypress' ? 'Cypress' : 'Playwright'} already waits for the next step to be ready, so the fixed ${site.ms} ms wait only added time.`,
         });
     }
-    const locator = next ? locatorIn(next) : null;
-    if (locator) {
-        candidates.push({
-            replacement: `${site.indent}await expect(${locator}).toBeVisible();`,
-            why: `Waits for ${locator} to be there instead of guessing ${site.ms} ms.`,
-        });
+    const thing = thingToWaitFor(next, framework);
+    if (thing) {
+        candidates.push(framework === 'cypress'
+            ? { replacement: `${site.indent}${thing}.should('be.visible');`, why: `Waits for ${thing} to be there instead of guessing ${site.ms} ms.` }
+            : { replacement: `${site.indent}await expect(${thing}).toBeVisible();`, why: `Waits for ${thing} to be there instead of guessing ${site.ms} ms.` });
     }
     return candidates;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.15.1",
+  "version": "0.16.1",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",