tewip-ai 0.15.0 → 0.16.0

package/dist/engine/autofix/locate.js

@@ -69,10 +69,17 @@ export function hasAssertionAfter(source, line) {
     for (let i = line - 1; i < lines.length; i++) {
         if (i > line - 1 && /^\s*test(\.\w+)?\(/.test(lines[i]))
             return false;
-        if (/\bexpect\s*\(/.test(lines[i]))
+        if (ASSERTION.test(lines[i]))
             return true;
         if (/^\}\);?\s*$/.test(lines[i]))
             return false;
     }
     return false;
 }
+/**
+ * What counts as checking something. `expect(` alone missed a whole framework: Cypress asserts with
+ * `cy.get(...).should('be.visible')` and writes `expect(` only inside `.then()`. Six real Cypress
+ * fixed-wait removals were refused as "nothing is asserted after the wait" when the very next line
+ * was an assertion, which is how this was found. Python's bare `assert` counts for the same reason.
+ */
+const ASSERTION = /\bexpect\s*\(|\.should\s*\(|\bassert\s+|\bassertThat\s*\(/;

package/dist/engine/autofix/wait-candidates.js

@@ -0,0 +1,61 @@
+// What to put in place of a fixed wait, and - more important - when there is nothing safe to put.
+//
+// Split out of waits.ts after the first measurement of this fix against real commits: 60 commits
+// from 47 public repositories in which a developer removed one fixed wait from a test. Two things
+// that measurement showed:
+//
+// 1. Tewip offered a second option in 0 of 43 proposals. The explicit-wait candidate existed but
+//    almost never fired, because it only recognised a Playwright action written in one shape. So in
+//    practice Tewip could only ever say "delete the wait", and disagreed with the developer in 27
+//    cases where they waited for something instead.
+// 2. More serious: deleting a fixed wait is only safe when the *next* step waits by itself.
+//    `await expect(locator).toBeVisible()` retries until it passes, so the fixed wait before it is
+//    pure delay. `const url = page.url()`, `.count()`, `.textContent()` and `if (await x.isEnabled())`
+//    read a value once and never retry - and those were common in the real commits. Deleting the
+//    wait in front of one of those removes the only thing making the read reliable, and because a
+//    race passes by luck, even three passing runs can fail to catch it.
+//
+// So a bare deletion is now offered only in front of a step that retries on its own. Everywhere
+// else Tewip must name something to wait for, or refuse.
+/** A step that waits by itself, so a fixed wait in front of it only adds time. */
+export function nextStepRetries(line, framework) {
+    if (framework === 'cypress') {
+        // Every cy.* query retries until it resolves, except cy.wait itself.
+        return /\bcy\.(?!wait\b)\w+\s*\(/.test(line);
+    }
+    if (framework !== 'playwright')
+        return false;
+    // `await expect(page.getByRole(...)).toBeVisible()` retries; `expect(page.url())` and
+    // `expect(await x.count())` read a value once and do not.
+    const assertion = /\bexpect\s*\(\s*([^)]*)/.exec(line);
+    if (assertion && !/\bawait\b/.test(assertion[1]) && LOCATOR_EXPRESSION.test(assertion[1]))
+        return true;
+    // A step that is itself a wait, or a navigation, already blocks until the page is ready: a fixed
+    // wait in front of `await this.cancelButton().waitFor({ state: 'visible' })` is pure delay, and
+    // that exact line appeared in the real commits.
+    if (SELF_WAITING.test(line))
+        return true;
+    // An action on a locator waits for that element to be actionable first.
+    return ACTION_ON_LOCATOR.test(line);
+}
+const SELF_WAITING = /\.\s*waitFor(?:Selector|URL|LoadState|Function|Event|Response|Request|Navigation)?\s*\(|\.\s*goto\s*\(/;
+const LOCATOR_EXPRESSION = /(?:page|frame|component|[\w$]+)\s*\.\s*(?:getBy\w+|locator|frameLocator)\s*\(|^\s*[\w$]+\s*$/;
+const ACTION_ON_LOCATOR = /(?:getBy\w+\([^;]*?\)|locator\([^;]*?\))(?:\s*\.\s*\w+\([^;]*?\))*?\s*\.\s*(?:click|fill|press|check|uncheck|selectOption|hover|type|tap|dragTo|setInputFiles|focus)\s*\(/;
+/**
+ * The thing worth waiting for instead of the guess: whatever the next step acts on or reads.
+ * Recognising only `locator.click()` was why the explicit-wait option never appeared in practice;
+ * real code just as often assigns the locator first, asserts on it, or reads from it.
+ */
+export function thingToWaitFor(line, framework) {
+    if (framework === 'cypress') {
+        const m = /\bcy\.get\s*\(\s*([^)]+?)\s*\)/.exec(line);
+        return m ? `cy.get(${m[1]})` : null;
+    }
+    const chain = /((?:page|frame|component|[\w$]+)\s*\.\s*(?:getBy\w+|locator|frameLocator)\s*\([^;]*?\)(?:\s*\.\s*(?:first|last|nth|filter|getBy\w+|locator)\s*\([^;]*?\))*)/.exec(line);
+    if (!chain)
+        return null;
+    const expression = chain[1].trim();
+    // `await page.locator('x').count()` and `const n = page.locator('x')` both point at the same
+    // element; the expression itself is what to wait for either way.
+    return expression.startsWith('.') ? `page${expression}` : expression;
+}

package/dist/engine/autofix/waits.js

@@ -6,17 +6,30 @@
 // time, so one passing run proves the fix. A race passes by luck, so one passing run proves nothing.
 // A wait fix is therefore only accepted after several passes in a row, and the whole change is
 // deleting or replacing a line whose only job was to wait.
+// What may follow the wait on its own line and still leave it a pure wait: a semicolon, and a
+// comment. Real code nearly always carries one - `cy.wait(500); // eslint-disable-line` is written
+// precisely *because* the line is a fixed wait, and `waitForTimeout(300); // allow debounce` says
+// what the guess was for. Requiring the line to end after the call made Tewip refuse 15 of 60 real
+// fixed-wait removals harvested from public repositories, which is how this was found.
+const TRAILING = String.raw `\s*;?\s*(?://.*)?$`;
+const TRAILING_PY = String.raw `\s*(?:#.*)?$`;
+/** Digits, allowing JavaScript's numeric separators: `3_000` is as common as `3000`. */
+const MS = String.raw `[\d_]+`;
+import { nextStepRetries, thingToWaitFor } from './wait-candidates.js';
 const FIXED_WAIT = [
     // Playwright and Puppeteer
-    /^(?<indent>\s*)(?:await\s+)?[\w.]*page\.waitForTimeout\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?[\w.]*page\.waitForTimeout\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // Cypress
-    /^(?<indent>\s*)cy\.wait\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)cy\.wait\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // WebdriverIO and anything with a sleep helper
-    /^(?<indent>\s*)(?:await\s+)?(?:browser\.pause|sleep|delay)\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?(?:browser\.pause|sleep|delay)\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
+    // The hand-rolled sleep, which is what people write when no helper exists:
+    // `await new Promise(resolve => setTimeout(resolve, 500))`.
+    new RegExp(String.raw `^(?<indent>\s*)(?:await\s+)?new\s+Promise\(\s*\(?\s*(?<arg>\w+)\s*\)?\s*=>\s*setTimeout\(\s*\k<arg>\s*,\s*(?<ms>${MS})\s*\)\s*\)${TRAILING}`),
     // Java and Kotlin
-    /^(?<indent>\s*)Thread\.sleep\(\s*(?<ms>\d+)\s*\)\s*;?\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)Thread\.sleep\(\s*(?<ms>${MS})\s*\)${TRAILING}`),
     // Python
-    /^(?<indent>\s*)time\.sleep\(\s*(?<seconds>[\d.]+)\s*\)\s*$/,
+    new RegExp(String.raw `^(?<indent>\s*)time\.sleep\(\s*(?<seconds>[\d._]+)\s*\)${TRAILING_PY}`),
 ];
 /** Every fixed wait in the file, in order. */
 export function findFixedWaits(source) {
@@ -26,7 +39,8 @@ export function findFixedWaits(source) {
             const m = line.match(pattern);
             if (!m?.groups)
                 continue;
-            const ms = m.groups.seconds ? Math.round(Number(m.groups.seconds) * 1000) : Number(m.groups.ms);
+            const digits = (text) => Number(text.replace(/_/g, ''));
+            const ms = m.groups.seconds ? Math.round(digits(m.groups.seconds) * 1000) : digits(m.groups.ms);
             out.push({ line: i + 1, text: line.trim(), ms, indent: m.groups.indent ?? '' });
             break;
         }
@@ -39,36 +53,33 @@ export function waitNearest(sites, failing) {
         return sites[0];
     return [...sites].sort((a, b) => Math.abs(a.line - failing) - Math.abs(b.line - failing))[0];
 }
-/** The Playwright locator a line acts on, when it has one: the thing worth waiting for instead. */
-function locatorIn(line) {
-    const m = line.match(/((?:page|frame|component)?\.?(?:getBy\w+\([^;]*?\)|locator\([^;]*?\))(?:\.\w+\([^;]*?\))*?)\.(?:click|fill|press|check|selectOption|hover|type|textContent|innerText|isVisible)\(/);
-    return m ? m[1].replace(/^\./, 'page.') : null;
-}
 /**
- * What to put in place of the fixed wait, best first.
+ * What to put in place of the fixed wait, best first - and nothing at all when nothing is safe.
  *
- * Removing it is first on purpose: Playwright and Cypress already wait for an element to be
- * actionable before clicking or asserting, so a fixed wait in front of such a step adds nothing but
- * time, and deleting it is the smallest possible change. Where the next line does act on an element,
- * waiting for that element is offered too, for suites whose style is an explicit wait.
+ * Deleting it comes first only where the next step waits by itself: Playwright and Cypress wait for
+ * an element before acting on it, and a web-first assertion retries until it passes, so a fixed wait
+ * in front of one of those is pure delay. In front of a step that reads a value once -
+ * `page.url()`, `.count()`, `if (await x.isEnabled())` - the fixed wait may be the only thing making
+ * the read reliable, so deletion is not offered there at all and Tewip must name something to wait
+ * for or refuse. Measured against 60 real fixed-wait removals in 47 public repositories.
  */
 export function waitFixCandidates(source, site, framework) {
     const lines = source.split('\n');
     const next = lines.slice(site.line, site.line + 3).find((l) => l.trim() && !l.trim().startsWith('//'));
-    const autoWaits = framework === 'playwright' || framework === 'cypress';
+    if (!next)
+        return [];
     const candidates = [];
-    if (autoWaits && next) {
+    if (nextStepRetries(next, framework)) {
         candidates.push({
             replacement: '',
-            why: `${framework === 'playwright' ? 'Playwright' : 'Cypress'} already waits for the element in the next step to be ready, so the fixed ${site.ms} ms wait only added time.`,
+            why: `${framework === 'cypress' ? 'Cypress' : 'Playwright'} already waits for the next step to be ready, so the fixed ${site.ms} ms wait only added time.`,
         });
     }
-    const locator = next ? locatorIn(next) : null;
-    if (locator) {
-        candidates.push({
-            replacement: `${site.indent}await expect(${locator}).toBeVisible();`,
-            why: `Waits for ${locator} to be there instead of guessing ${site.ms} ms.`,
-        });
+    const thing = thingToWaitFor(next, framework);
+    if (thing) {
+        candidates.push(framework === 'cypress'
+            ? { replacement: `${site.indent}${thing}.should('be.visible');`, why: `Waits for ${thing} to be there instead of guessing ${site.ms} ms.` }
+            : { replacement: `${site.indent}await expect(${thing}).toBeVisible();`, why: `Waits for ${thing} to be there instead of guessing ${site.ms} ms.` });
     }
     return candidates;
 }

package/dist/engine/learn/learn.js

@@ -25,12 +25,11 @@ function sharedWords(errors, min = 0.6) {
  */
 function ruleOf(record) {
     const diagnosis = record?.diagnosis;
-    if (diagnosis?.ruleId)
-        return diagnosis.ruleId;
-    for (const e of diagnosis?.evidence ?? [])
-        if (e.source && e.source !== 'engine' && e.source !== 'llm')
-            return e.source;
-    return '';
+    // Only a diagnosis a rule actually decided has a rule to blame. Reading a name out of the
+    // evidence instead was wrong twice over: a rule that *declined* still leaves evidence under its
+    // own name, so failures the model answered were being blamed on a rule that had explicitly said
+    // "not mine". A report that misnames the rule sends someone to read the wrong code.
+    return diagnosis?.source === 'rules' ? (diagnosis.ruleId ?? '') : '';
 }
 const layerOf = (record) => String(record?.diagnosis?.source ?? 'unknown');
 /** "rules 5, llm 2" - said in full, because "2 different layers" tells a person nothing. */
@@ -161,10 +160,11 @@ export function renderLearning(learned) {
             lines.push(`  - **${example.testId}** — ${example.error}${example.why ? ` (you said: ${example.why})` : ''}`);
         lines.push('');
         const topRule = mistake.rules[0];
+        const ruleDecided = mistake.rules.reduce((sum, r) => sum + r.count, 0);
         lines.push(mistake.rule
             ? `What to do: read \`${mistake.rule}\` against those failures - it decided ${topRule.count} of ${mistake.count}. If it is matching something it should not, narrow it; if a rule for ${mistake.corrected} should have matched first, order that one first.`
             : topRule
-                ? `What to do: no single rule is behind these - ${mistake.rules.length} different ones are, the commonest being \`${topRule.rule}\` (${topRule.count} of ${mistake.count}). That points at a missing rule for ${mistake.corrected} rather than a wrong one: the words above are what it would match on.`
+                ? `What to do: no rule is behind most of these. ${mistake.rules.length === 1 ? `\`${topRule.rule}\` decided ${topRule.count} of ${mistake.count}` : `${mistake.rules.length} different rules decided ${ruleDecided} of ${mistake.count}, the commonest being \`${topRule.rule}\``}; the model was left to answer the other ${mistake.count - ruleDecided}. That points at a missing rule for ${mistake.corrected} rather than a wrong one - one that settles these before the model is asked: the words above are what it would match on.`
                 : `What to do: no rule decided these, so the model was left to answer them. A rule for ${mistake.corrected} matching the words above would settle them without asking it.`);
         lines.push('');
     }

package/dist/engine/llm/prompt.js

@@ -3,28 +3,28 @@
 import { ROOT_CAUSES } from '../../reporters/types.js';
 /** Bump whenever the prompt, schema or context format changes; it is part of the cache key. */
 export const PROMPT_VERSION = 'llm-6';
-export const SYSTEM_PROMPT = `You are the root-cause classifier of a Playwright test-failure triage agent.
-You receive a structured summary of ONE failed test: the error, the test's run history, what the deterministic rules already checked, the test steps, the page at failure time (accessibility snapshot + notable DOM elements), failed network requests and console errors.
-
-Classify the root cause into exactly one of these categories:
-- REAL_BUG: the application under test behaves incorrectly. Examples: a wrong value is displayed; a feature or element the test needs is absent from the page and nothing equivalent exists; the app never reaches the expected state even though nothing external failed; uncaught page errors from application code.
-- LOCATOR_DRIFT: the element the test wants is on the page, but the selector no longer matches it (renamed id, class, test id or text). The page summary must show an element with the same purpose.
-- TIMING: the expected element or state does arrive, but not in time, or a race makes the result vary between attempts or runs. Signs: another attempt of the same code passed, intermittent history with varying errors, hard-coded waits, the element resolved but was not yet visible, stable or enabled.
-- ENV: something outside the app code and the test broke: failed or 5xx/502/503 responses from a backend or third-party service, connection or DNS errors, expired auth or credentials, or many tests in the same run failing the same way.
-- TEST_DATA: the test's preconditions or fixture data are wrong: missing or stale seeded data, a user or account in the wrong state, an expected value that depends on data the test did not set up, a 404 for a record the test assumed exists.
-
-How to decide:
-- Use only facts present in the summary. Do not invent page content, requests or history.
-- Prefer the category the evidence directly shows over one that is merely possible.
-- An element that is missing with no similar element on the page is not LOCATOR_DRIFT.
-- Playwright matches accessible names by substring, so a locator can match several elements at once ("strict mode violation"). That is LOCATOR_DRIFT only when one of the matched elements is the one the test means, which usually carries the locator's name as a whole word ("Edit" -> "Edit link"). When every match only contains the text inside a longer word or phrase ("Edit" -> "Editor canvas", "Gift" -> "Gifting options"), the element the test means is absent: judge it as REAL_BUG, TIMING or TEST_DATA on the other evidence, not LOCATOR_DRIFT.
-- A failed request or server error is only ENV evidence if the test did not provoke it. When "test.exercisesErrorHandling" is set, the test deliberately triggers failures (rejections, server errors, retries), so a failed request is the scenario under test: judge whether the app handled it as the test expects. If it did not, that is REAL_BUG, not ENV. A real environment outage also tends to hit unrelated tests and to pass on retry, rather than failing every attempt on one commit while other commits pass.
-- Check "thisRun" first. If anotherAttemptPassed is true, the same code both failed and passed, so the cause is not deterministic: a real bug or a drifted locator would fail every attempt. Choose among TIMING, ENV and TEST_DATA.
-- Then read "history.pattern", which compares this commit's attempts with other commits. If every attempt on this commit failed while other commits passed, something in this commit changed behaviour: choose among REAL_BUG, LOCATOR_DRIFT and TEST_DATA using the page summary, not TIMING. An overall failure rate that mixes different commits is not evidence of flakiness; only failing and passing on the same code is.
-- The "rules" section lists what the deterministic rules already examined and why they declined; treat those as checked facts.
-
-Confidence (0 to 1): 0.9 or higher only when the summary directly shows the cause; 0.6-0.8 when it is the most likely reading but another category remains plausible; below 0.5 when you are mostly guessing. Low confidence routes the failure to a human, so do not inflate it.
-
+export const SYSTEM_PROMPT = `You are the root-cause classifier of a Playwright test-failure triage agent.
+You receive a structured summary of ONE failed test: the error, the test's run history, what the deterministic rules already checked, the test steps, the page at failure time (accessibility snapshot + notable DOM elements), failed network requests and console errors.
+
+Classify the root cause into exactly one of these categories:
+- REAL_BUG: the application under test behaves incorrectly. Examples: a wrong value is displayed; a feature or element the test needs is absent from the page and nothing equivalent exists; the app never reaches the expected state even though nothing external failed; uncaught page errors from application code.
+- LOCATOR_DRIFT: the element the test wants is on the page, but the selector no longer matches it (renamed id, class, test id or text). The page summary must show an element with the same purpose.
+- TIMING: the expected element or state does arrive, but not in time, or a race makes the result vary between attempts or runs. Signs: another attempt of the same code passed, intermittent history with varying errors, hard-coded waits, the element resolved but was not yet visible, stable or enabled.
+- ENV: something outside the app code and the test broke: failed or 5xx/502/503 responses from a backend or third-party service, connection or DNS errors, expired auth or credentials, or many tests in the same run failing the same way.
+- TEST_DATA: the test's preconditions or fixture data are wrong: missing or stale seeded data, a user or account in the wrong state, an expected value that depends on data the test did not set up, a 404 for a record the test assumed exists.
+
+How to decide:
+- Use only facts present in the summary. Do not invent page content, requests or history.
+- Prefer the category the evidence directly shows over one that is merely possible.
+- An element that is missing with no similar element on the page is not LOCATOR_DRIFT.
+- Playwright matches accessible names by substring, so a locator can match several elements at once ("strict mode violation"). That is LOCATOR_DRIFT only when one of the matched elements is the one the test means, which usually carries the locator's name as a whole word ("Edit" -> "Edit link"). When every match only contains the text inside a longer word or phrase ("Edit" -> "Editor canvas", "Gift" -> "Gifting options"), the element the test means is absent: judge it as REAL_BUG, TIMING or TEST_DATA on the other evidence, not LOCATOR_DRIFT.
+- A failed request or server error is only ENV evidence if the test did not provoke it. When "test.exercisesErrorHandling" is set, the test deliberately triggers failures (rejections, server errors, retries), so a failed request is the scenario under test: judge whether the app handled it as the test expects. If it did not, that is REAL_BUG, not ENV. A real environment outage also tends to hit unrelated tests and to pass on retry, rather than failing every attempt on one commit while other commits pass.
+- Check "thisRun" first. If anotherAttemptPassed is true, the same code both failed and passed, so the cause is not deterministic: a real bug or a drifted locator would fail every attempt. Choose among TIMING, ENV and TEST_DATA.
+- Then read "history.pattern", which compares this commit's attempts with other commits. If every attempt on this commit failed while other commits passed, something in this commit changed behaviour: choose among REAL_BUG, LOCATOR_DRIFT and TEST_DATA using the page summary, not TIMING. An overall failure rate that mixes different commits is not evidence of flakiness; only failing and passing on the same code is.
+- The "rules" section lists what the deterministic rules already examined and why they declined; treat those as checked facts.
+
+Confidence (0 to 1): 0.9 or higher only when the summary directly shows the cause; 0.6-0.8 when it is the most likely reading but another category remains plausible; below 0.5 when you are mostly guessing. Low confidence routes the failure to a human, so do not inflate it.
+
 Return JSON only: the category, the confidence, a short reasoning (at most 3 sentences), and 1-4 evidence items quoting or pointing at specific facts from the summary.`;
 export const OUTPUT_SCHEMA_NAME = 'triage_diagnosis';
 export const OUTPUT_SCHEMA = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",