tewip-ai 0.15.0 → 0.15.1

package/dist/engine/learn/learn.js

@@ -25,12 +25,11 @@ function sharedWords(errors, min = 0.6) {
  */
 function ruleOf(record) {
     const diagnosis = record?.diagnosis;
-    if (diagnosis?.ruleId)
-        return diagnosis.ruleId;
-    for (const e of diagnosis?.evidence ?? [])
-        if (e.source && e.source !== 'engine' && e.source !== 'llm')
-            return e.source;
-    return '';
+    // Only a diagnosis a rule actually decided has a rule to blame. Reading a name out of the
+    // evidence instead was wrong twice over: a rule that *declined* still leaves evidence under its
+    // own name, so failures the model answered were being blamed on a rule that had explicitly said
+    // "not mine". A report that misnames the rule sends someone to read the wrong code.
+    return diagnosis?.source === 'rules' ? (diagnosis.ruleId ?? '') : '';
 }
 const layerOf = (record) => String(record?.diagnosis?.source ?? 'unknown');
 /** "rules 5, llm 2" - said in full, because "2 different layers" tells a person nothing. */
@@ -161,10 +160,11 @@ export function renderLearning(learned) {
             lines.push(`  - **${example.testId}** — ${example.error}${example.why ? ` (you said: ${example.why})` : ''}`);
         lines.push('');
         const topRule = mistake.rules[0];
+        const ruleDecided = mistake.rules.reduce((sum, r) => sum + r.count, 0);
         lines.push(mistake.rule
             ? `What to do: read \`${mistake.rule}\` against those failures - it decided ${topRule.count} of ${mistake.count}. If it is matching something it should not, narrow it; if a rule for ${mistake.corrected} should have matched first, order that one first.`
             : topRule
-                ? `What to do: no single rule is behind these - ${mistake.rules.length} different ones are, the commonest being \`${topRule.rule}\` (${topRule.count} of ${mistake.count}). That points at a missing rule for ${mistake.corrected} rather than a wrong one: the words above are what it would match on.`
+                ? `What to do: no rule is behind most of these. ${mistake.rules.length === 1 ? `\`${topRule.rule}\` decided ${topRule.count} of ${mistake.count}` : `${mistake.rules.length} different rules decided ${ruleDecided} of ${mistake.count}, the commonest being \`${topRule.rule}\``}; the model was left to answer the other ${mistake.count - ruleDecided}. That points at a missing rule for ${mistake.corrected} rather than a wrong one - one that settles these before the model is asked: the words above are what it would match on.`
                 : `What to do: no rule decided these, so the model was left to answer them. A rule for ${mistake.corrected} matching the words above would settle them without asking it.`);
         lines.push('');
     }

package/dist/engine/llm/prompt.js

@@ -3,28 +3,28 @@
 import { ROOT_CAUSES } from '../../reporters/types.js';
 /** Bump whenever the prompt, schema or context format changes; it is part of the cache key. */
 export const PROMPT_VERSION = 'llm-6';
-export const SYSTEM_PROMPT = `You are the root-cause classifier of a Playwright test-failure triage agent.
-You receive a structured summary of ONE failed test: the error, the test's run history, what the deterministic rules already checked, the test steps, the page at failure time (accessibility snapshot + notable DOM elements), failed network requests and console errors.
-
-Classify the root cause into exactly one of these categories:
-- REAL_BUG: the application under test behaves incorrectly. Examples: a wrong value is displayed; a feature or element the test needs is absent from the page and nothing equivalent exists; the app never reaches the expected state even though nothing external failed; uncaught page errors from application code.
-- LOCATOR_DRIFT: the element the test wants is on the page, but the selector no longer matches it (renamed id, class, test id or text). The page summary must show an element with the same purpose.
-- TIMING: the expected element or state does arrive, but not in time, or a race makes the result vary between attempts or runs. Signs: another attempt of the same code passed, intermittent history with varying errors, hard-coded waits, the element resolved but was not yet visible, stable or enabled.
-- ENV: something outside the app code and the test broke: failed or 5xx/502/503 responses from a backend or third-party service, connection or DNS errors, expired auth or credentials, or many tests in the same run failing the same way.
-- TEST_DATA: the test's preconditions or fixture data are wrong: missing or stale seeded data, a user or account in the wrong state, an expected value that depends on data the test did not set up, a 404 for a record the test assumed exists.
-
-How to decide:
-- Use only facts present in the summary. Do not invent page content, requests or history.
-- Prefer the category the evidence directly shows over one that is merely possible.
-- An element that is missing with no similar element on the page is not LOCATOR_DRIFT.
-- Playwright matches accessible names by substring, so a locator can match several elements at once ("strict mode violation"). That is LOCATOR_DRIFT only when one of the matched elements is the one the test means, which usually carries the locator's name as a whole word ("Edit" -> "Edit link"). When every match only contains the text inside a longer word or phrase ("Edit" -> "Editor canvas", "Gift" -> "Gifting options"), the element the test means is absent: judge it as REAL_BUG, TIMING or TEST_DATA on the other evidence, not LOCATOR_DRIFT.
-- A failed request or server error is only ENV evidence if the test did not provoke it. When "test.exercisesErrorHandling" is set, the test deliberately triggers failures (rejections, server errors, retries), so a failed request is the scenario under test: judge whether the app handled it as the test expects. If it did not, that is REAL_BUG, not ENV. A real environment outage also tends to hit unrelated tests and to pass on retry, rather than failing every attempt on one commit while other commits pass.
-- Check "thisRun" first. If anotherAttemptPassed is true, the same code both failed and passed, so the cause is not deterministic: a real bug or a drifted locator would fail every attempt. Choose among TIMING, ENV and TEST_DATA.
-- Then read "history.pattern", which compares this commit's attempts with other commits. If every attempt on this commit failed while other commits passed, something in this commit changed behaviour: choose among REAL_BUG, LOCATOR_DRIFT and TEST_DATA using the page summary, not TIMING. An overall failure rate that mixes different commits is not evidence of flakiness; only failing and passing on the same code is.
-- The "rules" section lists what the deterministic rules already examined and why they declined; treat those as checked facts.
-
-Confidence (0 to 1): 0.9 or higher only when the summary directly shows the cause; 0.6-0.8 when it is the most likely reading but another category remains plausible; below 0.5 when you are mostly guessing. Low confidence routes the failure to a human, so do not inflate it.
-
+export const SYSTEM_PROMPT = `You are the root-cause classifier of a Playwright test-failure triage agent.
+You receive a structured summary of ONE failed test: the error, the test's run history, what the deterministic rules already checked, the test steps, the page at failure time (accessibility snapshot + notable DOM elements), failed network requests and console errors.
+
+Classify the root cause into exactly one of these categories:
+- REAL_BUG: the application under test behaves incorrectly. Examples: a wrong value is displayed; a feature or element the test needs is absent from the page and nothing equivalent exists; the app never reaches the expected state even though nothing external failed; uncaught page errors from application code.
+- LOCATOR_DRIFT: the element the test wants is on the page, but the selector no longer matches it (renamed id, class, test id or text). The page summary must show an element with the same purpose.
+- TIMING: the expected element or state does arrive, but not in time, or a race makes the result vary between attempts or runs. Signs: another attempt of the same code passed, intermittent history with varying errors, hard-coded waits, the element resolved but was not yet visible, stable or enabled.
+- ENV: something outside the app code and the test broke: failed or 5xx/502/503 responses from a backend or third-party service, connection or DNS errors, expired auth or credentials, or many tests in the same run failing the same way.
+- TEST_DATA: the test's preconditions or fixture data are wrong: missing or stale seeded data, a user or account in the wrong state, an expected value that depends on data the test did not set up, a 404 for a record the test assumed exists.
+
+How to decide:
+- Use only facts present in the summary. Do not invent page content, requests or history.
+- Prefer the category the evidence directly shows over one that is merely possible.
+- An element that is missing with no similar element on the page is not LOCATOR_DRIFT.
+- Playwright matches accessible names by substring, so a locator can match several elements at once ("strict mode violation"). That is LOCATOR_DRIFT only when one of the matched elements is the one the test means, which usually carries the locator's name as a whole word ("Edit" -> "Edit link"). When every match only contains the text inside a longer word or phrase ("Edit" -> "Editor canvas", "Gift" -> "Gifting options"), the element the test means is absent: judge it as REAL_BUG, TIMING or TEST_DATA on the other evidence, not LOCATOR_DRIFT.
+- A failed request or server error is only ENV evidence if the test did not provoke it. When "test.exercisesErrorHandling" is set, the test deliberately triggers failures (rejections, server errors, retries), so a failed request is the scenario under test: judge whether the app handled it as the test expects. If it did not, that is REAL_BUG, not ENV. A real environment outage also tends to hit unrelated tests and to pass on retry, rather than failing every attempt on one commit while other commits pass.
+- Check "thisRun" first. If anotherAttemptPassed is true, the same code both failed and passed, so the cause is not deterministic: a real bug or a drifted locator would fail every attempt. Choose among TIMING, ENV and TEST_DATA.
+- Then read "history.pattern", which compares this commit's attempts with other commits. If every attempt on this commit failed while other commits passed, something in this commit changed behaviour: choose among REAL_BUG, LOCATOR_DRIFT and TEST_DATA using the page summary, not TIMING. An overall failure rate that mixes different commits is not evidence of flakiness; only failing and passing on the same code is.
+- The "rules" section lists what the deterministic rules already examined and why they declined; treat those as checked facts.
+
+Confidence (0 to 1): 0.9 or higher only when the summary directly shows the cause; 0.6-0.8 when it is the most likely reading but another category remains plausible; below 0.5 when you are mostly guessing. Low confidence routes the failure to a human, so do not inflate it.
+
 Return JSON only: the category, the confidence, a short reasoning (at most 3 sentences), and 1-4 evidence items quoting or pointing at specific facts from the summary.`;
 export const OUTPUT_SCHEMA_NAME = 'triage_diagnosis';
 export const OUTPUT_SCHEMA = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.15.0",
+  "version": "0.15.1",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",