tewip-ai 0.17.2 → 0.18.0

package/dist/engine/import/junit.js

@@ -47,7 +47,37 @@ function relativeFile(file, workspace) {
         return f.slice(ws.length);
     return f;
 }
+/**
+ * Does this report come from a run that re-ran tests? Either the runner said so with a rerun
+ * element, or some test in it is written twice with a failure recorded - both of which pytest's
+ * rerun plugin does and an ordinary passing run never does.
+ */
+function looksLikeReruns(root) {
+    let rerunElements = false;
+    let failureSeen = false;
+    const names = new Set();
+    let duplicateWithFailure = false;
+    const walk = (node) => {
+        for (const child of node.children) {
+            if (child.name === 'testcase') {
+                const key = `${child.attrs.classname ?? ''}::${child.attrs.name ?? ''}`;
+                const failed = child.children.some((c) => FAILURE.has(c.name));
+                if (child.children.some((c) => RETRY_FAILURE.has(c.name)))
+                    rerunElements = true;
+                if (failed)
+                    failureSeen = true;
+                if (names.has(key) && failureSeen)
+                    duplicateWithFailure = true;
+                names.add(key);
+            }
+            walk(child);
+        }
+    };
+    walk(root);
+    return rerunElements || duplicateWithFailure;
+}
 function junitCases(root, workspace) {
+    const rerunning = looksLikeReruns(root);
     const out = [];
     const walk = (e, suite) => {
         for (const c of e.children) {
@@ -70,10 +100,16 @@ function junitCases(root, workspace) {
         const previous = out.at(-1);
         if (previous?.testId === testId) {
             // Rerun plugins (pytest-rerunfailures and others) write every attempt as its own entry and
-            // only re-run after a failure, so an earlier entry of the same test is a failed attempt even
-            // when the plugin left its error out. Only adjacent entries are merged.
+            // only re-run after a failure, so an earlier entry of the same test can be a failed attempt
+            // whose error the plugin left out. Only adjacent entries are merged.
+            //
+            // The inference is only safe where the report shows this runner re-runs tests at all. Without
+            // that condition two adjacent entries of the same name were enough to invent a failure, and
+            // real suites produce those innocently: node:test repeats a name for a subtest and its
+            // parent, so piscina's own 152 passing tests came back as two failing ones, diagnosed as
+            // flaky. A test that passed must never be reported as having failed.
             const earlier = previous.attempts.at(-1);
-            if (!earlier.failed)
+            if (!earlier.failed && rerunning)
                 previous.attempts[previous.attempts.length - 1] = { failed: true, message: '', stack: '' };
             previous.attempts.push(...attempts);
             return;

package/dist/engine/missing-dependency.js

@@ -0,0 +1,90 @@
+import { truncate } from './signature.js';
+/** The module name each runner prints when it cannot load something. */
+const PATTERNS = [
+    { language: 'Python', pattern: /ModuleNotFoundError:\s*No module named ['"]([^'"]+)['"]/ },
+    { language: 'Node', pattern: /Cannot find module ['"]([^'"]+)['"]/ },
+    { language: 'Node', pattern: /ERR_MODULE_NOT_FOUND[^'"]*['"]([^'"]+)['"]/ },
+    { language: 'Java', pattern: /(?:NoClassDefFoundError|ClassNotFoundException):\s*([\w.$]+)/ },
+    { language: 'Ruby', pattern: /cannot load such file --\s*(\S+)/ },
+    { language: 'Go', pattern: /cannot find package ['"]([^'"]+)['"]/ },
+    { language: '.NET', pattern: /Could not load file or assembly ['"]([^'",]+)/ },
+    { language: 'PHP', pattern: /Failed opening required ['"]([^'"]+)['"]|Class ['"]([^'"]+)['"] not found/ },
+];
+// `ImportError: cannot import name 'x'` is deliberately not here. It usually means the code asked
+// for something that does not exist - a rename that missed a caller - which is a real bug, and only
+// sometimes a version mismatch. "No module named" says plainly that nothing could be found.
+/** A module that is part of this project rather than something it depends on. */
+const LOOKS_INTERNAL = /^\.|^(?:src|tests?|app|lib|internal|pkg|main)\b|_test$|\.spec$|\.test$/i;
+/** The file could be found but not understood, which is a mistake in the code, not the machine. */
+const A_CODE_ERROR = /\b(?:SyntaxError|IndentationError|TabError|TypeError|NameError|AttributeError|error TS\d+|Unexpected token)\b/;
+/**
+ * pytest reports a missing plugin as a missing fixture, which on its own is ambiguous: a typo in a
+ * fixture name reads exactly the same. What separates them is how many tests it hits. A typo breaks
+ * the one test that made it; an uninstalled plugin breaks every test that asks for its fixture.
+ *
+ * Found on humanize's real suite, where 15 tests failed with "fixture 'benchmark' not found". The
+ * model answered those 15 identical failures with 13 ENV and 2 TEST_DATA - the same input getting
+ * different answers, which is worse than either answer, and exactly what a rule is for.
+ */
+const MISSING_FIXTURE = /fixture ['"]([^'"]+)['"] not found/;
+export const FIXTURE_SHARED_BY = 3;
+export const fixtureMissing = (errorMessage) => MISSING_FIXTURE.exec(errorMessage)?.[1] ?? null;
+export function missingDependency(errorMessage) {
+    if (A_CODE_ERROR.test(errorMessage))
+        return null;
+    for (const { language, pattern } of PATTERNS) {
+        const found = pattern.exec(errorMessage);
+        const module = found?.slice(1).find(Boolean);
+        if (!module)
+            continue;
+        if (LOOKS_INTERNAL.test(module))
+            return null;
+        return { language, module };
+    }
+    return null;
+}
+/**
+ * Nothing about the test ran, so nothing about the test or the code has been shown. The suite needs
+ * a dependency the machine does not have.
+ */
+export const envMissingDependency = {
+    id: 'env-missing-dependency',
+    evaluate({ record, run }) {
+        const fixture = fixtureMissing(record.errorMessage);
+        if (fixture) {
+            const sharing = run.failures.filter((f) => fixtureMissing(f.errorMessage) === fixture).length;
+            if (sharing < FIXTURE_SHARED_BY)
+                return null; // one test only: a typo is as likely, so say nothing
+            return {
+                category: 'ENV',
+                confidence: 0.85,
+                evidence: [
+                    {
+                        source: 'env-missing-dependency',
+                        message: `${sharing} tests in this run all need the fixture "${truncate(fixture, 40)}" and none can find it: the plugin that provides it is not installed, so no test code ran`,
+                        data: { fixture, sharing },
+                    },
+                ],
+            };
+        }
+        const found = missingDependency(record.errorMessage);
+        if (!found)
+            return null;
+        // When the whole run failed this way, the install step is the obvious suspect and there is
+        // nothing test-specific left to doubt.
+        const wholeRun = run.failures.length >= 3 && run.failures.every((f) => missingDependency(f.errorMessage) !== null);
+        return {
+            category: 'ENV',
+            confidence: wholeRun ? 0.92 : 0.85,
+            evidence: [
+                {
+                    source: 'env-missing-dependency',
+                    message: wholeRun
+                        ? `every failure in this run is a module that could not be loaded (${found.language}: "${truncate(found.module, 60)}"): the dependencies were not installed, so no test code ran`
+                        : `the test could not be loaded because ${found.language} could not find "${truncate(found.module, 60)}": a dependency is missing, so nothing about the test itself has been shown`,
+                    data: { ...found, wholeRun },
+                },
+            ],
+        };
+    },
+};

package/dist/engine/rules.js

@@ -2,6 +2,7 @@ import { deviceEnvSignal } from './device-errors.js';
 import { INFRA_MEANING, infraSignal } from './infra-errors.js';
 import { isElementNotFound, networkSignal, valueMismatch } from './error-parse.js';
 import { locatorDrift, locatorDriftStrict } from './drift-rules.js';
+import { envMissingDependency } from './missing-dependency.js';
 import { timingFlaky, timingIntercepted, timingIntermittent, timingKnownFlaky, timingRetryPass, timingStaleElement } from './timing-rules.js';
 import { errorScenarioOf } from './llm/context.js';
 import { errorShape, truncate } from './signature.js';
@@ -191,4 +192,4 @@ const realBugAssertion = {
  * Priority order: the first matching rule decides. Signals from outside the test (ENV)
  * come first, so a 5xx that also fails an assertion is ENV, not REAL_BUG.
  */
-export const RULES = [envMassFailure, envNetwork, envInfra, envDevice, testDataCollision, envServerError, locatorDriftStrict, timingIntercepted, timingStaleElement, timingRetryPass, locatorDrift, timingFlaky, timingIntermittent, timingKnownFlaky, realBugAssertion];
+export const RULES = [envMissingDependency, envMassFailure, envNetwork, envInfra, envDevice, testDataCollision, envServerError, locatorDriftStrict, timingIntercepted, timingStaleElement, timingRetryPass, locatorDrift, timingFlaky, timingIntermittent, timingKnownFlaky, realBugAssertion];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.17.2",
+  "version": "0.18.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",