testaro 59.2.5 → 59.2.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "59.2.5",
+  "version": "59.2.8",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/standardize.js

@@ -611,24 +611,16 @@ const convert = (toolName, data, result, standardResult) => {
     rules.forEach(rule => {
       // Copy its instances to the standard result.
       const ruleResult = result[rule];
-      if (ruleResult.standardInstances) {
-        standardResult.instances.push(... ruleResult.standardInstances);
-      }
-      else {
-        console.log(`ERROR: Testaro rule ${rule} result has no standardInstances property`);
-      }
+      ruleResult.standardInstances ??= [];
+      standardResult.instances.push(... ruleResult.standardInstances);
       // Initialize a record of its sample-ratio-weighted totals.
       data.ruleTotals[rule] = [0, 0, 0, 0];
       // Add those totals to the record and to the standard result.
-      if (ruleResult.totals) {
-        for (const index in ruleResult.totals) {
-          const ruleTotal = ruleResult.totals[index];
-          data.ruleTotals[rule][index] += ruleTotal;
-          standardResult.totals[index] += ruleTotal;
-        }
-      }
-      else {
-        console.log(`ERROR: Testaro rule ${rule} result has no totals property`);
+      ruleResult.totals ??= [0, 0, 0, 0];
+      for (const index in ruleResult.totals) {
+        const ruleTotal = ruleResult.totals[index];
+        data.ruleTotals[rule][index] += ruleTotal;
+        standardResult.totals[index] += ruleTotal;
       }
     });
     const preventionCount = result.preventions && result.preventions.length;

package/procs/visChange.js

@@ -34,6 +34,7 @@
 
 // IMPORTS
 
+const pixelmatch = require('pixelmatch').default;
 const {PNG} = require('pngjs');
 
 // FUNCTIONS
@@ -103,8 +104,6 @@ exports.visChange = async (page, options = {}) => {
       // Get their dimensions.
       const {width, height} = pngs[0];
       // Get the count of differing pixels between the shots.
-      const pixelmatchModule = await import('pixelmatch');
-      const pixelmatch = pixelmatchModule.default;
       const pixelChanges = pixelmatch(pngs[0].data, pngs[1].data, null, width, height);
       // Get the ratio of differing to all pixels as a percentage.
       const changePercent = 100 * pixelChanges / (width * height);

package/run.js

@@ -205,7 +205,9 @@ const goTo = async (report, page, url, timeout, waitUntil) => {
     }
   }
   catch(error) {
-    console.log(`ERROR visiting ${url} (${error.message.slice(0, 200)})`);
+    if (debug) {
+      console.log(`ERROR visiting ${url} (${error.message.slice(0, 200)})`);
+    }
     return {
       success: false,
       error: 'noVisit'
@@ -258,7 +260,9 @@ const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
   }
 };
 // Launches a browser and navigates to a URL.
-const launch = exports.launch = async (report, debug, waits, tempBrowserID, tempURL) => {
+const launch = exports.launch = async (
+  report, debug, waits, tempBrowserID, tempURL, retries = 2
+) => {
   const act = report.acts[actIndex];
   const {device} = report;
   const deviceID = device && device.id;
@@ -276,22 +280,21 @@ const launch = exports.launch = async (report, debug, waits, tempBrowserID, temp
     const browserOptions = {
       logger: {
         isEnabled: () => false,
-        log: (name, severity, message) => console.log(message.slice(0, 100))
-      }
+        log: (name, severity, message) => {
+          if (['warning', 'error'].includes(severity)) {
+            console.log(`${severity.toUpperCase()}: ${message.slice(0, 200)}`);
+          }
+        }
+      },
+      headless: ! debug,
+      slowMo: waits || 0,
+      args: ['--disable-dev-shm-usage']
     };
-    browserOptions.headless = ! debug;
-    browserOptions.slowMo = waits || 0;
     try {
       // Replace the browser with a new one.
       browser = await browserType.launch(browserOptions);
-      // Open a context (i.e. browser window).
-      const browserContext = await browser.newContext(device.windowOptions);
-      // Create a diagnostic listener for its unintentional closing.
-      browserContext.on('close', () => {
-        if (! browserCloseIntentional) {
-          console.log('ERROR: Browser context unexpectedly closed');
-        }
-      });
+      // Redefine the context (i.e. browser window).
+      browserContext = await browser.newContext(device.windowOptions);
       // Prevent default timeouts.
       browserContext.setDefaultTimeout(0);
       // When a page (i.e. browser tab) is added to the browser context (i.e. browser window):
@@ -380,18 +383,38 @@ const launch = exports.launch = async (report, debug, waits, tempBrowserID, temp
         // Report this.
         addError(true, false, report, actIndex, 'status429');
       }
-      // Otherwise, i.e. if the launch or navigation failed:
+      // Otherwise, i.e. if the launch or navigation failed for another reason:
       else {
-        // Report this.
-        addError(true, false, report, actIndex, `ERROR: Launch failed (${navResult.error})`);
-        page = null;
+        // Cause another attempt to launch and navigate, if retries remain.
+        throw new Error(`Navigation failed (${navResult.error})`);
       }
     }
     // If an error occurred:
     catch(error) {
-      // Report this.
-      addError(true, false, report, actIndex, `ERROR launching or navigating ${error.message}`);
-      page = null;
+      // If retries remain:
+      if (retries > 0) {
+        console.log(`WARNING: Retrying launch (${retries} retries left)`);
+        await wait(2000);
+        return launch(report, debug, waits, tempBrowserID, tempURL, retries - 1);
+      }
+      // Otherwise, i.e. if no retries remain:
+      else {
+        // Report this.
+        addError(
+          true, false, report, actIndex, `FINAL ERROR launching or navigating (${error.message})`
+        );
+        // If the browser was created, and thus not a context of it:
+        if (browser) {
+          // Report this.
+          console.log('ERROR: Browser was created but context creation failed');
+          // Close the browser.
+          await browser.close().catch(() => {
+            console.log('ERROR: Could not close browser after context creation failure');
+          });
+          browser = null;
+        }
+        page = null;
+      }
     };
   }
   // Otherwise, i.e. if the browser or device ID is invalid:

package/testaro/hovInd.js

@@ -140,113 +140,104 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
   const sample = locsAll.filter((loc, index) => sampleIndexes.includes(index));
   // For each trigger in the sample:
   for (const loc of sample) {
-    // Get its style properties.
-    const preStyles = await getHoverStyles(loc);
-    // Try to focus it.
     try {
+      // Get its style properties.
+      const preStyles = await getHoverStyles(loc);
+      // Focus it.
       await loc.focus({timeout: 500});
       // If focusing succeeds, get its style properties.
       const focStyles = await getHoverStyles(loc);
-      // Try to blur it.
-      try {
-        await loc.blur({timeout: 500});
-        // If blurring succeeds, try to hover over it.
-        try {
-          await loc.hover({timeout: 500});
-          // If hovering succeeds, get its style properties.
-          const hovStyles = await getHoverStyles(loc);
-          // If all 3 style declarations belong to the same element:
-          if ([focStyles, hovStyles].every(style => style.code === preStyles.code)) {
-            // Get data on the element if itemization is required.
-            const elData = withItems ? await getLocatorData(loc) : null;
-            // If the hover cursor is nonstandard:
-            const cursorData = getCursorData(hovStyles);
-            if (! cursorData.ok) {
-              // Add to the totals.
-              totals[2] += psRatio;
-              data.typeTotals.badCursor += psRatio;
-              // If itemization is required:
-              if (withItems) {
-                // Add an instance to the result.
-                standardInstances.push({
-                  ruleID: 'hovInd',
-                  what: `Element has a nonstandard hover cursor (${cursorData.cursor})`,
-                  ordinalSeverity: 2,
-                  tagName: elData.tagName,
-                  id: elData.id,
-                  location: elData.location,
-                  excerpt: elData.excerpt
-                });
-              }
-            }
-            // If the element is a button and the hover and default states are not distinct:
-            if (hovStyles.tagName === 'BUTTON' && areAlike(preStyles, hovStyles)) {
-              // Add to the totals.
-              totals[1] += psRatio;
-              data.typeTotals.hoverLikeDefault += psRatio;
-              // If itemization is required:
-              if (withItems) {
-                // Add an instance to the result.
-                standardInstances.push({
-                  ruleID: 'hovInd',
-                  what: 'Element border, outline, color, and background color do not change when hovered over',
-                  ordinalSeverity: 1,
-                  tagName: elData.tagName,
-                  id: elData.id,
-                  location: elData.location,
-                  excerpt: elData.excerpt
-                });
-              }
-            }
-            // If the hover and focus states are indistinct but differ from the default state:
-            if (areAlike(hovStyles, focStyles) && ! areAlike(hovStyles, preStyles)) {
-              // Add to the totals.
-              totals[1] += psRatio;
-              data.typeTotals.hoverLikeFocus += psRatio;
-              // If itemization is required:
-              if (withItems) {
-                // Add an instance to the result.
-                standardInstances.push({
-                  ruleID: 'hovInd',
-                  what: 'Element border, outline, color, and background color are alike on hover and focus',
-                  ordinalSeverity: 1,
-                  tagName: elData.tagName,
-                  id: elData.id,
-                  location: elData.location,
-                  excerpt: elData.excerpt
-                });
-              }
-            }
+      // Blur it.
+      await loc.blur({timeout: 500});
+      // If blurring succeeds, try to hover over it.
+      await loc.hover({timeout: 500});
+      // If hovering succeeds, get its style properties.
+      const hovStyles = await getHoverStyles(loc);
+      // If all 3 style declarations belong to the same element:
+      if ([focStyles, hovStyles].every(style => style.code === preStyles.code)) {
+        // Get data on the element if itemization is required.
+        const elData = withItems ? await getLocatorData(loc) : null;
+        // If the hover cursor is nonstandard:
+        const cursorData = getCursorData(hovStyles);
+        if (! cursorData.ok) {
+          // Add to the totals.
+          totals[2] += psRatio;
+          data.typeTotals.badCursor += psRatio;
+          // If itemization is required:
+          if (withItems) {
+            // Add an instance to the result.
+            standardInstances.push({
+              ruleID: 'hovInd',
+              what: `Element has a nonstandard hover cursor (${cursorData.cursor})`,
+              ordinalSeverity: 2,
+              tagName: elData.tagName,
+              id: elData.id,
+              location: elData.location,
+              excerpt: elData.excerpt
+            });
           }
-          // Otherwise, i.e. if the style properties do not all belong to the same element:
-          else {
-            // Report this and quit.
-            data.prevented = true;
-            data.error = 'ERROR: Page changes on focus or hover prevent test';
-            break;
+        }
+        // If the element is a button and the hover and default states are not distinct:
+        if (hovStyles.tagName === 'BUTTON' && areAlike(preStyles, hovStyles)) {
+          // Add to the totals.
+          totals[1] += psRatio;
+          data.typeTotals.hoverLikeDefault += psRatio;
+          // If itemization is required:
+          if (withItems) {
+            // Add an instance to the result.
+            standardInstances.push({
+              ruleID: 'hovInd',
+              what: 'Element border, outline, color, and background color do not change when hovered over',
+              ordinalSeverity: 1,
+              tagName: elData.tagName,
+              id: elData.id,
+              location: elData.location,
+              excerpt: elData.excerpt
+            });
           }
         }
-        // If hovering fails:
-        catch(error) {
-          // Report this.
-          data.prevented = true;
-          data.error = 'ERROR: Hovering failed';
-          break;
+        // If the hover and focus states are indistinct but differ from the default state:
+        if (areAlike(hovStyles, focStyles) && ! areAlike(hovStyles, preStyles)) {
+          // Add to the totals.
+          totals[1] += psRatio;
+          data.typeTotals.hoverLikeFocus += psRatio;
+          // If itemization is required:
+          if (withItems) {
+            // Add an instance to the result.
+            standardInstances.push({
+              ruleID: 'hovInd',
+              what: 'Element border, outline, color, and background color are alike on hover and focus',
+              ordinalSeverity: 1,
+              tagName: elData.tagName,
+              id: elData.id,
+              location: elData.location,
+              excerpt: elData.excerpt
+            });
+          }
         }
       }
-      // If blurring fails:
-      catch(error) {
-        // Report this.
+      // Otherwise, i.e. if the style properties do not all belong to the same element:
+      else {
+        // Report this and quit.
         data.prevented = true;
-        data.error = 'ERROR: Blurring failed';
+        data.error = 'ERROR: Page changes on focus or hover prevent test';
         break;
       }
     }
-    // If focusing fails:
     catch(error) {
-      // Report this.
+      // If the page closed:
+      if (
+        ['Target page', 'detached', 'null', 'closed'].some(string => error.message.includes(string))
+      ) {
+        data.error = `ERROR during hovInd test: ${error.message}`;
+      }
+      else {
+        const elementText = loc ? await loc.textContent({timeout: 200}) : '';
+        const excerpt = elementText ? elementText.trim().slice(0, 100) : '<no text>';
+        data.error = `ERROR manipulating element (${excerpt}) during hovInd test`;
+      }
       data.prevented = true;
-      data.error = 'ERROR: Focusing failed';
+      // Abort this test.
       break;
     }
   }

package/testaro/imageLink.js

@@ -34,12 +34,10 @@ exports.reporter = async (page, withItems) => {
   const ruleData = {
     ruleID: 'imageLink',
     selector: 'a[href]',
-    pruner: async (loc) => {
-      return loc.evaluate(el => {
-        const href = el.getAttribute('href') || '';
-        return /\.(?:png|jpe?g|gif|svg|webp|ico)(?:$|[?#])/i.test(href);
-      });
-    },
+    pruner: async loc => await loc.evaluate(el => {
+      const href = el.getAttribute('href') || '';
+      return /\.(?:png|jpe?g|gif|svg|webp|ico)(?:$|[?#])/i.test(href);
+    }),
     complaints: {
       instance: 'Link destination is an image file',
       summary: 'Links have image files as their destinations'

package/testaro/legendLoc.js

@@ -34,20 +34,18 @@ exports.reporter = async (page, withItems) => {
   const ruleData = {
     ruleID: 'legendLoc',
     selector: 'legend',
-    pruner: async (loc) => {
-      return loc.evaluate(el => {
-        const parent = el.parentElement;
-        if (!parent) return true;
-        if (parent.tagName.toUpperCase() !== 'FIELDSET') return true;
-        // Check if this legend is the first element child of the fieldset
-        for (const child of parent.children) {
-          if (child.nodeType === 1) {
-            return child !== el; // true if not first child
-          }
+    pruner: async (loc) => await loc.evaluate(el => {
+      const parent = el.parentElement;
+      if (!parent) return true;
+      if (parent.tagName.toUpperCase() !== 'FIELDSET') return true;
+      // Check if this legend is the first element child of the fieldset
+      for (const child of parent.children) {
+        if (child.nodeType === 1) {
+          return child !== el; // true if not first child
         }
-        return true;
-      });
-    },
+      }
+      return true;
+    }),
     complaints: {
       instance: 'Element is not the first child of a fieldset element',
       summary: 'legend elements are not the first children of fieldset elements'

package/testaro/lineHeight.js

@@ -1,5 +1,6 @@
 /*
   © 2023–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -31,10 +32,12 @@
   their subtrees are excluded.
 */
 
+// IMPORTS
+
 // Module to perform common operations.
 const {init, report} = require('../procs/testaro');
 
-// ########## FUNCTIONS
+// FUNCTIONS
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {

package/testaro/optRoleSel.js

@@ -34,11 +34,9 @@ exports.reporter = async (page, withItems) => {
   const ruleData = {
     ruleID: 'optRoleSel',
     selector: '[role="option"]',
-    pruner: async (loc) => {
-      return loc.evaluate(el => {
-        return ! el.hasAttribute('aria-selected');
-      });
-    },
+    pruner: async (loc) => await loc.evaluate(el => {
+      return ! el.hasAttribute('aria-selected');
+    }),
     complaints: {
       instance: 'Element has an explicit option role but no aria-selected attribute',
       summary: 'Elements with explicit option roles have no aria-selected attributes'

package/tests/alfa.js

@@ -27,14 +27,18 @@
   This test implements the alfa ruleset for accessibility.
 */
 
+// IMPORTS
+
+const {Audit} = require('@siteimprove/alfa-act');
+const {Playwright} = require('@siteimprove/alfa-playwright');
+let alfaRules = require('@siteimprove/alfa-rules').default;
+
 // FUNCTIONS
 
 // Conducts and reports the alfa tests.
-exports.reporter = async (page, report, actIndex, timeLimit) => {
+exports.reporter = async (page, report, actIndex) => {
   const act = report.acts[actIndex];
   const {rules} = act;
-  const alfaRulesModule = await import('@siteimprove/alfa-rules');
-  const alfaRules = alfaRulesModule.default;
   // If only some rules are to be employed:
   if (rules && rules.length) {
     // Remove the other rules.
@@ -84,11 +88,7 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
     }
     // Test the page content with the specified rules.
     const doc = await page.evaluateHandle('document');
-    const alfaPlaywrightModule = await import('@siteimprove/alfa-playwright');
-    const {Playwright} = alfaPlaywrightModule;
     const alfaPage = await Playwright.toPage(doc);
-    const alfaActModule = await import('@siteimprove/alfa-act');
-    const {Audit} = alfaActModule;
     const audit = Audit.of(alfaPage, alfaRules);
     const outcomes = Array.from(await audit.evaluate());
     // For each failure or warning:
@@ -161,7 +161,7 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
     });
   }
   catch(error) {
-    console.log(`ERROR: navigation to URL timed out (${error})`);
+    console.log(`ERROR: Navigation to URL timed out (${error})`);
     data.prevented = true;
     data.error = 'ERROR: Act failed';
   }

package/tests/testaro.js

@@ -28,7 +28,7 @@
   This test implements the Testaro evaluative rules.
 */
 
-// ######## IMPORTS
+// IMPORTS
 
 // Module to perform common operations.
 const {init, report} = require('../procs/testaro');
@@ -37,7 +37,7 @@ const {launch} = require('../run');
 // Module to handle files.
 const fs = require('fs/promises');
 
-// ######## CONSTANTS
+// CONSTANTS
 
 // The validation job data for the tests listed below are in the pending directory.
 /*
@@ -146,12 +146,18 @@ const slowTestLimits = {
   hovInd: 10,
   labClash: 10,
   lineHeight: 10,
+  linkUl: 10,
   motion: 15,
   opFoc: 10,
   tabNav: 10,
   textSem: 10
 };
 
+// ERROR HANDLER
+process.on('unhandledRejection', reason => {
+  console.error(`ERROR: Unhandled Promise Rejection (${reason})`);
+});
+
 // ######## FUNCTIONS
 
 // Conducts a JSON-defined test.
@@ -182,6 +188,7 @@ const wait = ms => {
 };
 // Conducts and reports Testaro tests.
 exports.reporter = async (page, report, actIndex) => {
+  // Report page crashes.
   const url = await page.url();
   const act = report.acts[actIndex];
   const {args, stopOnFail, withItems} = act;
@@ -216,17 +223,15 @@ exports.reporter = async (page, report, actIndex) => {
     let calledRules = rules[0] === 'y'
       ? rules.slice(1)
       : Object.keys(evalRules).filter(ruleID => ! rules.slice(1).includes(ruleID));
-    const calledContaminators = calledRules.filter(rule => contaminators.includes(rule)).sort();
+    const calledContaminators = calledRules.filter(rule => contaminators.includes(rule));
     const firstCalledContaminator = calledContaminators[0];
-    const calledBenignRules = calledRules.filter(rule => ! contaminators.includes(rule)).sort();
+    const calledBenignRules = calledRules.filter(rule => ! contaminators.includes(rule));
     const testTimes = [];
     let contaminatorsStarted = false;
     // Starting with the noncontaminators, for each rule invoked:
     for (const rule of calledBenignRules.concat(calledContaminators)) {
-      console.log(`Starting rule ${rule}`);
-      if (rule === firstCalledContaminator) {
-        console.log('  It is the first contaminator');
-      }
+      const contaminatorSuffix = rule === firstCalledContaminator ? ' (first contaminator)' : '';
+      console.log(`Starting rule ${rule}${contaminatorSuffix}`);
       const pageClosed = page ? page.isClosed() : true;
       const isContaminator = contaminators.includes(rule);
       // If it is a contaminator other than the first one or the page has closed:
@@ -250,6 +255,22 @@ exports.reporter = async (page, report, actIndex) => {
       if (isContaminator) {
         contaminatorsStarted = true;
       }
+      // Report crashes and disconnections during this test.
+      let crashHandler;
+      let disconnectHandler;
+      const {browser} = require('../run');
+      if (page && ! page.isClosed()) {
+        crashHandler = () => {
+          console.log(`ERROR: Page crashed during ${rule} test`);
+        };
+        page.on('crash', crashHandler);
+      }
+      if (browser) {
+        disconnectHandler = () => {
+          console.log(`ERROR: Browser disconnected during ${rule} test`);
+        };
+        browser.on('disconnected', disconnectHandler);
+      }
       // Initialize an argument array.
       const ruleArgs = [page, withItems];
       const ruleFileNames = await fs.readdir(`${__dirname}/../testaro`);
@@ -269,52 +290,103 @@ exports.reporter = async (page, report, actIndex) => {
         const what = evalRules[rule] || etcRules[rule];
         result[rule].what = what;
         const startTime = Date.now();
-        try {
-          // Apply a time limit to the test.
-          const timeLimit = 1000 * (slowTestLimits[rule] ?? 5);
-          let timeout;
-          // If the time limit expires during the test:
-          const timer = new Promise(resolve => {
-            timeout = setTimeout(() => {
-              // Add data about the test, including its prevention, to the result.
+        let timeout;
+        let testRetries = 2;
+        let testSuccess = false;
+        while (testRetries > 0 && ! testSuccess) {
+          try {
+            // Apply a time limit to the test.
+            const timeLimit = 1000 * (slowTestLimits[rule] ?? 5);
+            // If the time limit expires during the test:
+            const timer = new Promise(resolve => {
+              timeout = setTimeout(() => {
+                // Add data about the test, including its prevention, to the result.
+                const endTime = Date.now();
+                testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
+                data.rulePreventions.push(rule);
+                result[rule].totals = [0, 0, 0, 0];
+                result[rule].standardInstances = [];
+                console.log(`ERROR: Test of testaro rule ${rule} timed out`);
+                resolve({timedOut: true});
+              }, timeLimit);
+            });
+            // Perform the test, subject to the time limit.
+            const ruleReport = isJS
+              ? require(`../testaro/${rule}`).reporter(... ruleArgs)
+              : jsonTest(rule, ruleArgs);
+            // Get the test result or a timeout result.
+            const ruleOrTimeoutReport = await Promise.race([timer, ruleReport]);
+            // If the test was completed:
+            if (! ruleOrTimeoutReport.timedOut) {
+              // Add data from the test to the result.
               const endTime = Date.now();
               testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
+              Object.keys(ruleOrTimeoutReport).forEach(key => {
+                result[rule][key] = ruleOrTimeoutReport[key];
+              });
+              result[rule].totals = result[rule].totals.map(total => Math.round(total));
+              // Prevent a retry of the test.
+              testSuccess = true;
+              // If testing is to stop after a failure and the page failed the test:
+              if (stopOnFail && ruleOrTimeoutReport.totals.some(total => total)) {
+                // Stop testing.
+                break;
+              }
+            }
+          }
+          // If an error is thrown by the test:
+          catch(error) {
+            const isPageClosed = ['closed', 'Protocol error', 'Target page'].some(phrase =>
+              error.message.includes(phrase)
+            );
+            // If the page has closed and there are retries left:
+            if (isPageClosed && testRetries) {
+              // Report this and decrement the allowed retry count.
+              console.log(
+                `WARNING: Retry ${3 - testRetries--} of test ${rule} starting after page closed`
+              );
+              await wait(2000);
+              // Replace the browser and the page and navigate to the target.
+              await launch(
+                report,
+                process.env.DEBUG === 'true',
+                Number.parseInt(process.env.WAITS) || 0,
+                report.browserID,
+                url
+              );
+              page = require('../run').page;
+              // If the page replacement failed:
+              if (! page) {
+                // Report this.
+                console.log(`ERROR: Browser relaunch to retry test ${rule} failed`);
+                data.rulePreventions.push(rule);
+                data.rulePreventionMessages[rule] = 'Retry failure due to browser relaunch failure';
+                // Stop retrying the test.
+                break;
+              }
+              // Update the rule arguments with the current page.
+              ruleArgs[0] = page;
+            }
+            // Otherwise, i.e. if the page is open or it is closed but no retries are left:
+            else {
+              // Treat the test as prevented.
               data.rulePreventions.push(rule);
-              result[rule].totals = [0, 0, 0, 0];
-              result[rule].standardInstances = [];
-              console.log(`ERROR: Test of testaro rule ${rule} timed out`);
-              resolve({timedOut: true});
-            }, timeLimit);
-          });
-          // Perform the test, subject to the time limit.
-          const ruleReport = isJS
-            ? require(`../testaro/${rule}`).reporter(... ruleArgs)
-            : jsonTest(rule, ruleArgs);
-          // Get the test result or a timeout result.
-          const ruleOrTimeoutReport = await Promise.race([timer, ruleReport]);
-          clearTimeout(timeout);
-          // If the test was completed:
-          if (! ruleOrTimeoutReport.timedOut) {
-            // Add data from the test to the result.
-            const endTime = Date.now();
-            testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
-            Object.keys(ruleOrTimeoutReport).forEach(key => {
-              result[rule][key] = ruleOrTimeoutReport[key];
-            });
-            result[rule].totals = result[rule].totals.map(total => Math.round(total));
-            // If testing is to stop after a failure and the page failed the test:
-            if (stopOnFail && ruleOrTimeoutReport.totals.some(total => total)) {
-              // Stop testing.
+              data.rulePreventionMessages[rule] = error.message;
+              console.log(`ERROR: Test of testaro rule ${rule} prevented (${error.message})`);
+              // Do not retry the test even if retries are left.
               break;
             }
           }
-        }
-        // If an error is thrown by the test:
-        catch(error) {
-          // Report the error.
-          data.rulePreventions.push(rule);
-          data.rulePreventionMessages[rule] = error.message;
-          console.log(`ERROR: Test of testaro rule ${rule} prevented (${error.message})`);
+          finally {
+            // Clear the timeout and the error listeners.
+            clearTimeout(timeout);
+            if (page && ! page.isClosed() && crashHandler) {
+              page.off('crash', crashHandler);
+            }
+            if (browser && disconnectHandler) {
+              browser.off('disconnected', disconnectHandler);
+            }
+          }
         }
       }
       // Otherwise, i.e. if the rule is undefined or doubly defined:
@@ -322,6 +394,10 @@ exports.reporter = async (page, report, actIndex) => {
         // Report this.
         data.rulesInvalid.push(rule);
         console.log(`ERROR: Rule ${rule} not validly defined`);
+        // Clear the crash listener.
+        if (page && ! page.isClosed()) {
+          page.off('crash', crashHandler);
+        }
       }
     }
     // Record the test times in descending order.