testaro 67.0.0 → 68.0.0

package/tests/testaro.js

@@ -1,6 +1,6 @@
 /*
   © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
-  © 2025 Jonathan Robert Pool.
+  © 2025–2026 Jonathan Robert Pool.
 
   Licensed under the MIT License. See LICENSE file at the project root or
   https://opensource.org/license/mit/ for details.
@@ -16,387 +16,436 @@
 // IMPORTS
 
 // Function to launch a browser.
-const {launch} = require('../run');
+const {launch} = require('../procs/launch');
 
 // CONSTANTS
 
-// Metadata of all rules in default execution order.
+/*
+  Metadata of all rules in default execution order.
+  Property needsLaunch is true if the rule is first or the previous one contaminates the page.
+*/
 const allRules = [
   {
     id: 'shoot0',
     what: 'first page screenshot',
-    launchRole: 'owner',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'adbID',
     what: 'elements with ambiguous or missing referenced descriptions',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'allCaps',
     what: 'leaf elements with entirely upper-case text longer than 7 characters',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'allHidden',
     what: 'page that is entirely or mostly hidden',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'allSlanted',
     what: 'leaf elements with entirely italic or oblique text longer than 39 characters',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'altScheme',
     what: 'img elements with alt attributes having URLs as their entire values',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'attVal',
     what: 'elements with attributes having illicit values',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: false
   },
-  {
-    id: 'dupAtt',
-    what: 'duplicate attribute values',
-    launchRole: 'sharer',
-    timeOut: 5,
-    defaultOn: true
-  },
-  {
-    id: 'autocomplete',
-    what: 'name and email inputs without autocomplete attributes',
-    launchRole: 'sharer',
-    timeOut: 5,
-    defaultOn: true
-  },
   {
     id: 'bulk',
     what: 'large count of visible elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'captionLoc',
     what: 'caption elements that are not first children of table elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'datalistRef',
     what: 'elements with ambiguous or missing referenced datalist elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'distortion',
     what: 'distorted text',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'docType',
     what: 'document without a doctype property',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'dupAtt',
-    what: 'elements with duplicate attributes',
-    launchRole: 'sharer',
+    what: 'duplicate attribute values',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'embAc',
     what: 'active elements embedded in links or buttons',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'headEl',
     what: 'invalid elements within the head',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'headingAmb',
     what: 'same-level sibling headings with identical texts',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'hr',
     what: 'hr element instead of styles used for vertical segmentation',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'imageLink',
     what: 'links with image files as their destinations',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'labClash',
     what: 'labeling inconsistencies',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'legendLoc',
     what: 'legend elements that are not first children of fieldset elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'lineHeight',
     what: 'text with a line height less than 1.5 times its font size',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'linkAmb',
     what: 'links with identical texts but different destinations',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 20,
     defaultOn: true
   },
   {
     id: 'linkExt',
     what: 'links that automatically open new windows',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'linkOldAtt',
     what: 'links with deprecated attributes',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'linkTo',
     what: 'links without destinations',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'linkUl',
     what: 'missing underlines on inline links',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'miniText',
     what: 'text smaller than 11 pixels',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'nonTable',
     what: 'table elements used for layout',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'optRoleSel',
     what: 'Non-option elements with option roles that have no aria-selected attributes',
-    launchRole: 'sharer',
-    timeOut: 5,
-    defaultOn: true
-  },
-  {
-    id: 'phOnly',
-    what: 'input elements with placeholders but no accessible names',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'pseudoP',
     what: 'adjacent br elements suspected of nonsemantically simulating p elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'radioSet',
     what: 'radio buttons not grouped into standard field sets',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'role',
     what: 'native-replacing explicit roles',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 20,
     defaultOn: true
   },
   {
     id: 'secHeading',
     what: 'headings that violate the logical level order in their sectioning containers',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'styleDiff',
     what: 'style inconsistencies',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'targetSmall',
     what: 'labels, buttons, inputs, and links too near each other',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'textSem',
     what: 'semantically vague elements i, b, and/or small',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'title',
     what: 'page title',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: false
   },
   {
     id: 'titledEl',
     what: 'title attributes on inappropriate elements',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'zIndex',
     what: 'non-default Z indexes',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'shoot1',
     what: 'second page screenshot',
-    launchRole: 'owner',
+    contaminates: false,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'motion',
     what: 'motion without user request, measured across tests',
-    launchRole: 'sharer',
+    contaminates: false,
+    needsAccessibleName: false,
+    timeOut: 5,
+    defaultOn: true
+  },
+  {
+    id: 'autocomplete',
+    what: 'name and email inputs without autocomplete attributes',
+    contaminates: false,
+    needsAccessibleName: true,
+    timeOut: 5,
+    defaultOn: true
+  },
+  {
+    id: 'phOnly',
+    what: 'input elements with placeholders but no accessible names',
+    contaminates: false,
+    needsAccessibleName: true,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'buttonMenu',
     what: 'nonstandard keyboard navigation between items of button-controlled menus',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 15,
     defaultOn: true
   },
   {
     id: 'elements',
     what: 'data on specified elements',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: false
   },
   {
     id: 'focAll',
     what: 'discrepancies between focusable and Tab-focused elements',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: true
   },
   {
     id: 'focAndOp',
     what: 'Tab-focusable elements that are not operable or vice versa',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 5,
     defaultOn: true
   },
   {
     id: 'focInd',
     what: 'missing and nonstandard focus indicators',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: true
   },
   {
     id: 'focVis',
     what: 'links that are not entirely visible when focused',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: true
   },
   {
     id: 'hover',
     what: 'hover-caused content changes',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 20,
     defaultOn: true
   },
   {
     id: 'hovInd',
     what: 'hover indication nonstandard',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: true
   },
   {
     id: 'tabNav',
     what: 'nonstandard keyboard navigation between elements with the tab role',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: true
   },
   {
     id: 'textNodes',
     what: 'data on specified text nodes',
-    launchRole: 'waster',
+    contaminates: true,
+    needsAccessibleName: false,
     timeOut: 10,
     defaultOn: false
   }
@@ -426,244 +475,210 @@ exports.reporter = async (page, report, actIndex) => {
   const url = target.url;
   const browserID = act.launch ? act.launch.browserID || report.browserID : report.browserID;
   const argRules = args ? Object.keys(args) : null;
-  // Get the specification of rules to be tested for.
+  // Get the specification of rules to be tested for or, by default, all rules with defaultOn true.
   const ruleSpec = act.rules
   || ['y', ... allRules.filter(rule => rule.defaultOn).map(rule => rule.id)];
-  // Initialize the act data and result.
+  // Initialize the act data.
   const data = {
     prevented: false,
     error: '',
-    rulePreventions: [],
-    rulePreventionMessages: {},
+    rulePreventions: {},
     rulesInvalid: [],
-    ruleTestTimes: {},
+    ruleTestTimes: [],
     ruleData: {}
   };
-  const result = {};
+  // Initialize the act result.
+  const result = {
+    nativeResult: {},
+    standardResult: {
+      prevented: false,
+      totals: [0, 0, 0, 0],
+      instances: []
+    }
+  };
+  const {standardResult} = result;
   const allRuleIDs = allRules.map(rule => rule.id);
-  // If the rule specification is invalid:
-  if (! (
+  // If the rule specification is valid:
+  if (
     ruleSpec.length > 1
     && ['y', 'n'].includes(ruleSpec[0])
     && ruleSpec.slice(1).every(ruleID => allRuleIDs.includes(ruleID))
-  )) {
-    // Report this and stop testing.
-    data.prevented = true;
-    data.error = 'ERROR: Testaro rule specification invalid';
-    console.log('ERROR: Testaro rule specification invalid');
-    return {
-      data,
-      result
-    };
-  }
-  // Wait 1 second to prevent out-of-order logging with granular reporting.
-  await wait(1000);
-  // Get the rules to be tested for and their execution order.
-  const jobRuleIDs = ruleSpec[0] === 'y'
-  ? ruleSpec.slice(1)
-  : allRules.filter(rule => rule.defaultOn && ! allRuleIDs.includes(rule.id));
-  const jobRules = allRules.filter(rule => jobRuleIDs.includes(rule.id));
-  const testTimes = [];
-  // For each rule to be tested for:
-  for (const ruleIndexString in jobRules) {
-    const ruleIndex = Number.parseInt(ruleIndexString);
-    const rule = jobRules[ruleIndex];
-    const ruleID = rule.id;
-    console.log(`Starting rule ${ruleID}`);
-    // Make the browser emulate headedness in all cases, because performance does not suffer.
-    const headEmulation = ruleID.startsWith('shoot') ? 'high' : 'high';
-    // Get whether it needs a new browser launched.
-    const needsLaunch = ruleIndex
-    && jobRules[ruleIndex - 1].launchRole !== 'sharer'
-    && rule.launchRole !== 'owner'
-    || ! ruleIndex;
-    const pageClosed = page && page.isClosed();
-    // If it does, or if the page has closed:
-    if (needsLaunch || pageClosed) {
-      // If the page has closed when it is expected to be open:
-      if (pageClosed && ! needsLaunch) {
-        // Report this.
-        console.log(`WARNING: Relaunching browser for test ${rule} after abnormal closure`);
-      }
-      // Replace the browser and the page and navigate to the target.
-      await launch(
-        report,
-        actIndex,
-        headEmulation,
-        browserID,
-        url
-      );
-      page = require('../run').page;
-    }
-    // Get the current browser.
-    const {browser} = require('../run');
-    // Report crashes and disconnections during this test.
-    let crashHandler;
-    let disconnectHandler;
-    if (page && ! page.isClosed()) {
-      crashHandler = () => {
-        console.log(`ERROR: Page crashed during ${rule} test`);
+  ) {
+    // Wait 1 second to prevent out-of-order logging with granular reporting.
+    await wait(1000);
+    // Get the rules to be tested for and their execution order.
+    const jobRuleIDs = ruleSpec[0] === 'y'
+    ? ruleSpec.slice(1)
+    : allRules.filter(rule => rule.defaultOn && ! allRuleIDs.includes(rule.id));
+    const jobRules = allRules.filter(rule => jobRuleIDs.includes(rule.id));
+    // For each rule to be tested for:
+    for (let ruleIndex = 0; ruleIndex < jobRules.length; ruleIndex++) {
+      const rule = jobRules[ruleIndex];
+      // Initialize the rule result.
+      const ruleResult = {
+        id: rule.id,
+        prevented: false,
+        error: '',
+        data: {},
+        totals: [0, 0, 0, 0],
+        instances: [],
+        elapsedTime: 0
       };
-      page.on('crash', crashHandler);
-    }
-    if (browser) {
-      disconnectHandler = () => {
-        console.log(`ERROR: Browser disconnected during ${rule} test`);
-      };
-      browser.on('disconnected', disconnectHandler);
-    }
-    // Initialize an argument array for the reporter.
-    const ruleArgs = [page, withItems];
-    // If the rule has extra arguments:
-    if (argRules && argRules.includes(ruleID)) {
-      // Add them to the argument array.
-      ruleArgs.push(... args[ruleID]);
-    }
-    // Initialize the rule result.
-    result[ruleID] ??= {};
-    const ruleResult = result[ruleID];
-    ruleResult.what = rule.what || '';
-    const startTime = Date.now();
-    let timeout;
-    let testRetries = 2;
-    let testSuccess = false;
-    // Until all permitted retries are exhausted or the test succeeds:
-    while (testRetries > 0 && ! testSuccess) {
+      console.log(`Starting rule ${ruleResult.id}`);
+      // Make the browser emulate headedness in all cases, because performance does not suffer.
+      const headEmulation = ruleResult.id.startsWith('shoot') ? 'high' : 'high';
+      // Get whether the rule needs a new browser launched.
+      const needsLaunch = ruleIndex === 0
+      || jobRules[ruleIndex - 1].contaminates
+      || jobRules[ruleIndex].needsAccessibleName && ! jobRules[ruleIndex - 1].needsAccessibleName;
+      const pageClosed = page && page.isClosed();
+      // If it does, or if the page has closed:
+      if (needsLaunch || pageClosed) {
+        // If the page has closed when it is expected to be open:
+        if (pageClosed && ! needsLaunch) {
+          // Report this.
+          console.log(`WARNING: Relaunching browser for test ${rule} after abnormal closure`);
+        }
+        // Create a browser, replace the page, and visit the target, retrying twice if necessary.
+        page = await launch({
+          report,
+          actIndex,
+          tempBrowserID: browserID,
+          tempURL: url,
+          xPathNeed: 'script',
+          needsAccessibleName: jobRules[ruleIndex].needsAccessibleName,
+          retries: 2
+        });
+      }
+      // Report crashes and disconnections during this test.
+      let crashHandler;
+      let disconnectHandler;
+      if (page && ! page.isClosed()) {
+        crashHandler = () => {
+          console.log(`ERROR: Page crashed during ${rule} test`);
+        };
+        page.on('crash', crashHandler);
+      }
+      const browser = page.context().browser();
+      if (browser) {
+        disconnectHandler = () => {
+          console.log(`ERROR: Browser disconnected during ${rule} test`);
+        };
+        browser.on('disconnected', disconnectHandler);
+      }
+      // Initialize an argument array for the reporter.
+      const ruleArgs = [page, report.catalog, withItems];
+      // If the rule has extra arguments:
+      if (argRules?.includes(ruleResult.id)) {
+        // Add them to the argument array.
+        ruleArgs.push(... args[ruleResult.id]);
+      }
+      const startTime = Date.now();
+      let timer;
       try {
         // Apply a time limit to the test.
         const timeLimit = 1000 * timeoutMultiplier * rule.timeOut;
+        let timeout;
         // If the time limit expires during the test:
-        const timer = new Promise(resolve => {
+        timer = new Promise(resolve => {
           timeout = setTimeout(() => {
-            // Add data about the test, including its prevention, to the result.
-            const endTime = Date.now();
-            testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
-            data.rulePreventions.push(ruleID);
-            data.rulePreventionMessages[ruleID] = 'Timeout';
-            ruleResult.totals = [0, 0, 0, 0];
-            ruleResult.standardInstances = [];
-            console.log(`ERROR: Test of testaro rule ${ruleID} timed out`);
+            // Add data about the timeout to the rule result.
+            ruleResult.prevented = true;
+            ruleResult.error = 'Timeout';
+            console.log(`ERROR: Test of testaro rule ${ruleResult.id} timed out`);
             resolve({timedOut: true});
           }, timeLimit);
         });
-        // Perform the test, subject to the time limit.
-        const ruleReport = require(`../testaro/${ruleID}`).reporter(... ruleArgs);
-        // Get the rule report or a timeout report.
-        const ruleOrTimeoutReport = await Promise.race([timer, ruleReport]);
-        // If the test was completed:
-        if (! ruleOrTimeoutReport.timedOut) {
-          // Add data from the rule report to the tool result.
-          const endTime = Date.now();
-          testTimes.push([ruleID, Math.round((endTime - startTime) / 1000)]);
-          Object.keys(ruleOrTimeoutReport).forEach(key => {
-            ruleResult[key] = ruleOrTimeoutReport[key];
-          });
-          // If the test was prevented:
-          if (ruleResult.data?.prevented && ruleResult.data.error) {
-            // Add this to the tool result.
-            data.rulePreventions.push(ruleID);
-            data.rulePreventionMessages[ruleID] = ruleResult.data.error;
+        // Try to perform the test and get a test report.
+        const testReport = require(`../testaro/${ruleResult.id}`).reporter(... ruleArgs);
+        // Get a test or timeout report.
+        const ruleReport = await Promise.race([timer, testReport]);
+        clearTimeout(timeout);
+        // If it was a test report:
+        if (! ruleReport.timedOut) {
+          // Add the rule-report properties to the rule result.
+          ruleResult.data = ruleReport.data;
+          ruleResult.totals = ruleReport.totals;
+          ruleResult.instances = ruleReport.standardInstances;
+          // Add the rule-result properties to the result.
+          if (Object.keys(ruleReport.data).length) {
+            data.ruleData[ruleResult.id] = ruleResult.data;
           }
-          // If the result includes totals:
           if (ruleResult.totals) {
-            // Round them.
-            ruleResult.totals = ruleResult.totals.map(total => Math.round(total));
+            ruleResult.totals.forEach((total, index) => {
+              standardResult.totals[index] += Math.round(total);
+            });
+          }
+          if (ruleResult.instances) {
+            standardResult.instances.push(... ruleResult.instances);
           }
-          const ruleDataMiscKeys = Object
-          .keys(ruleResult.data)
-          .filter(key => ! ['prevented', 'error'].includes(key));
-          // For any other property of the rule report data object:
-          ruleDataMiscKeys.forEach(key => {
-            data.ruleData[ruleID] ??= {};
-            // Add it to the tool result.
-            data.ruleData[ruleID][key] = ruleResult.data[key];
-          });
-          // Prevent a retry of the test.
-          testSuccess = true;
           // If testing is to stop after a failure and the page failed the test:
-          if (stopOnFail && ruleResult.totals && ruleResult.totals.some(total => total)) {
-            // Stop testing.
+          if (stopOnFail && ruleReport.totals?.some(total => total)) {
+            // Test for no more rules.
             break;
           }
         }
-        // Otherwise, i.e. if the test timed out:
-        else {
-          // Report this.
-          data.rulePreventions.push(ruleID);
-          data.rulePreventionMessages[ruleID] = 'Timeout';
-          // Stop retrying the test.
-          break;
-        }
       }
       // If an error is thrown by the test:
       catch(error) {
         const isPageClosed = ['closed', 'Protocol error', 'Target page'].some(phrase =>
           error.message.includes(phrase)
         );
-        // If the page has closed and there are retries left:
-        if (isPageClosed && testRetries) {
-          // Report this and decrement the allowed retry count.
-          console.log(
-            `WARNING: Retry ${3 - testRetries--} of test ${ruleID} starting after page closed`
-          );
-          await wait(2000);
-          // Replace the browser and the page in the run module and navigate to the target.
-          await launch(
-            report,
-            actIndex,
-            headEmulation,
-            report.browserID,
-            url
-          );
-          page = require('../run').page;
-          // If the page replacement failed:
-          if (! page) {
-            // Report this.
-            console.log(`ERROR: Browser relaunch to retry test ${ruleID} failed`);
-            data.rulePreventions.push(ruleID);
-            data.rulePreventionMessages[ruleID] = 'Retry failure due to browser relaunch failure';
-            // Stop retrying the test.
-            break;
-          }
-          // Update the rule arguments with the current page.
-          ruleArgs[0] = page;
+        // If the page has closed:
+        if (isPageClosed) {
+          // Report this.
+          console.log(`ERROR: Test ${ruleResult.id} failed because page closed`);
         }
-        // Otherwise, i.e. if the page is open or it is closed but no retries are left:
+        // Otherwise, i.e. if the page is open:
         else {
-          // Treat the test as prevented.
-          data.rulePreventions.push(ruleID);
-          data.rulePreventionMessages[ruleID] = error.message;
-          console.log(`ERROR: Test of testaro rule ${ruleID} prevented (${error.message})`);
-          // Do not retry the test even if retries are left.
-          break;
+          // Add this to the rule result.
+          ruleResult.prevented = true;
+          ruleResult.error = error.message;
+          console.log(
+            `ERROR: Test of testaro rule ${ruleResult.id} prevented (${error.message})`
+          );
         }
       }
       finally {
-        // Clear the timeout.
-        clearTimeout(timeout);
+        // Add the elapsed time to the rule result.
+        ruleResult.elapsedTime = Math.round((Date.now() - startTime) / 1000);
+        // Add the elapsed time to the data.
+        data.ruleTestTimes.push([ruleResult.id, ruleResult.elapsedTime]);
+        // If the test timed out or otherwise failed:
+        if (ruleResult.prevented) {
+          // Add this and the error to the data.
+          data.rulePreventions[ruleResult.id] = ruleResult.error;
+        }
       }
-    }
-    // Clear the error listeners.
-    if (page && ! page.isClosed() && crashHandler) {
-      page.off('crash', crashHandler);
-      crashHandler = null;
-    }
-    if (browser && disconnectHandler) {
-      browser.off('disconnected', disconnectHandler);
-      disconnectHandler = null;
-    }
-    // Force a garbage collection.
-    try {
-      if (global.gc) {
-        global.gc();
+      // Sort the rule test times.
+      data.ruleTestTimes.sort((a, b) => b[1] - a[1]);
+      // Clear the error listeners.
+      if (page && ! page.isClosed() && crashHandler) {
+        page.off('crash', crashHandler);
+        crashHandler = null;
       }
-    }
-    catch(error) {}
-  };
-  // Record the test times in descending order.
-  testTimes.sort((a, b) => b[1] - a[1]).forEach(pair => {
-    data.ruleTestTimes[pair[0]] = pair[1];
-  });
+      if (browser && disconnectHandler) {
+        browser.off('disconnected', disconnectHandler);
+        disconnectHandler = null;
+      }
+      // Force a garbage collection.
+      try {
+        if (global.gc) {
+          global.gc();
+        }
+      }
+      catch(error) {}
+    };
+  }
+  // Otherwise, i.e. if the rule specification is invalid:
+  else {
+    // Report this and stop testing.
+    standardResult.prevented = true;
+    data.prevented = true;
+    const message = 'ERROR: Testaro rule specification invalid';
+    data.error = message;
+    console.log(message);
+  }
   return {
     data,
     result