testaro 60.12.0 → 60.13.0

package/UPGRADES.md

@@ -3305,3 +3305,5 @@ In a run by the Kilotest server on the [home page of the Open Source Collective]
 All of the tests with elapsed times longer than 2 seconds were not yet refactored. Some of the refactored tests applied `checkVisibility` to all `body` descendant elements.
 
 Credit for the speed improvement in refactored tests is apparently owed to the encapsulation of the entire test logic in a browser function, versus the repeated element-by-element execution of the same logic in Node.js with Playwright methods.
+
+Evidence for this hypothesis is provided by the change in elapsed time after refactoring of the `focOp` and `opFoc` tests. These two tests consumed 18 seconds before the refactoring. The refactoring combined them into a single `focAndOp` test with functionality equivalent to both original tests. The refactored test on the same target consumed 2 seconds, even though it reported and itemized 223 violations.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "60.12.0",
+  "version": "60.13.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/doTestAct.js

@@ -43,7 +43,7 @@ const tmpDir = os.tmpdir();
 
 // FUNCTIONS
 
-// Performs the tests of the act specified by the caller.
+// Performs the tests of an act.
 const doTestAct = async actIndex => {
   const reportPath = `${tmpDir}/report.json`;
   // Get the report from the temporary directory.

package/procs/testaro.js

@@ -90,7 +90,7 @@ const getRuleResult = exports.getRuleResult = async (
     // If itemization is required:
     if (withItems) {
       // Get the bounding box of the element.
-      const {tagName,id, location, excerpt} = elData;
+      const {tagName, id, location, excerpt} = elData;
       const box = location.type === 'box' ? location.spec : await boxOf(loc);
       // Add a standard instance to the result.
       standardInstances.push({
@@ -169,7 +169,7 @@ exports.doTest = async (
   getBadWhatString
 ) => {
   // Return totals and standard instances for the rule.
-  return await page.evaluate(args => {
+  return await page.evaluate(async args => {
     const [
       withItems,
       ruleID,
@@ -186,8 +186,8 @@ exports.doTest = async (
     // Get a violation function.
     const getBadWhat = eval(`(${getBadWhatString})`);
     // For each candidate:
-    candidates.forEach(element => {
-      const violationWhat = getBadWhat(element);
+    for (const candidate of candidates) {
+      const violationWhat = await getBadWhat(candidate);
       // If it violates the rule:
       if (violationWhat) {
         // Increment the violation count.
@@ -205,11 +205,11 @@ exports.doTest = async (
           }
           // Add an instance to the instances.
           instances.push(
-            window.getInstance(element, ruleID, ruleWhat, 1, ruleSeverity)
+            window.getInstance(candidate, ruleID, ruleWhat, 1, ruleSeverity)
           );
         }
       }
-    });
+    }
     // If there are any violations and itemization is not required:
     if (violationCount && ! withItems) {
       // Add a summary instance to the instances.
@@ -233,3 +233,104 @@ exports.doTest = async (
     ]
   );
 };
+// Returns a result from a basic test.
+exports.getBasicResult = async (
+  page, withItems, ruleID, ordinalSeverity, summaryTagName, whats, data, violations
+) => {
+  // If the test was prevented:
+  if (data.prevented) {
+    // Return this.
+    return {
+      data,
+      totals: [0, 0, 0, 0],
+      standardInstances: []
+    };
+  }
+  // Otherwise, i.e. if the test was not prevented:
+  const totals = [0, 0, 0, 0];
+  totals[ordinalSeverity] = violations.length;
+  const standardInstances = [];
+  // If itemization is required:
+  if (withItems) {
+    // For each violation:
+    for (const violation of violations) {
+      const {loc, what} = violation;
+      const elData = await getLocatorData(loc);
+      // Get the bounding box of the element.
+      const {tagName, id, location, excerpt} = elData;
+      const box = location.type === 'box' ? location.spec : await boxOf(loc);
+      // Add a standard instance to the instances.
+      standardInstances.push({
+        ruleID,
+        what,
+        ordinalSeverity,
+        tagName,
+        id,
+        location,
+        excerpt,
+        boxID: boxToString(box),
+        pathID: tagName === 'HTML' ? '/html' : await xPath(loc)
+      });
+    }
+  }
+  // Otherwise, i.e. if itemization is not required:
+  else {
+    // Add a summary instance to the instances.
+    standardInstances.push({
+      ruleID,
+      what: whats,
+      ordinalSeverity,
+      summaryTagName,
+      id: '',
+      location: {},
+      excerpt: '',
+      boxID: '',
+      pathID: ''
+    });
+  }
+  // Return the result.
+  return {
+    data,
+    totals,
+    standardInstances
+  };
+};
+// Returns an awaited change in a visible element count.
+exports.getVisibleCountChange = async (
+  rootLoc, elementCount0, timeLimit = 400, settleInterval = 75
+) => {
+  const startTime = Date.now();
+  let timeout;
+  let settleChecker;
+  let elementCount1 = elementCount0;
+  // Set a time limit on the change.
+  const timeoutPromise = new Promise(resolve => {
+    timeout = setTimeout(() => {
+      clearInterval(settleChecker);
+      resolve();
+    }, timeLimit);
+  });
+  // Until the time limit expires, periodically:
+  const settlePromise = new Promise(resolve => {
+    settleChecker = setInterval(async () => {
+      const visiblesLoc = await rootLoc.locator('*:visible');
+      // Get the count.
+      elementCount1 = await visiblesLoc.count();
+      // If the count has changed:
+      if (elementCount1 !== elementCount0) {
+        // Stop.
+        clearTimeout(timeout);
+        clearInterval(settleChecker);
+        resolve();
+      }
+    }, settleInterval);
+  });
+  // When a change occurs or the time limit expires:
+  await Promise.race([timeoutPromise, settlePromise]);
+  const elapsedTime = Math.round(Date.now() - startTime);
+  // Return the change.
+  return {
+    change: elementCount1 - elementCount0,
+    elapsedTime
+  };
+};

package/run.js

@@ -98,7 +98,7 @@ const timeLimits = {
   alfa: 20,
   ed11y: 30,
   ibm: 30,
-  testaro: 150 + Math.round(6 * waits / 1000)
+  testaro: 200 + Math.round(6 * waits / 1000)
 };
 // Timeout multiplier.
 const timeoutMultiplier = Number.parseFloat(process.env.TIMEOUT_MULTIPLIER) || 1;

package/testaro/adbID.js

@@ -40,7 +40,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get the IDs in the aria-describedby attribute of the element.
     const IDs = element.getAttribute('aria-describedby').trim().split(/\s+/).filter(Boolean);
@@ -74,8 +73,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'Elements have aria-describedby attributes with missing or invalid id values';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'adbID', '[aria-describedby]', whats, 3, null, getBadWhat.toString()
   );
 };

package/testaro/altScheme.js

@@ -37,7 +37,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get the value of the alt attribute of the element.
     const alt = (element.getAttribute('alt') || '').trim();
@@ -56,8 +55,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'img elements have alt attributes with URL or filename values';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'altScheme', 'img[alt]', whats, 1, 'IMG', getBadWhat.toString()
   );
 };

package/testaro/captionLoc.js

@@ -36,7 +36,6 @@ const {doTest} = require('../procs/testaro');
 // FUNCTIONS
 
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     const parent = element.parentElement;
     // If the element is not the first child of a table element:
@@ -46,8 +45,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'caption elements are not the first children of table elements';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'captionLoc', 'caption', whats, 3, 'CAPTION', getBadWhat.toString()
   );
 };

package/testaro/datalistRef.js

@@ -61,7 +61,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'list attributes of input elements are empty or IDs of no or non-datalist elements';
-  return doTest(
+  return await doTest(
     page, withItems, 'datalistRef', 'input[list]', whats, 3, 'INPUT', getBadWhat.toString()
   );
 };

package/testaro/embAc.js

@@ -48,5 +48,5 @@ exports.reporter = async (page, withItems) => {
   .map(tag => `a ${tag}, button ${tag}`)
   .join(', ');
   const whats = 'interactive elements are embedded in links or buttons';
-  return doTest(page, withItems, 'embAc', selector, whats, 2, null, getBadWhat.toString());
+  return await doTest(page, withItems, 'embAc', selector, whats, 2, null, getBadWhat.toString());
 };

package/testaro/focAndOp.js

@@ -45,7 +45,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get whether the element is visible.
     const isVisible = element.checkVisibility({
@@ -139,8 +138,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'Elements are Tab-focusable but not operable or vice versa';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'focAndOp', 'body *', whats, 2, null, getBadWhat.toString()
   );
 };

package/testaro/focInd.js

@@ -50,7 +50,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get whether the element is visible.
     const isVisible = element.checkVisibility({
@@ -108,8 +107,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'Elements fail to have standard focus indicators';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'focInd', 'body *', whats, 1, null, getBadWhat.toString()
   );
 };

package/testaro/focVis.js

@@ -29,45 +29,33 @@
   This test reports links that are at least partly off the display when focused.
 */
 
-// ########## IMPORTS
+// IMPORTS
 
-// Module to perform common operations.
-const {init, getRuleResult} = require('../procs/testaro');
+const {doTest} = require('../procs/testaro');
 
-// ########## FUNCTIONS
+// FUNCTIONS
 
-// Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Initialize a sample of locators and a result.
-  const all = await init(100, page, 'a:visible');
-  // For each locator:
-  for (const loc of all.allLocs) {
-    // Focus it.
-    await loc.focus();
-    // Get its location.
-    const box = await loc.boundingBox();
-    // Get how its element violates the rule, if it does.
-    const isBad = [box.x < 0, box.y < 0];
-    // If it does:
-    if (isBad.some(item => item)) {
-      // Add the locator to the array of violators.
-      let param;
-      if (isBad.every(item => item)) {
-        param = 'above and to the left of';
+  const getBadWhat = element => {
+    const isVisible = element.checkVisibility({
+      contentVisibilityAuto: true,
+      opacityProperty: true,
+      visibilityProperty: true
+    });
+    // If the element is visible:
+    if (isVisible) {
+      // Focus it.
+      element.focus();
+      const box = element.getBoundingClientRect();
+      // If it violates the rule:
+      if (box.x < 0 || box.y < 0) {
+        // Return a violation description.
+        return 'Upper left corner of the element is above or to the left of the display';
       }
-      else if (isBad[0]) {
-        param = 'to the left of';
-      }
-      else {
-        param = 'above';
-      }
-      all.locs.push([loc, param]);
     }
-  }
-  // Populate and return the result.
-  const whats = [
-    'Visible link is __param__ the display',
-    'Visible links are above or to the left of the display'
-  ];
-  return await getRuleResult(withItems, all, 'focVis', whats, 2);
+  };
+  const whats = 'Visible links are above or to the left of the display';
+  return await doTest(
+    page, withItems, 'focVis', 'a', whats, 2, 'A', getBadWhat.toString()
+  );
 };

package/testaro/hover-draft0.js

@@ -0,0 +1,110 @@
+/*
+  © 2021–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+
+  MIT License
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/*
+  hover
+  This test reports unexpected impacts of hovering. The elements that are subjected to hovering
+  (called “triggers”) include all the elements that have ARIA attributes associated with control
+  over the visibility of other elements and all the elements that have onmouseenter or
+  onmouseover attributes, as well as a sample of all visible elements in the body. If hovering over
+  an element results in an increase or decrease in the total count of visible elements in the body,
+  the rule is considered violated.
+*/
+
+// IMPORTS
+
+const {doTest} = require('../procs/testaro');
+
+// FUNCTIONS
+
+exports.reporter = async (page, withItems) => {
+  const getBadWhat = element => {
+    let violationDescription;
+    const hoverEvent = new MouseEvent('mouseover', {
+      bubbles: true,
+      cancelable: true,
+      view: window
+    });
+    let timer;
+    // Create a mutation observer.
+    const observer = new MutationObserver(mutations => {
+      // When any mutation occurs in any other element(s):
+      const otherMutatedRecords = mutations.filter(
+        record => record.target !== element && record.target.getAttribute('role') !== 'tooltip'
+      );
+      // Update the count of mutated elements and the violation description.
+      const impactCount = otherMutatedRecords.length;
+      const impactWhat = impactCount === 1 ? '1 other element1' : `${impactCount} other elements`;
+      violationDescription = `Hovering over the element adds, removes, or changes ${impactWhat}`;
+      // Stop the observer.
+      observer.disconnect();
+      // Clear the timer.
+      clearTimeout(timer);
+    });
+    // Ensure that the mouse is in the home position.
+    document.body.dispatchEvent(hoverEvent);
+    // Start observing.
+    observer.observe(document.body, {
+      attributes: true,
+      subtree: true,
+      childList: true
+    });
+    // Start hovering over the element.
+    element.dispatchEvent(hoverEvent);
+    // In case no other elements were mutated within 200ms, stop the observer.
+    timer = setTimeout(() => {
+      observer.disconnect();
+    }, 200);
+    // If any other elements were mutated within 200ms:
+    if (violationDescription) {
+      // Return the violation description.
+      return violationDescription;
+    }
+  };
+  const selector = [
+   '[aria-controls]',
+   '[aria-expanded]',
+   '[aria-haspopup]',
+   '[onmouseenter]',
+   '[onmouseover]',
+   '[onmouseenter]',
+   '[onmouseover]',
+   '[role="menu"]',
+   '[role="menubar"]',
+   '[role="menuitem"]',
+   '[data-tooltip]',
+   '[data-popover]',
+   '[data-hover]',
+   '[data-menu]',
+   '[data-dropdown]',
+   '[role="tab"]',
+   '[role="combobox"]',
+   'li'
+  ].join(', ');
+  const whats = 'Hovering over elements adds, removes, or changes other elements';
+  return await doTest(
+    page, withItems, 'hover', selector, whats, 0, '', getBadWhat.toString()
+  );
+};

package/testaro/hover-draft1.js

@@ -0,0 +1,185 @@
+/*
+  © 2021–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+
+  MIT License
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/*
+  hover
+  This test reports unexpected impacts of hovering. The elements that are subjected to hovering
+  (called “triggers”) include all the elements that have ARIA attributes associated with control
+  over the visibility of other elements and all the elements that have onmouseenter or
+  onmouseover attributes, as well as a sample of all visible elements in the body. If hovering over
+  an element results in an increase or decrease in the total count of visible elements in the body,
+  the rule is considered violated.
+*/
+
+// IMPORTS
+
+const {doTest} = require('../procs/testaro');
+
+// FUNCTIONS
+
+exports.reporter = async (page, withItems) => {
+  const getBadWhat = async element => {
+    const isVisible = element.checkVisibility({
+      contentVisibilityAuto: true,
+      opacityProperty: true,
+      visibilityProperty: true
+    });
+    // If the element is visible and is not a tooltip:
+    if (isVisible && element.getAttribute('role') !== 'tooltip') {
+      let timer;
+      let observer;
+      const options = {
+        bubbles: true,
+        cancelable: true,
+        view: window
+      };
+      const hoverEvents = [
+        new MouseEvent('mouseover', options),
+        new MouseEvent('mousemove', options),
+        new PointerEvent('pointerover', options),
+        new PointerEvent('pointermove', options)
+      ];
+      const {__lastHoveredElement} = window;
+      // Exit the prior hover location, if any.
+      if (__lastHoveredElement) {
+        [
+          [MouseEvent, 'mouseout', true],
+          [MouseEvent, 'mouseleave', false],
+          [PointerEvent, 'pointerout', true],
+          [PointerEvent, 'pointerleave', false]
+        ].forEach(([event, type, bubbles]) => {
+          __lastHoveredElement.dispatchEvent(new event(type, {bubbles}));
+        });
+      }
+      // Allow time for handlers of these events to complete execution.
+      await new Promise(resolve => setTimeout(resolve, 800));
+      // Check whether the visibility of the element was due solely to the prior hovering.
+      const isStillVisible = element.checkVisibility({
+        contentVisibilityAuto: true,
+        opacityProperty: true,
+        visibilityProperty: true
+      });
+      // If so:
+      if (isStillVisible) {
+        const observationStart = Date.now();
+        // Execute a Promise that resolves when a mutation is observed.
+        const mutationPromise = new Promise(resolve => {
+          // When mutations are observed:
+          observer = new MutationObserver(mutationRecords => {
+            const otherMutationRecords = mutationRecords.filter(record => {
+              const {target, type} = record;
+              return type !== 'childList'
+              && target !== element
+              && target.getAttribute('role') !== 'tooltip';
+            });
+            // If any are reportable:
+            if (otherMutationRecords.length) {
+              // Get a non-duplicative set of their types and XPaths.
+              const impacts = new Set();
+              otherMutationRecords.forEach(record => {
+                const {attributeName, target, type} = record;
+                const xPath = getXPath(target);
+                const attributeSuffix = attributeName ? `:${attributeName}` : '';
+                const textStart = target.textContent?.slice(0, 20).trim().replace(/\s+/g, ' ') || '';
+                impacts.add(`${type}${attributeSuffix}@${xPath} (“${textStart}”)`);
+              });
+              const impactTime = Math.round(Date.now() - observationStart);
+              // Create a violation description with the elapsed time and the mutation details.
+              const violationWhat = `Hovering over the element makes these changes after ${impactTime}ms: ${Array.from(impacts).join(', ')}`;
+              // Clear the timer.
+              clearTimeout(timer);
+              // Stop the observer.
+              observer.disconnect();
+              // Resolve the Promise with the violation description.
+              resolve(violationWhat);
+            }
+          });
+          let observationRoot = element.parentElement.parentElement;
+          const rootTagName = observationRoot.tagName;
+          if (['MAIN', 'BODY'].includes(rootTagName)) {
+            observationRoot = element.parentElement;
+          }
+          // Start observing.
+          observer.observe(observationRoot, {
+            attributes: true,
+            attributeFilter: ['style', 'class', 'hidden', 'aria-hidden', 'disabled', 'open'],
+            subtree: true,
+            childList: true
+          });
+          // Start hovering over the element.
+          hoverEvents.forEach(event => {
+            element.dispatchEvent(event);
+          });
+          // Record the element for future mouseout events.
+          window.__lastHoveredElement = element;
+        });
+        // Execute a Promise that resolves when a time limit expires.
+        const timeoutPromise = new Promise(resolve => {
+          // If no mutation is observed before the time limit:
+          timer = setTimeout(() => {
+            // Stop the observer.
+            observer.disconnect();
+            // Resolve the Promise with an empty string.
+            resolve('');
+          }, 400);
+        });
+        // Get the violation description or timeout report.
+        const violationWhat = await Promise.race([mutationPromise, timeoutPromise]);
+        // If any mutations occurred before the time limit:
+        if (violationWhat) {
+          // Return the violation description.
+          return violationWhat;
+        }
+        //XXX Temp
+        return 'No mutations';
+      }
+    }
+  };
+  const selector = [
+   '[aria-controls]',
+   '[aria-expanded]',
+   '[aria-haspopup]',
+   '[onmouseenter]',
+   '[onmouseover]',
+   '[onmouseenter]',
+   '[onmouseover]',
+   '[role="menu"]',
+   '[role="menubar"]',
+   '[role="menuitem"]',
+   '[data-tooltip]',
+   '[data-popover]',
+   '[data-hover]',
+   '[data-menu]',
+   '[data-dropdown]',
+   '[role="tab"]',
+   '[role="combobox"]',
+   'a',
+   'button'
+  ].join(', ');
+  const whats = 'Hovering over elements adds, removes, or changes other elements';
+  return await doTest(
+    page, withItems, 'hover', selector, whats, 0, '', getBadWhat.toString()
+  );
+};