testaro 60.11.0 → 60.13.0

package/UPGRADES.md

@@ -3239,3 +3239,71 @@ Given your goal to move on with [testaro](cci:7://file:///Users/pool/Users/pool/
 - Plan separately for a future **element-matching module** that consumes all tools’ outputs and decides “these instances refer to the same DOM element”.
 
 That way, you get immediate performance and implementation wins in [testaro](cci:7://file:///Users/pool/Users/pool/Documents/Topics/work/testaro:0:0-0:0), without locking yourself into the fragile assumption that “XPath strings must match exactly across all tools.
+
+## Sampling and performance
+
+Refactoring Testaro tests to eliminate sampling of elements began in December 2025. Initial results suggest that refactoring decreases elapsed test times despite the fact that all applicable elements are examined rather than only a sample.
+
+In a run by the Kilotest server on the [home page of the Open Source Collective](https://opencollective.com/opensource), with about 2700 visible elements,the elapsed times of Testaro tests were:
+
+```json
+"ruleTestTimes": {
+  "allCaps": 11,
+  "opFoc": 10,
+  "allSlanted": 9,
+  "hovInd": 9,
+  "focOp": 8,
+  "targetSmall": 7,
+  "focAll": 7,
+  "focVis": 6,
+  "distortion": 5,
+  "linkAmb": 5,
+  "titledEl": 5,
+  "zIndex": 5,
+  "targetTiny": 4,
+  "shoot1": 4,
+  "linkTitle": 3,
+  "hover": 3,
+  "shoot0": 2,
+  "adbID": 2,
+  "linkUl": 2,
+  "buttonMenu": 2,
+  "focInd": 2,
+  "tabNav": 2,
+  "dupAtt": 0,
+  "imageLink": 1,
+  "labClash": 1,
+  "allHidden": 0,
+  "altScheme": 0,
+  "autocomplete": 0,
+  "bulk": 0,
+  "captionLoc": 0,
+  "datalistRef": 0,
+  "docType": 0,
+  "embAc": 0,
+  "headEl": 0,
+  "headingAmb": 0,
+  "hr": 0,
+  "legendLoc": 0,
+  "lineHeight": 0,
+  "linkExt": 0,
+  "linkOldAtt": 0,
+  "linkTo": 0,
+  "miniText": 0,
+  "nonTable": 0,
+  "optRoleSel": 0,
+  "phOnly": 0,
+  "pseudoP": 0,
+  "radioSet": 0,
+  "role": 0,
+  "secHeading": 0,
+  "styleDiff": 0,
+  "textSem": 0
+}
+```
+
+All of the tests with elapsed times longer than 2 seconds were not yet refactored. Some of the refactored tests applied `checkVisibility` to all `body` descendant elements.
+
+Credit for the speed improvement in refactored tests is apparently owed to the encapsulation of the entire test logic in a browser function, versus the repeated element-by-element execution of the same logic in Node.js with Playwright methods.
+
+Evidence for this hypothesis is provided by the change in elapsed time after refactoring of the `focOp` and `opFoc` tests. These two tests consumed 18 seconds before the refactoring. The refactoring combined them into a single `focAndOp` test with functionality equivalent to both original tests. The refactored test on the same target consumed 2 seconds, even though it reported and itemized 223 violations.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "60.11.0",
+  "version": "60.13.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/doTestAct.js

@@ -43,7 +43,7 @@ const tmpDir = os.tmpdir();
 
 // FUNCTIONS
 
-// Performs the tests of the act specified by the caller.
+// Performs the tests of an act.
 const doTestAct = async actIndex => {
   const reportPath = `${tmpDir}/report.json`;
   // Get the report from the temporary directory.

package/procs/screenShot.js

@@ -18,7 +18,7 @@ exports.screenShot = async (page, exclusion = null) => {
   const options = {
     fullPage: true,
     omitBackground: true,
-    timeout: 2000
+    timeout: 4000
   };
   if (exclusion) {
     options.mask = [exclusion];

package/procs/testaro.js

@@ -90,7 +90,7 @@ const getRuleResult = exports.getRuleResult = async (
     // If itemization is required:
     if (withItems) {
       // Get the bounding box of the element.
-      const {tagName,id, location, excerpt} = elData;
+      const {tagName, id, location, excerpt} = elData;
       const box = location.type === 'box' ? location.spec : await boxOf(loc);
       // Add a standard instance to the result.
       standardInstances.push({
@@ -169,7 +169,7 @@ exports.doTest = async (
   getBadWhatString
 ) => {
   // Return totals and standard instances for the rule.
-  return await page.evaluate(args => {
+  return await page.evaluate(async args => {
     const [
       withItems,
       ruleID,
@@ -186,21 +186,30 @@ exports.doTest = async (
     // Get a violation function.
     const getBadWhat = eval(`(${getBadWhatString})`);
     // For each candidate:
-    candidates.forEach(element => {
-      const violationWhat = getBadWhat(element);
+    for (const candidate of candidates) {
+      const violationWhat = await getBadWhat(candidate);
       // If it violates the rule:
       if (violationWhat) {
         // Increment the violation count.
         violationCount++;
         // If itemization is required:
         if (withItems) {
+          const violationWhatStart = violationWhat.slice(0, 2);
+          let ruleSeverity = severity;
+          let ruleWhat = violationWhat
+          // If this violation has a custom severity:
+          if (/[0-3]:/.test(violationWhatStart)) {
+            // Get it and remove it from the violation description.
+            ruleSeverity = Number(violationWhat[0]);
+            ruleWhat = violationWhat.slice(2);
+          }
           // Add an instance to the instances.
           instances.push(
-            window.getInstance(element, ruleID, violationWhat, 1, severity)
+            window.getInstance(candidate, ruleID, ruleWhat, 1, ruleSeverity)
           );
         }
       }
-    });
+    }
     // If there are any violations and itemization is not required:
     if (violationCount && ! withItems) {
       // Add a summary instance to the instances.
@@ -224,3 +233,104 @@ exports.doTest = async (
     ]
   );
 };
+// Returns a result from a basic test.
+exports.getBasicResult = async (
+  page, withItems, ruleID, ordinalSeverity, summaryTagName, whats, data, violations
+) => {
+  // If the test was prevented:
+  if (data.prevented) {
+    // Return this.
+    return {
+      data,
+      totals: [0, 0, 0, 0],
+      standardInstances: []
+    };
+  }
+  // Otherwise, i.e. if the test was not prevented:
+  const totals = [0, 0, 0, 0];
+  totals[ordinalSeverity] = violations.length;
+  const standardInstances = [];
+  // If itemization is required:
+  if (withItems) {
+    // For each violation:
+    for (const violation of violations) {
+      const {loc, what} = violation;
+      const elData = await getLocatorData(loc);
+      // Get the bounding box of the element.
+      const {tagName, id, location, excerpt} = elData;
+      const box = location.type === 'box' ? location.spec : await boxOf(loc);
+      // Add a standard instance to the instances.
+      standardInstances.push({
+        ruleID,
+        what,
+        ordinalSeverity,
+        tagName,
+        id,
+        location,
+        excerpt,
+        boxID: boxToString(box),
+        pathID: tagName === 'HTML' ? '/html' : await xPath(loc)
+      });
+    }
+  }
+  // Otherwise, i.e. if itemization is not required:
+  else {
+    // Add a summary instance to the instances.
+    standardInstances.push({
+      ruleID,
+      what: whats,
+      ordinalSeverity,
+      summaryTagName,
+      id: '',
+      location: {},
+      excerpt: '',
+      boxID: '',
+      pathID: ''
+    });
+  }
+  // Return the result.
+  return {
+    data,
+    totals,
+    standardInstances
+  };
+};
+// Returns an awaited change in a visible element count.
+exports.getVisibleCountChange = async (
+  rootLoc, elementCount0, timeLimit = 400, settleInterval = 75
+) => {
+  const startTime = Date.now();
+  let timeout;
+  let settleChecker;
+  let elementCount1 = elementCount0;
+  // Set a time limit on the change.
+  const timeoutPromise = new Promise(resolve => {
+    timeout = setTimeout(() => {
+      clearInterval(settleChecker);
+      resolve();
+    }, timeLimit);
+  });
+  // Until the time limit expires, periodically:
+  const settlePromise = new Promise(resolve => {
+    settleChecker = setInterval(async () => {
+      const visiblesLoc = await rootLoc.locator('*:visible');
+      // Get the count.
+      elementCount1 = await visiblesLoc.count();
+      // If the count has changed:
+      if (elementCount1 !== elementCount0) {
+        // Stop.
+        clearTimeout(timeout);
+        clearInterval(settleChecker);
+        resolve();
+      }
+    }, settleInterval);
+  });
+  // When a change occurs or the time limit expires:
+  await Promise.race([timeoutPromise, settlePromise]);
+  const elapsedTime = Math.round(Date.now() - startTime);
+  // Return the change.
+  return {
+    change: elementCount1 - elementCount0,
+    elapsedTime
+  };
+};

package/run.js

@@ -98,7 +98,7 @@ const timeLimits = {
   alfa: 20,
   ed11y: 30,
   ibm: 30,
-  testaro: 150 + Math.round(6 * waits / 1000)
+  testaro: 200 + Math.round(6 * waits / 1000)
 };
 // Timeout multiplier.
 const timeoutMultiplier = Number.parseFloat(process.env.TIMEOUT_MULTIPLIER) || 1;

package/testaro/adbID.js

@@ -40,7 +40,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get the IDs in the aria-describedby attribute of the element.
     const IDs = element.getAttribute('aria-describedby').trim().split(/\s+/).filter(Boolean);
@@ -74,8 +73,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'Elements have aria-describedby attributes with missing or invalid id values';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'adbID', '[aria-describedby]', whats, 3, null, getBadWhat.toString()
   );
 };

package/testaro/altScheme.js

@@ -37,7 +37,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get the value of the alt attribute of the element.
     const alt = (element.getAttribute('alt') || '').trim();
@@ -56,8 +55,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'img elements have alt attributes with URL or filename values';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'altScheme', 'img[alt]', whats, 1, 'IMG', getBadWhat.toString()
   );
 };

package/testaro/captionLoc.js

@@ -36,7 +36,6 @@ const {doTest} = require('../procs/testaro');
 // FUNCTIONS
 
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     const parent = element.parentElement;
     // If the element is not the first child of a table element:
@@ -46,8 +45,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'caption elements are not the first children of table elements';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'captionLoc', 'caption', whats, 3, 'CAPTION', getBadWhat.toString()
   );
 };

package/testaro/datalistRef.js

@@ -61,7 +61,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'list attributes of input elements are empty or IDs of no or non-datalist elements';
-  return doTest(
+  return await doTest(
     page, withItems, 'datalistRef', 'input[list]', whats, 3, 'INPUT', getBadWhat.toString()
   );
 };

package/testaro/embAc.js

@@ -48,5 +48,5 @@ exports.reporter = async (page, withItems) => {
   .map(tag => `a ${tag}, button ${tag}`)
   .join(', ');
   const whats = 'interactive elements are embedded in links or buttons';
-  return doTest(page, withItems, 'embAc', selector, whats, 2, null, getBadWhat.toString());
+  return await doTest(page, withItems, 'embAc', selector, whats, 2, null, getBadWhat.toString());
 };

package/testaro/focAndOp.js

@@ -0,0 +1,144 @@
+/*
+  © 2021–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+
+  MIT License
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/*
+  focAndOp
+  Related to Tenon rule 190.
+
+  This test reports discrepancies between Tab-focusability and operability. The standard practice
+  is to make focusable elements operable and operable elements focusable. If focusable elements are
+  not operable, users are likely to be surprised that nothing happens when they try to operate such
+  elements. Conversely, if operable elements are not focusable, users who navigate with a
+  keyboard are prevented from operating those elements.  The test considers an element
+  Tab-focusable if its tabIndex property has the value 0. The test considers an element operable if
+  it has a non-inherited pointer cursor and is not a 'LABEL' element, has an operable tag name, has
+  an interactive explicit role, or has an 'onclick' attribute.
+*/
+
+// IMPORTS
+
+const {doTest} = require('../procs/testaro');
+
+// FUNCTIONS
+
+// Runs the test and returns the result.
+exports.reporter = async (page, withItems) => {
+  const getBadWhat = element => {
+    // Get whether the element is visible.
+    const isVisible = element.checkVisibility({
+      contentVisibilityAuto: true,
+      opacityProperty: true,
+      visibilityProperty: true
+    });
+    // If so:
+    if (isVisible) {
+      // Get whether it is focusable.
+      const isFocusable = element.tabIndex === 0;
+      // Get the operable tagnames.
+      const opTags = new Set(['A', 'BUTTON', 'IFRAME', 'INPUT','OPTION', 'SELECT', 'TEXTAREA']);
+      // Get the operable roles.
+      const opRoles = new Set([
+        'button',
+        'checkbox',
+        'combobox',
+        'composite',
+        'grid',
+        'gridcell',
+        'input',
+        'link',
+        'listbox',
+        'menu',
+        'menubar',
+        'menuitem',
+        'menuitemcheckbox',
+        'option',
+        'radio',
+        'radiogroup',
+        'scrollbar',
+        'searchbox',
+        'select',
+        'slider',
+        'spinbutton',
+        'switch',
+        'tab',
+        'tablist',
+        'textbox',
+        'tree',
+        'treegrid',
+        'treeitem',
+        'widget',
+      ]);
+      // Initialize the operabilities of the element.
+      const opHow = [];
+      let hasPointer = false;
+      // If the element is not a label:
+      if (element.tagName !== 'LABEL') {
+        const styleDec = window.getComputedStyle(element);
+        hasPointer = styleDec.cursor === 'pointer';
+        // If it has a pointer cursor:
+        if (hasPointer) {
+          // Neutralize the cursor style of the parent element of the element.
+          element.parentElement.style.cursor = 'default';
+          // Get whether, after this, the element still has a pointer cursor.
+          hasPointer = styleDec.cursor === 'pointer';
+          // Add this to the operabilities of the element.
+          opHow.push('pointer cursor');
+        }
+      }
+      // If the element has a click event listener:
+      if (element.onclick) {
+        // Add this to the operabilities.
+        opHow.push('click listener');
+      }
+      // If the element has an operable explicit role:
+      const role = element.getAttribute('role');
+      if (opRoles.has(role)) {
+        // Add this to the operabilities.
+        opHow.push(`role ${role}`);
+      }
+      // If the element has an operable tagname:
+      const tagName = element.tagName;
+      if (opTags.has(tagName)) {
+        // Add this to the operabilities.
+        opHow.push(`tagname ${tagName}`);
+      }
+      const isOperable = opHow.length > 0;
+      // If the element is focusable but not operable:
+      if (isFocusable && ! isOperable) {
+        // Return a severity and violation description.
+        return '2:Element is Tab-focusable but not operable';
+      }
+      // Otherwise, if it is operable but not focusable:
+      else if (isOperable && ! isFocusable) {
+        // Return a severity and violation description.
+        return `3:Element is operable (${opHow.join(', ')}) but not Tab-focusable`;
+      }
+    }
+  };
+  const whats = 'Elements are Tab-focusable but not operable or vice versa';
+  return await doTest(
+    page, withItems, 'focAndOp', 'body *', whats, 2, null, getBadWhat.toString()
+  );
+};

package/testaro/focInd.js

@@ -1,5 +1,6 @@
 /*
   © 2021–2023 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -49,7 +50,6 @@ const {doTest} = require('../procs/testaro');
 
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Define a violation function for execution in the browser.
   const getBadWhat = element => {
     // Get whether the element is visible.
     const isVisible = element.checkVisibility({
@@ -107,8 +107,7 @@ exports.reporter = async (page, withItems) => {
     }
   };
   const whats = 'Elements fail to have standard focus indicators';
-  // Perform the test and return the result.
-  return doTest(
+  return await doTest(
     page, withItems, 'focInd', 'body *', whats, 1, null, getBadWhat.toString()
   );
 };

package/testaro/focVis.js

@@ -29,45 +29,33 @@
   This test reports links that are at least partly off the display when focused.
 */
 
-// ########## IMPORTS
+// IMPORTS
 
-// Module to perform common operations.
-const {init, getRuleResult} = require('../procs/testaro');
+const {doTest} = require('../procs/testaro');
 
-// ########## FUNCTIONS
+// FUNCTIONS
 
-// Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
-  // Initialize a sample of locators and a result.
-  const all = await init(100, page, 'a:visible');
-  // For each locator:
-  for (const loc of all.allLocs) {
-    // Focus it.
-    await loc.focus();
-    // Get its location.
-    const box = await loc.boundingBox();
-    // Get how its element violates the rule, if it does.
-    const isBad = [box.x < 0, box.y < 0];
-    // If it does:
-    if (isBad.some(item => item)) {
-      // Add the locator to the array of violators.
-      let param;
-      if (isBad.every(item => item)) {
-        param = 'above and to the left of';
+  const getBadWhat = element => {
+    const isVisible = element.checkVisibility({
+      contentVisibilityAuto: true,
+      opacityProperty: true,
+      visibilityProperty: true
+    });
+    // If the element is visible:
+    if (isVisible) {
+      // Focus it.
+      element.focus();
+      const box = element.getBoundingClientRect();
+      // If it violates the rule:
+      if (box.x < 0 || box.y < 0) {
+        // Return a violation description.
+        return 'Upper left corner of the element is above or to the left of the display';
       }
-      else if (isBad[0]) {
-        param = 'to the left of';
-      }
-      else {
-        param = 'above';
-      }
-      all.locs.push([loc, param]);
     }
-  }
-  // Populate and return the result.
-  const whats = [
-    'Visible link is __param__ the display',
-    'Visible links are above or to the left of the display'
-  ];
-  return await getRuleResult(withItems, all, 'focVis', whats, 2);
+  };
+  const whats = 'Visible links are above or to the left of the display';
+  return await doTest(
+    page, withItems, 'focVis', 'a', whats, 2, 'A', getBadWhat.toString()
+  );
 };