testaro 21.0.0 → 22.0.0

package/README.md

@@ -587,6 +587,10 @@ The `testaro` tool (like the `ibm` tool) has a `withItems` property. If you set
 
 Unlike any other tool, the `testaro` tools requires a `stopOnFail` property, which specifies whether a failure to conform to any rule (i.e. any value of `totals` other than `[0, 0, 0, 0]`) should terminate the execution of tests for the remaining rules.
 
+Warnings in the `testaro/hover.js`, `testaro/motion.js`, and `procs/visChange.js` files advise you to avoid launching particular browser types for the performance of particular Testaro tests.
+
+Several Testaro tests make use of the `init()` function in the `procs/testaro` module. That function samples elements if the population of elements to be tested is larger than 100. The purpose is to achieve reasonable performance. The sampling overweights elements near the beginning of a page, because of the tendency of that location to have important and atypical elements.
+
 You can add custom rules to the rules of any tool. Testaro provides a template, `data/template.js`, for the definition of a rule to be added. Once you have created a copy of the template with revisions, you can move the copy into the `testaro` directory and add an entry for your custom rule to the `evalRules` object in the `tests/testaro.js` file. Then your custom rule will act as a Testaro rule.
 
 ###### WAVE

package/data/template.js

@@ -13,12 +13,12 @@ const {init, report} = require('../procs/testaro');
 // Runs the test and returns the result.
 exports.reporter = async (page, withItems) => {
   // Initialize the locators and result.
-  const all = await init(page, 'body *');
+  const all = await init(page, 'body a');
   // For each locator:
   for (const loc of all.allLocs) {
     // Get whether its element violates the rule.
     const isBad = await loc.evaluate(el => {
-      const isViolator = el.tagName.length > 300;
+      const isViolator = ! el.href;
       return isViolator;
     });
     // If it does:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "21.0.0",
+  "version": "22.0.0",
   "description": "Run 920 web accessibility tests from 9 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/sample.js

@@ -10,7 +10,7 @@ exports.getSample = (population, sampleSize) => {
   const popSize = population.length;
   // If the sample is smaller than the population:
   if (sampleSize < popSize) {
-    // Assign to each trigger a priority randomly decreasing with its index.
+    // Assign to each individual a priority randomly decreasing with its index.
     const WeightedPopulation = population.map((item, index) => {
       const weight = 1 + Math.sin(Math.PI * index / popSize + Math.PI / 2);
       const priority = weight * Math.random();
@@ -20,7 +20,7 @@ exports.getSample = (population, sampleSize) => {
     const sortedPopulation = WeightedPopulation.sort((a, b) => b[1] - a[1]);
     const sample = sortedPopulation.slice(0, sampleSize);
     const domOrderSample = sample.sort((a, b) => a[0] - b[0]);
-    return domOrderSample.map(trigger => trigger[0]);
+    return domOrderSample.map(individual => individual[0]);
   }
   // Otherwise, i.e. if the sample is at least as large as the population:
   else {

package/procs/testaro.js

@@ -2,18 +2,32 @@
 
 // ########## IMPORTS
 
+// Module to sample a population.
+const {getSample} = require('../procs/sample');
 // Module to get locator data.
 const {getLocatorData} = require('../procs/getLocatorData');
 
+// ########## CONSTANTS
+
+let sampleSize = 100;
+
 // ########## FUNCTIONS
 
 // Initializes locators and a result.
 exports.init = async (page, locAllSelector, options = {}) => {
   // Get locators for the specified elements.
-  const locAll = page.locator(locAllSelector, options);
-  const allLocs = await locAll.all();
+  const locPop = page.locator(locAllSelector, options);
+  const locPops = await locPop.all();
+  const populationSize = locPops.length;
+  sampleSize = Math.min(sampleSize, populationSize);
+  const locIndexes = getSample(locPops, sampleSize);
+  const allLocs = locIndexes.map(index => locPops[index]);
   const result = {
-    data: {},
+    data: {
+      populationSize,
+      sampleSize,
+      populationRatio: sampleSize ? populationSize / sampleSize : null
+    },
     totals: [0, 0, 0, 0],
     standardInstances: []
   };
@@ -28,7 +42,7 @@ exports.init = async (page, locAllSelector, options = {}) => {
 // Populates a result.
 exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName = '') => {
   const {locs, result} = all;
-  const {totals, standardInstances} = result;
+  const {data, totals, standardInstances} = result;
   // For each instance locator:
   for (const locItem of locs) {
     // Get data on its element.
@@ -42,7 +56,7 @@ exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName
     }
     const elData = await getLocatorData(loc);
     // Add to the totals.
-    totals[ordinalSeverity]++;
+    totals[ordinalSeverity] += data.populationRatio;
     // If itemization is required:
     if (withItems) {
       // Add a standard instance to the result.
@@ -64,7 +78,7 @@ exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName
       ruleID,
       what: whats[1],
       ordinalSeverity,
-      count: totals[ordinalSeverity],
+      count: Math.round(totals[ordinalSeverity]),
       tagName,
       id: '',
       location: {

package/procs/visChange.js

@@ -23,7 +23,6 @@ const shoot = async (page, exclusion = null) => {
     timeout: 2000
   };
   if (exclusion) {
-    const exclusionText = await exclusion.textContent();
     options.mask = [exclusion];
   }
   return await page.screenshot(options)

package/standardize.js

@@ -400,6 +400,20 @@ const convert = (toolName, result, standardResult) => {
         console.log(`ERROR: Testaro rule ${rule} result has no standardInstances property`);
       }
     });
+    const preventionCount = result.preventions && result.preventions.length;
+    if (preventionCount) {
+      standardResult.totals[3] += preventionCount;
+      standardResult.instances.push({
+        ruleID: 'testPrevention',
+        what: 'Page prevented tests from being performed',
+        ordinalSeverity: 3,
+        count: preventionCount,
+        tagName: '',
+        id: '',
+        location: '',
+        excerpt: ''
+      });
+    }
     standardResult.totals = standardResult.totals.map(total => Math.round(total));
   }
   // wave

package/testaro/hover.js

@@ -2,7 +2,7 @@
   hover
   This test reports unexpected impacts of hovering on the visible page. Impacts are measured by
   pixel changes outside the hovered element and by unhoverability.
-  
+
   The elements that are subjected to hovering (called “triggers”) are the Playwright-visible
   elements that have 'A', 'BUTTON', or (if not with role=menuitem) 'LI' tag names or have
   'onmouseenter' or 'onmouseover' attributes.
@@ -24,8 +24,8 @@
   action on the location where the center of the element is, rather than some other element with a
   higher zIndex value in the same location being the target.
 
-  WARNING: This test uses the Playwright page.screenshot method, which is not implemented for the
-  firefox browser type.
+  WARNING: This test uses the procs/visChange module. See the warning in that module about browser
+  types.
 */
 
 // IMPORTS
@@ -54,7 +54,7 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
   const psRatio = Math.max(1, locsAll.length / sampleSize);
   // Get a sample of the triggers.
   const sampleIndexes = getSample(locsAll, sampleSize);
-  const sample = locsAll.filter((loc, index) => sampleIndexes.includes(index));
+  const sample = sampleIndexes.map(index => locsAll[index]);
   // For each trigger in the sample:
   for (const loc of sample) {
     // Hover over it and get the fractional pixel change.

package/testaro/motion.js

@@ -7,9 +7,9 @@
   test compares two screen shots of the viewport 2 seconds and 6 seconds after page load. It
   reports a rule violation if any pixels change. The larger the change fraction, the greater the
   ordinal severity.
-  
-  WARNING: This test uses the Playwright page.screenshot method, which is not implemented for the
-  firefox browser type.
+
+  WARNING: This test uses the procs/visChange module. See the warning in that module about browser
+  types.
 */
 
 // IMPORTS

package/tests/testaro.js

@@ -1,6 +1,6 @@
 /*
   testaro
-  This test implements the Testaro evaluative ruleset for accessibility.
+  This test implements the Testaro evaluative rules.
 */
 
 // CONSTANTS
@@ -54,7 +54,7 @@ const etcRules = {
 
 // FUNCTIONS
 
-// Conducts and reports a Testaro test.
+// Conducts and reports Testaro tests.
 exports.reporter = async (page, options) => {
   const {withItems, stopOnFail, args} = options;
   const argRules = args ? Object.keys(args) : null;
@@ -62,7 +62,8 @@ exports.reporter = async (page, options) => {
   // Initialize the data.
   const data = {
     rules: {},
-    preventions: []
+    preventions: [],
+    testTimes: {}
   };
   // If the rule specification is valid:
   if (
@@ -74,6 +75,7 @@ exports.reporter = async (page, options) => {
     const realRules = rules[0] === 'y'
       ? rules.slice(1)
       : Object.keys(evalRules).filter(ruleID => ! rules.slice(1).includes(ruleID));
+    const testTimes = [];
     for (const rule of realRules) {
       // Initialize an argument array.
       const ruleArgs = [page, withItems];
@@ -90,9 +92,15 @@ exports.reporter = async (page, options) => {
       data.rules[rule].what = what;
       console.log(`>>>>>> ${rule} (${what})`);
       try {
+        const startTime = Date.now();
         const report = await require(`../testaro/${rule}`).reporter(... ruleArgs);
+        const endTime = Date.now();
+        testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
         Object.keys(report).forEach(key => {
           data.rules[rule][key] = report[key];
+          if (report.prevented) {
+            data.preventions.push(rule);
+          }
         });
         // If testing is to stop after a failure and the page failed the test:
         if (stopOnFail && report.totals.some(total => total)) {
@@ -107,6 +115,9 @@ exports.reporter = async (page, options) => {
         console.log(`ERROR: Test of testaro rule ${rule} prevented (${error.message})`);
       }
     }
+    testTimes.sort((a, b) => b[1] - a[1]).forEach(pair => {
+      data.testTimes[pair[0]] = pair[1];
+    });
   }
   // Otherwise, i.e. if the rule specification is invalid:
   else {