testaro 4.10.4 → 4.12.0

package/commands.js

@@ -153,7 +153,7 @@ exports.commands = {
     axe: [
       'Perform an Axe test',
       {
-        withItems: [true, 'boolean'],
+        detailLevel: [true, 'number', '', '0 = least, 4 = most'],
         rules: [true, 'array', 'areStrings', 'rule names, or empty if all']
       }
     ],
@@ -206,7 +206,7 @@ exports.commands = {
       }
     ],
     menuNav: [
-      'Perform a tabNav test',
+      'Perform a menuNav test',
       {
         withItems: [true, 'boolean']
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.10.4",
+  "version": "4.12.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/tests/aatt.js

@@ -65,9 +65,12 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
       const issueArray = JSON.parse(reportJSON);
       // Remove the notices from the array.
       const nonNotices = issueArray.filter(issue => issue.type !== 'notice');
-      // Convert the technique property from a string to an array of strings.
+      // For each non-notice issue:
+      let errors = 0;
+      let warnings = 0;
       nonNotices.forEach(issue => {
         if (issue.type) {
+          // Convert the technique property from a string to an array of strings.
           const longTech = issue.techniques;
           issue.techniques = longTech.replace(/a><a/g, 'a>%<a').split('%');
           issue.id = issue
@@ -75,12 +78,24 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
           .map(technique => technique.replace(/^.+?>|<\/a>$/g, ''))
           .sort()
           .join('+');
+          // Add the issue to the totals.
+          if (issue.type === 'error') {
+            errors++;
+          }
+          else if (issue.type === 'warning') {
+            warnings++;
+          }
         }
       });
+      // Return the result.
       return {
         result: {
+          totals: {
+            errors,
+            warnings
+          },
           report: nonNotices,
-          triesLeft
+          preventionCount: tryLimit - triesLeft - 1
         }
       };
     }

package/tests/alfa.js

@@ -44,7 +44,13 @@ exports.reporter = async page => {
       });
       await rulePage.close();
     }
-    const data = [];
+    const data = {
+      totals: {
+        failures: 0,
+        warnings: 0
+      },
+      items: []
+    };
     await Scraper.with(async scraper => {
       for (const input of await scraper.scrape(page.url())) {
         const audit = Audit.of(input, alfaRules.default);
@@ -96,7 +102,13 @@ exports.reporter = async page => {
               if (etcTags.length) {
                 outcomeData.etcTags = etcTags;
               }
-              data.push(outcomeData);
+              if (outcomeData.verdict === 'failed') {
+                data.totals.failures++;
+              }
+              else if (outcomeData.verdict === 'cantTell') {
+                data.totals.warnings++;
+              }
+              data.items.push(outcomeData);
             }
           }
         });

package/tests/axe.js

@@ -3,13 +3,26 @@
   This test implements the axe-core ruleset for accessibility.
 
   The rules argument defaults to all rules; otherwise, specify an array of rule names.
-  Experimental, needs-review, and best-practice rules are ignored.
+
+  The detailLevel argument specifies how many result categories are to be included in the
+  details. 0 = none; 1 = violations; 2 = violations and incomplete; 3 = violations, incomplete,
+  and passes; 4 = violations, incomplete, passes, and inapplicable. Regardless of the value of this
+  argument, Axe-core is instructed to report all nodes with violation or incomplete results, but only
+  1 node per rule found to be passed or inapplicable. Therefore, from the results of this test it
+  is possible to count the rules passed and the inapplicable rules, but not the nodes for which each
+  rule is passed or inapplicable. To count those nodes, one would need to revise the 'resultTypes'
+  property of the 'axeOptions' object.
+
+  The report of this test shows rule totals by result category and, within the violation and
+  incomplete categories, node totals by severity. It does not show rule or node totals by test
+  category (“tag”), such as 'wcag21aaa'. Scoring can consider test categories by getting the value
+  of the 'tags' property of each rule.
 */
 // IMPORTS
-const {injectAxe, getViolations} = require('axe-playwright');
+const {injectAxe, getAxeResults} = require('axe-playwright');
 // FUNCTIONS
 // Conducts and reports an Axe test.
-exports.reporter = async (page, withItems, rules = []) => {
+exports.reporter = async (page, detailLevel, rules = []) => {
   // Initialize the report.
   const data = {};
   // Inject axe-core into the page.
@@ -22,81 +35,65 @@ exports.reporter = async (page, withItems, rules = []) => {
   // If the injection succeeded:
   if (! data.prevented) {
     // Get the data on the elements violating the specified axe-core rules.
-    const axeOptions = {};
+    const axeOptions = {
+      resultTypes: ['violations', 'incomplete']
+    };
     if (rules.length) {
       axeOptions.runOnly = rules;
     }
-    const axeReport = await getViolations(page, null, axeOptions)
+    else {
+      axeOptions.runOnly = ['experimental', 'best-practice', 'wcag2a', 'wcag2aa', 'wcag2aaa', 'wcag21a', 'wcag21aa', 'wcag21aaa'];
+    }
+    const axeReport = await getAxeResults(page, null, axeOptions)
     .catch(error => {
       console.log(`ERROR: Axe failed (${error.message}'`);
       return '';
     });
     // If the test succeeded:
-    if (Array.isArray(axeReport)) {
-      // Initialize a report.
-      data.warnings = 0;
-      data.violations = {
-        minor: 0,
-        moderate: 0,
-        serious: 0,
-        critical: 0
+    const {inapplicable, passes, incomplete, violations} = axeReport;
+    if (violations) {
+      // Initialize the result.
+      data.totals = {
+        rulesNA: 0,
+        rulesPassed: 0,
+        rulesWarned: 0,
+        rulesViolated: 0,
+        warnings: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        },
+        violations: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        }
       };
-      if (withItems) {
-        data.items = [];
-      }
-      // If there were any violations:
-      if (axeReport.length) {
-        // FUNCTION DEFINITIONS START
-        // Compacts a check violation.
-        const compactCheck = checkObj => {
-          return {
-            check: checkObj.id,
-            description: checkObj.message,
-            impact: checkObj.impact
-          };
-        };
-        // Compacts a violating element.
-        const compactViolator = elObj => {
-          const out = {
-            selector: elObj.target[0],
-            impact: elObj.impact
-          };
-          if (elObj.any && elObj.any.length) {
-            out['must pass any of'] = elObj.any.map(checkObj => compactCheck(checkObj));
-          }
-          if (elObj.none && elObj.none.length) {
-            out['must pass all of'] = elObj.none.map(checkObj => compactCheck(checkObj));
-          }
-          return out;
-        };
-        // Compacts a violated rule.
-        const compactRule = rule => {
-          const out = {
-            rule: rule.id,
-            description: rule.description,
-            impact: rule.impact,
-            elements: {}
-          };
-          if (rule.nodes && rule.nodes.length) {
-            out.elements = rule.nodes.map(el => compactViolator(el));
-          }
-          return out;
-        };
-        // FUNCTION DEFINITIONS END
-        // For each rule violated:
-        axeReport.forEach(rule => {
-          // For each element violating the rule:
-          rule.nodes.forEach(element => {
-            // Increment the element count of the impact of its violation.
-            data.violations[element.impact]++;
-          });
-          // If details are required:
-          if (withItems) {
-            // Add it to the report.
-            data.items.push(compactRule(rule));
-          }
+      data.details = axeReport;
+      // Populate the totals.
+      const {totals} = data;
+      totals.rulesNA = inapplicable.length;
+      totals.rulesPassed = passes.length;
+      incomplete.forEach(rule => {
+        totals.rulesWarned++;
+        rule.nodes.forEach(node => {
+          totals.warnings[node.impact]++;
+        });
+      });
+      violations.forEach(rule => {
+        totals.rulesViolated++;
+        rule.nodes.forEach(node => {
+          totals.violations[node.impact]++;
         });
-      }
+      });
+      // Delete irrelevant properties from the report details.
+      const irrelevants = ['inapplicable', 'passes', 'incomplete', 'violations']
+      .slice(0, 4 - detailLevel);
+      irrelevants.forEach(irrelevant => {
+        delete axeReport[irrelevant];
+      });
     }
     // Otherwise, i.e. if the test failed:
     else {

package/tests/focAll.js

@@ -1,17 +1,12 @@
 /*
   focAll
-  This test reports discrepancies between focusable and Tab-focused element counts.
-  The test first counts all the visible focusable (i.e. with tabIndex 0) elements
-  (except counting each group of radio buttons as only one focusable element). Then
-  it repeatedly presses the Tab (or Option-Tab in webkit) key until it has reached
-  all the elements it can and counts those elements. If the latter are more than the
-  former, Tab-key navigation made more focusable elements visible. If the latter are
-  fewer than the former, the page manages focus so as to prevent Tab-key navigation
-  from reaching all focusable elements. Either of these can complicate navigation for
-  users. It may disappoint the expectation that the content will remain stable as they
-  move with the Tab key, or the expectation that they can reach every focusable element
-  (or widget, such as one radio button or tab in each group) merely by pressing the Tab
-  key.
+  This test reports discrepancies between focusable and Tab-focused element counts. The test first
+  counts all the visible focusable (i.e. with tabIndex 0) elements (except counting each group of
+  radio buttons as only one focusable element). Then it repeatedly presses the Tab (or Option-Tab
+  in webkit) key until it has reached all the elements it can and counts those elements. If the
+  two counts differ, navigation can be made more difficult. The cause may be surprising changes in
+  content during navigation with the Tab key, or inability to reach every focusable element (or
+  widget, such as one radio button or tab in each group) merely by pressing the Tab key.
 */
 exports.reporter = async page => {
   // Identify the count of visible focusable elements.

package/tests/focInd.js

@@ -5,10 +5,13 @@
   line thickness and non-transparent color. The test is based on the assumption that outlines are
   the standard and thus most familiar focus indicator. Other focus indicators are assumed better
   than none, but more likely to be misunderstood. For example, underlines may be mistaken for
-  selection indicators. Some pages delay the appearance of focus indicators. This test waits for
-  focus indicators to appear if specified and, if there is a delay, reports on its magnitude.
+  selection indicators. Some pages delay the appearance of focus indicators. If a wait is
+  specified, the test checks every 100 ms for an outline until the allow wait time expires,
+  and once more after it expires. If no outline appears by then, the test checks for other focus
+  indicators. If an outline does not appear immediately but appears on a subsequent check, the test
+  reports the amount of the delay.
 
-  Bug: This test fails to recognize outlines when run with firefox.
+  WARNING: This test fails to recognize outlines when run with firefox.
 */
 exports.reporter = async (page, revealAll, allowedDelay, withItems) => {
   // If required, make all elements visible.

package/tests/motion.js

@@ -9,7 +9,8 @@
   between screen shots (interval), and how many screen shots to make (count). The test compares the
   screen shots and reports 9 statistics:
     0. bytes: an array of the sizes of the screen shots, in bytes
-    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs of screen shots
+    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs
+      of screen shots
     2. meanLocalRatio: the mean of the ratios in the localRatios array
     3. maxLocalRatio: the greatest of the ratios in the localRatios array
     4. globalRatio: the ratio of bytes of the largest to the smallest screen shot
@@ -17,6 +18,10 @@
     6. meanPixelChange: the mean of the counts in the pixelChanges array
     7. maxPixelChange: the greatest of the counts in the pixelChanges array
     8. changeFrequency: what fraction of the adjacent pairs of screen shots has pixel differences
+  
+  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
+  when the browser type is chromium and is not implemented for the firefox browser type. The only
+  browser type usable with this test is webkit.
 */
 const pixelmatch = require('pixelmatch');
 const {PNG} = require('pngjs');

package/tests/tabNav.js

@@ -1,6 +1,6 @@
 /*
   tabNav
-  This test reports nonstandard keyboard navigation among tab elements in tab lists.
+  This test reports nonstandard keyboard navigation among tab elements in visible tab lists.
   Standards are based on https://www.w3.org/TR/wai-aria-practices-1.1/#tabpanel.
 */
 
@@ -73,8 +73,8 @@ exports.reporter = async (page, withItems) => {
       correct: []
     };
   }
-  // Identify an array of the tablists.
-  const tabLists = await page.$$('[role=tablist]');
+  // Identify an array of the visible tablists.
+  const tabLists = await page.$$('[role=tablist]:visible');
   if (tabLists.length) {
     // FUNCTION DEFINITIONS START
     // Returns text associated with an element.
@@ -94,22 +94,34 @@ exports.reporter = async (page, withItems) => {
     ) => {
       let pressed = true;
       // Click the tab element, to make the focus on it effective.
-      await tabElement.click({timeout: 1500})
+      await tabElement.click({
+        timeout: 500
+      })
+      .catch(async error => {
+        await tabElement.click({
+          force: true
+        });
+      })
       .catch(error => {
         console.log(`ERROR: could not click tab element ${itemData.text} (${error.message})`);
-        pressed = false;
+      pressed = false;
       });
+      // Increment the counts of navigations and key navigations.
+      data.totals.navigations.all.total++;
+      data.totals.navigations.specific[keyProp].total++;
+      const {navigationErrors} = itemData;
+      // If the click succeeded:
       if (pressed) {
         // Refocus the tab element and press the specified key (page.keyboard.press may fail).
-        await tabElement.press(keyName)
+        await tabElement.press(keyName, {
+          timeout: 1000
+        })
         .catch(error => {
           console.log(`ERROR: could not press ${keyName} (${error.message})`);
           pressed = false;
         });
+        // If the refocus and keypress succeeded:
         if (pressed) {
-          // Increment the counts of navigations and key navigations.
-          data.totals.navigations.all.total++;
-          data.totals.navigations.specific[keyProp].total++;
           // Identify which tab element is now focused, if any.
           const focusIndex = await focusedTab(tabs);
           // If the focus is correct:
@@ -128,15 +140,36 @@ exports.reporter = async (page, withItems) => {
             // If itemization is required:
             if (withItems) {
               // Update the element report.
-              itemData.navigationErrors.push(keyName);
+              navigationErrors.push(keyName);
             }
           }
           return elementIsCorrect;
         }
+        // Otherwise, i.e. if the refocus or keypress failed:
         else {
+          // Increment the counts of incorrect navigations and incorrect key navigations.
+          data.totals.navigations.all.incorrect++;
+          data.totals.navigations.specific[keyProp].incorrect++;
+          // If itemization is required and a focus failure has not yet been reported:
+          if (withItems && ! navigationErrors.includes('focus')) {
+            // Update the element report.
+            navigationErrors.push('focus');
+          }
           return false;
         }
       }
+      // Otherwise, i.e. if the click failed:
+      else {
+        // Increment the counts of incorrect navigations and incorrect key navigations.
+        data.totals.navigations.all.incorrect++;
+        data.totals.navigations.specific[keyProp].incorrect++;
+        // If itemization is required and a click failure has not yet been reported:
+        if (withItems && ! navigationErrors.includes('click')) {
+          // Update the element report.
+          navigationErrors.push('click');
+        }
+        return false;
+      }
     };
     // Returns the index to which an arrow key should move the focus.
     const arrowTarget = (startIndex, tabCount, orientation, direction) => {