testaro 4.10.2 → 4.11.0

package/commands.js

@@ -153,7 +153,7 @@ exports.commands = {
     axe: [
       'Perform an Axe test',
       {
-        withItems: [true, 'boolean'],
+        detailLevel: [true, 'number', '', 'count to include of: violations, incomplete, passes, inapplicable'],
         rules: [true, 'array', 'areStrings', 'rule names, or empty if all']
       }
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.10.2",
+  "version": "4.11.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -64,10 +64,25 @@ const tenonData = {
   accessToken: '',
   requestIDs: {}
 };
+// Keywords in log messages indicating errors.
+const errorWords = [
+  'failed',
+  'error',
+  'suspicious',
+  'refused',
+  'content security policy',
+  'unrecognized',
+  'requires',
+  'warning',
+  'missing',
+  'deprecated'
+];
 // ########## VARIABLES
 // Facts about the current session.
 let logCount = 0;
 let logSize = 0;
+let errorLogCount = 0;
+let errorLogSize = 0;
 let prohibitedCount = 0;
 let visitTimeoutCount = 0;
 let visitRejectionCount = 0;
@@ -253,12 +268,18 @@ const launch = async typeName => {
     browserContext = await browser.newContext(viewport);
     // When a page is added to the browser context:
     browserContext.on('page', page => {
-      // Make its console messages appear in the Playwright console.
+      // Make its console messages get reported and appear in the Playwright console.
       page.on('console', msg => {
         const msgText = msg.text();
         console.log(`[${msgText}]`);
+        const msgTextLC = msgText.toLowerCase();
+        const msgLength = msgText.length;
         logCount++;
-        logSize += msgText.length;
+        logSize += msgLength;
+        if (errorWords.some(word => msgTextLC.includes(word))) {
+          errorLogCount++;
+          errorLogSize += msgLength;
+        }
         const msgLC = msgText.toLowerCase();
         if (msgText.includes('403') && (msgLC.includes('status') || msgLC.includes('prohibited'))) {
           prohibitedCount++;
@@ -1224,7 +1245,13 @@ const doActs = async (report, actIndex, page) => {
 // Performs the commands in a script.
 const doScript = async (report) => {
   // Reinitialize the log statistics.
-  logCount = logSize = prohibitedCount = visitTimeoutCount = visitRejectionCount= 0;
+  logCount = 0;
+  logSize = 0;
+  errorLogCount = 0;
+  errorLogSize = 0;
+  prohibitedCount = 0;
+  visitTimeoutCount = 0;
+  visitRejectionCount = 0;
   // Add the start time to the report.
   const startTime = new Date();
   report.startTime = startTime.toISOString().slice(0, 19);
@@ -1239,6 +1266,8 @@ const doScript = async (report) => {
   // Add the log statistics to the report.
   report.logCount = logCount;
   report.logSize = logSize;
+  report.errorLogCount = errorLogCount;
+  report.errorLogSize = errorLogSize;
   report.prohibitedCount = prohibitedCount;
   report.visitTimeoutCount = visitTimeoutCount;
   report.visitRejectionCount = visitRejectionCount;

package/tests/axe.js

@@ -3,13 +3,26 @@
   This test implements the axe-core ruleset for accessibility.
 
   The rules argument defaults to all rules; otherwise, specify an array of rule names.
-  Experimental, needs-review, and best-practice rules are ignored.
+
+  The detailLevel argument specifies how many result categories are to be included in the
+  details. 0 = none; 1 = violations; 2 = violations and incomplete; 3 = violations, incomplete,
+  and passes; 4 = violations, incomplete, passes, and inapplicable. Regardless of the value of this
+  argument, Axe-core is instructed to report all nodes with violation or incomplete results, but only
+  1 node per rule found to be passed or inapplicable. Therefore, from the results of this test it
+  is possible to count the rules passed and the inapplicable rules, but not the nodes for which each
+  rule is passed or inapplicable. To count those nodes, one would need to revise the 'resultTypes'
+  property of the 'axeOptions' object.
+
+  The report of this test shows rule totals by result category and, within the violation and
+  incomplete categories, node totals by severity. It does not show rule or node totals by test
+  category (“tag”), such as 'wcag21aaa'. Scoring can consider test categories by getting the value
+  of the 'tags' property of each rule.
 */
 // IMPORTS
-const {injectAxe, getViolations} = require('axe-playwright');
+const {injectAxe, getAxeResults} = require('axe-playwright');
 // FUNCTIONS
 // Conducts and reports an Axe test.
-exports.reporter = async (page, withItems, rules = []) => {
+exports.reporter = async (page, detailLevel, rules = []) => {
   // Initialize the report.
   const data = {};
   // Inject axe-core into the page.
@@ -22,81 +35,65 @@ exports.reporter = async (page, withItems, rules = []) => {
   // If the injection succeeded:
   if (! data.prevented) {
     // Get the data on the elements violating the specified axe-core rules.
-    const axeOptions = {};
+    const axeOptions = {
+      resultTypes: ['violations', 'incomplete']
+    };
     if (rules.length) {
       axeOptions.runOnly = rules;
     }
-    const axeReport = await getViolations(page, null, axeOptions)
+    else {
+      axeOptions.runOnly = ['experimental', 'best-practice', 'wcag2a', 'wcag2aa', 'wcag2aaa', 'wcag21a', 'wcag21aa', 'wcag21aaa'];
+    }
+    const axeReport = await getAxeResults(page, null, axeOptions)
     .catch(error => {
       console.log(`ERROR: Axe failed (${error.message}'`);
       return '';
     });
     // If the test succeeded:
-    if (Array.isArray(axeReport)) {
-      // Initialize a report.
-      data.warnings = 0;
-      data.violations = {
-        minor: 0,
-        moderate: 0,
-        serious: 0,
-        critical: 0
+    const {inapplicable, passes, incomplete, violations} = axeReport;
+    if (violations) {
+      // Initialize the result.
+      data.totals = {
+        rulesNA: 0,
+        rulesPassed: 0,
+        rulesWarned: 0,
+        rulesViolated: 0,
+        warnings: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        },
+        violations: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        }
       };
-      if (withItems) {
-        data.items = [];
-      }
-      // If there were any violations:
-      if (axeReport.length) {
-        // FUNCTION DEFINITIONS START
-        // Compacts a check violation.
-        const compactCheck = checkObj => {
-          return {
-            check: checkObj.id,
-            description: checkObj.message,
-            impact: checkObj.impact
-          };
-        };
-        // Compacts a violating element.
-        const compactViolator = elObj => {
-          const out = {
-            selector: elObj.target[0],
-            impact: elObj.impact
-          };
-          if (elObj.any && elObj.any.length) {
-            out['must pass any of'] = elObj.any.map(checkObj => compactCheck(checkObj));
-          }
-          if (elObj.none && elObj.none.length) {
-            out['must pass all of'] = elObj.none.map(checkObj => compactCheck(checkObj));
-          }
-          return out;
-        };
-        // Compacts a violated rule.
-        const compactRule = rule => {
-          const out = {
-            rule: rule.id,
-            description: rule.description,
-            impact: rule.impact,
-            elements: {}
-          };
-          if (rule.nodes && rule.nodes.length) {
-            out.elements = rule.nodes.map(el => compactViolator(el));
-          }
-          return out;
-        };
-        // FUNCTION DEFINITIONS END
-        // For each rule violated:
-        axeReport.forEach(rule => {
-          // For each element violating the rule:
-          rule.nodes.forEach(element => {
-            // Increment the element count of the impact of its violation.
-            data.violations[element.impact]++;
-          });
-          // If details are required:
-          if (withItems) {
-            // Add it to the report.
-            data.items.push(compactRule(rule));
-          }
+      data.details = axeReport;
+      // Populate the totals.
+      const {totals} = data;
+      totals.rulesNA = inapplicable.length;
+      totals.rulesPassed = passes.length;
+      incomplete.forEach(rule => {
+        totals.rulesWarned++;
+        rule.nodes.forEach(node => {
+          totals.warnings[node.impact]++;
+        });
+      });
+      violations.forEach(rule => {
+        totals.rulesViolated++;
+        rule.nodes.forEach(node => {
+          totals.violations[node.impact]++;
         });
-      }
+      });
+      // Delete irrelevant properties from the report details.
+      const irrelevants = ['inapplicable', 'passes', 'incomplete', 'violations']
+      .slice(0, 4 - detailLevel);
+      irrelevants.forEach(irrelevant => {
+        delete axeReport[irrelevant];
+      });
     }
     // Otherwise, i.e. if the test failed:
     else {

package/tests/focOp.js

@@ -1,5 +1,10 @@
 /*
   focOp
+
+  WARNING: The chromium and firefox browsers in Playwright make errors on this test by
+  misclassifying the cursor property values of the computed styles of elements. Launch the
+  webkit browser to run this test.
+
   This test reports descrepancies between Tab-focusability and operability. The standard
   practice is to make focusable elements operable and vice versa. If focusable elements are not
   operable, users are likely to be surprised that nothing happens when they try to operate such