testaro 4.10.3 → 4.11.1

package/commands.js

@@ -153,7 +153,7 @@ exports.commands = {
     axe: [
       'Perform an Axe test',
       {
-        withItems: [true, 'boolean'],
+        detailLevel: [true, 'number', '', '0 = least, 4 = most'],
         rules: [true, 'array', 'areStrings', 'rule names, or empty if all']
       }
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.10.3",
+  "version": "4.11.1",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -64,10 +64,25 @@ const tenonData = {
   accessToken: '',
   requestIDs: {}
 };
+// Keywords in log messages indicating errors.
+const errorWords = [
+  'failed',
+  'error',
+  'suspicious',
+  'refused',
+  'content security policy',
+  'unrecognized',
+  'requires',
+  'warning',
+  'missing',
+  'deprecated'
+];
 // ########## VARIABLES
 // Facts about the current session.
 let logCount = 0;
 let logSize = 0;
+let errorLogCount = 0;
+let errorLogSize = 0;
 let prohibitedCount = 0;
 let visitTimeoutCount = 0;
 let visitRejectionCount = 0;
@@ -253,12 +268,18 @@ const launch = async typeName => {
     browserContext = await browser.newContext(viewport);
     // When a page is added to the browser context:
     browserContext.on('page', page => {
-      // Make its console messages appear in the Playwright console.
+      // Make its console messages get reported and appear in the Playwright console.
       page.on('console', msg => {
         const msgText = msg.text();
         console.log(`[${msgText}]`);
+        const msgTextLC = msgText.toLowerCase();
+        const msgLength = msgText.length;
         logCount++;
-        logSize += msgText.length;
+        logSize += msgLength;
+        if (errorWords.some(word => msgTextLC.includes(word))) {
+          errorLogCount++;
+          errorLogSize += msgLength;
+        }
         const msgLC = msgText.toLowerCase();
         if (msgText.includes('403') && (msgLC.includes('status') || msgLC.includes('prohibited'))) {
           prohibitedCount++;
@@ -1224,7 +1245,13 @@ const doActs = async (report, actIndex, page) => {
 // Performs the commands in a script.
 const doScript = async (report) => {
   // Reinitialize the log statistics.
-  logCount = logSize = prohibitedCount = visitTimeoutCount = visitRejectionCount= 0;
+  logCount = 0;
+  logSize = 0;
+  errorLogCount = 0;
+  errorLogSize = 0;
+  prohibitedCount = 0;
+  visitTimeoutCount = 0;
+  visitRejectionCount = 0;
   // Add the start time to the report.
   const startTime = new Date();
   report.startTime = startTime.toISOString().slice(0, 19);
@@ -1239,6 +1266,8 @@ const doScript = async (report) => {
   // Add the log statistics to the report.
   report.logCount = logCount;
   report.logSize = logSize;
+  report.errorLogCount = errorLogCount;
+  report.errorLogSize = errorLogSize;
   report.prohibitedCount = prohibitedCount;
   report.visitTimeoutCount = visitTimeoutCount;
   report.visitRejectionCount = visitRejectionCount;

package/tests/axe.js

@@ -3,13 +3,26 @@
   This test implements the axe-core ruleset for accessibility.
 
   The rules argument defaults to all rules; otherwise, specify an array of rule names.
-  Experimental, needs-review, and best-practice rules are ignored.
+
+  The detailLevel argument specifies how many result categories are to be included in the
+  details. 0 = none; 1 = violations; 2 = violations and incomplete; 3 = violations, incomplete,
+  and passes; 4 = violations, incomplete, passes, and inapplicable. Regardless of the value of this
+  argument, Axe-core is instructed to report all nodes with violation or incomplete results, but only
+  1 node per rule found to be passed or inapplicable. Therefore, from the results of this test it
+  is possible to count the rules passed and the inapplicable rules, but not the nodes for which each
+  rule is passed or inapplicable. To count those nodes, one would need to revise the 'resultTypes'
+  property of the 'axeOptions' object.
+
+  The report of this test shows rule totals by result category and, within the violation and
+  incomplete categories, node totals by severity. It does not show rule or node totals by test
+  category (“tag”), such as 'wcag21aaa'. Scoring can consider test categories by getting the value
+  of the 'tags' property of each rule.
 */
 // IMPORTS
-const {injectAxe, getViolations} = require('axe-playwright');
+const {injectAxe, getAxeResults} = require('axe-playwright');
 // FUNCTIONS
 // Conducts and reports an Axe test.
-exports.reporter = async (page, withItems, rules = []) => {
+exports.reporter = async (page, detailLevel, rules = []) => {
   // Initialize the report.
   const data = {};
   // Inject axe-core into the page.
@@ -22,81 +35,65 @@ exports.reporter = async (page, withItems, rules = []) => {
   // If the injection succeeded:
   if (! data.prevented) {
     // Get the data on the elements violating the specified axe-core rules.
-    const axeOptions = {};
+    const axeOptions = {
+      resultTypes: ['violations', 'incomplete']
+    };
     if (rules.length) {
       axeOptions.runOnly = rules;
     }
-    const axeReport = await getViolations(page, null, axeOptions)
+    else {
+      axeOptions.runOnly = ['experimental', 'best-practice', 'wcag2a', 'wcag2aa', 'wcag2aaa', 'wcag21a', 'wcag21aa', 'wcag21aaa'];
+    }
+    const axeReport = await getAxeResults(page, null, axeOptions)
     .catch(error => {
       console.log(`ERROR: Axe failed (${error.message}'`);
       return '';
     });
     // If the test succeeded:
-    if (Array.isArray(axeReport)) {
-      // Initialize a report.
-      data.warnings = 0;
-      data.violations = {
-        minor: 0,
-        moderate: 0,
-        serious: 0,
-        critical: 0
+    const {inapplicable, passes, incomplete, violations} = axeReport;
+    if (violations) {
+      // Initialize the result.
+      data.totals = {
+        rulesNA: 0,
+        rulesPassed: 0,
+        rulesWarned: 0,
+        rulesViolated: 0,
+        warnings: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        },
+        violations: {
+          minor: 0,
+          moderate: 0,
+          serious: 0,
+          critical: 0
+        }
       };
-      if (withItems) {
-        data.items = [];
-      }
-      // If there were any violations:
-      if (axeReport.length) {
-        // FUNCTION DEFINITIONS START
-        // Compacts a check violation.
-        const compactCheck = checkObj => {
-          return {
-            check: checkObj.id,
-            description: checkObj.message,
-            impact: checkObj.impact
-          };
-        };
-        // Compacts a violating element.
-        const compactViolator = elObj => {
-          const out = {
-            selector: elObj.target[0],
-            impact: elObj.impact
-          };
-          if (elObj.any && elObj.any.length) {
-            out['must pass any of'] = elObj.any.map(checkObj => compactCheck(checkObj));
-          }
-          if (elObj.none && elObj.none.length) {
-            out['must pass all of'] = elObj.none.map(checkObj => compactCheck(checkObj));
-          }
-          return out;
-        };
-        // Compacts a violated rule.
-        const compactRule = rule => {
-          const out = {
-            rule: rule.id,
-            description: rule.description,
-            impact: rule.impact,
-            elements: {}
-          };
-          if (rule.nodes && rule.nodes.length) {
-            out.elements = rule.nodes.map(el => compactViolator(el));
-          }
-          return out;
-        };
-        // FUNCTION DEFINITIONS END
-        // For each rule violated:
-        axeReport.forEach(rule => {
-          // For each element violating the rule:
-          rule.nodes.forEach(element => {
-            // Increment the element count of the impact of its violation.
-            data.violations[element.impact]++;
-          });
-          // If details are required:
-          if (withItems) {
-            // Add it to the report.
-            data.items.push(compactRule(rule));
-          }
+      data.details = axeReport;
+      // Populate the totals.
+      const {totals} = data;
+      totals.rulesNA = inapplicable.length;
+      totals.rulesPassed = passes.length;
+      incomplete.forEach(rule => {
+        totals.rulesWarned++;
+        rule.nodes.forEach(node => {
+          totals.warnings[node.impact]++;
+        });
+      });
+      violations.forEach(rule => {
+        totals.rulesViolated++;
+        rule.nodes.forEach(node => {
+          totals.violations[node.impact]++;
         });
-      }
+      });
+      // Delete irrelevant properties from the report details.
+      const irrelevants = ['inapplicable', 'passes', 'incomplete', 'violations']
+      .slice(0, 4 - detailLevel);
+      irrelevants.forEach(irrelevant => {
+        delete axeReport[irrelevant];
+      });
     }
     // Otherwise, i.e. if the test failed:
     else {

package/tests/motion.js

@@ -9,7 +9,8 @@
   between screen shots (interval), and how many screen shots to make (count). The test compares the
   screen shots and reports 9 statistics:
     0. bytes: an array of the sizes of the screen shots, in bytes
-    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs of screen shots
+    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs
+      of screen shots
     2. meanLocalRatio: the mean of the ratios in the localRatios array
     3. maxLocalRatio: the greatest of the ratios in the localRatios array
     4. globalRatio: the ratio of bytes of the largest to the smallest screen shot
@@ -17,6 +18,10 @@
     6. meanPixelChange: the mean of the counts in the pixelChanges array
     7. maxPixelChange: the greatest of the counts in the pixelChanges array
     8. changeFrequency: what fraction of the adjacent pairs of screen shots has pixel differences
+  
+  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
+  when the browser type is chromium and is not implemented for the firefox browser type. The only
+  browser type usable with this test is webkit.
 */
 const pixelmatch = require('pixelmatch');
 const {PNG} = require('pngjs');