testaro 4.11.1 → 4.12.2

package/aceconfig.js

@@ -2,5 +2,5 @@ module.exports = {
   reportLevels: [
     'violation',
     'recommendation'
-  ],
+  ]
 };

package/commands.js

@@ -206,7 +206,7 @@ exports.commands = {
       }
     ],
     menuNav: [
-      'Perform a tabNav test',
+      'Perform a menuNav test',
       {
         withItems: [true, 'boolean']
       }

package/high.js

@@ -1,7 +1,7 @@
 /*
   high.js
   Invokes Testaro with the high-level method.
-  Usage example: node high tp10 weborgs
+  Usage example: node high tp11 weborgs
 */
 
 const {runJob} = require('./create');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.11.1",
+  "version": "4.12.2",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -2,12 +2,16 @@
   index.js
   testaro main script.
 */
+
 // ########## IMPORTS
+
 // Module to keep secrets.
 require('dotenv').config();
 // Requirements for commands.
 const {commands} = require('./commands');
+
 // ########## CONSTANTS
+
 // Set DEBUG environment variable to 'true' to add debugging features.
 const debug = process.env.DEBUG === 'true';
 // Set WAITS environment variable to a positive number to insert delays (in ms).
@@ -77,7 +81,9 @@ const errorWords = [
   'missing',
   'deprecated'
 ];
+
 // ########## VARIABLES
+
 // Facts about the current session.
 let logCount = 0;
 let logSize = 0;
@@ -91,7 +97,11 @@ let actCount = 0;
 let browserContext;
 let browserTypeName;
 let requestedURL = '';
+// All browsers launched.
+let browsers = [];
+
 // ########## VALIDATORS
+
 // Validates a browser type.
 const isBrowserType = type => ['chromium', 'firefox', 'webkit'].includes(type);
 // Validates a load state.
@@ -234,12 +244,17 @@ const isValidReport = async report => {
     return false;
   }
 };
+
 // ########## OTHER FUNCTIONS
-// Closes any existing browser.
-const closeBrowser = async () => {
-  const browser = browserContext && browserContext.browser();
-  if (browser) {
-    await browser.close();
+
+// Closes all existing browsers.
+const closeBrowsers = async () => {
+  for (const browserObj of browsers) {
+    const {browser, typeName} = browserObj;
+    if (browser) {
+      console.log(`Closing ${typeName} browser, version ${browser.version()}`);
+      await browser.close();
+    }
   }
 };
 // Launches a browser.
@@ -248,7 +263,7 @@ const launch = async typeName => {
   // If the specified browser type exists:
   if (browserType) {
     // Close any existing browser.
-    await closeBrowser();
+    await closeBrowsers();
     // Launch a browser of the specified type.
     const browserOptions = {};
     if (debug) {
@@ -258,6 +273,11 @@ const launch = async typeName => {
       browserOptions.slowMo = waits;
     }
     const browser = await browserType.launch(browserOptions);
+    // Register it.
+    browsers.push({
+      browser,
+      typeName
+    });
     // Create a new context (window) in it, taller if debugging is on.
     const viewport = debug ? {
       viewPort: {
@@ -1261,8 +1281,8 @@ const doScript = async (report) => {
   report.testTimes = [];
   // Perform the specified acts.
   await doActs(report, 0, null);
-  // Close the browser.
-  await closeBrowser();
+  // Close all browsers.
+  await closeBrowsers();
   // Add the log statistics to the report.
   report.logCount = logCount;
   report.logSize = logSize;

package/tests/aatt.js

@@ -65,9 +65,12 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
       const issueArray = JSON.parse(reportJSON);
       // Remove the notices from the array.
       const nonNotices = issueArray.filter(issue => issue.type !== 'notice');
-      // Convert the technique property from a string to an array of strings.
+      // For each non-notice issue:
+      let errors = 0;
+      let warnings = 0;
       nonNotices.forEach(issue => {
         if (issue.type) {
+          // Convert the technique property from a string to an array of strings.
           const longTech = issue.techniques;
           issue.techniques = longTech.replace(/a><a/g, 'a>%<a').split('%');
           issue.id = issue
@@ -75,12 +78,24 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
           .map(technique => technique.replace(/^.+?>|<\/a>$/g, ''))
           .sort()
           .join('+');
+          // Add the issue to the totals.
+          if (issue.type === 'error') {
+            errors++;
+          }
+          else if (issue.type === 'warning') {
+            warnings++;
+          }
         }
       });
+      // Return the result.
       return {
         result: {
+          totals: {
+            errors,
+            warnings
+          },
           report: nonNotices,
-          triesLeft
+          preventionCount: tryLimit - triesLeft - 1
         }
       };
     }

package/tests/alfa.js

@@ -44,7 +44,13 @@ exports.reporter = async page => {
       });
       await rulePage.close();
     }
-    const data = [];
+    const data = {
+      totals: {
+        failures: 0,
+        warnings: 0
+      },
+      items: []
+    };
     await Scraper.with(async scraper => {
       for (const input of await scraper.scrape(page.url())) {
         const audit = Audit.of(input, alfaRules.default);
@@ -96,7 +102,13 @@ exports.reporter = async page => {
               if (etcTags.length) {
                 outcomeData.etcTags = etcTags;
               }
-              data.push(outcomeData);
+              if (outcomeData.verdict === 'failed') {
+                data.totals.failures++;
+              }
+              else if (outcomeData.verdict === 'cantTell') {
+                data.totals.warnings++;
+              }
+              data.items.push(outcomeData);
             }
           }
         });

package/tests/focAll.js

@@ -1,17 +1,12 @@
 /*
   focAll
-  This test reports discrepancies between focusable and Tab-focused element counts.
-  The test first counts all the visible focusable (i.e. with tabIndex 0) elements
-  (except counting each group of radio buttons as only one focusable element). Then
-  it repeatedly presses the Tab (or Option-Tab in webkit) key until it has reached
-  all the elements it can and counts those elements. If the latter are more than the
-  former, Tab-key navigation made more focusable elements visible. If the latter are
-  fewer than the former, the page manages focus so as to prevent Tab-key navigation
-  from reaching all focusable elements. Either of these can complicate navigation for
-  users. It may disappoint the expectation that the content will remain stable as they
-  move with the Tab key, or the expectation that they can reach every focusable element
-  (or widget, such as one radio button or tab in each group) merely by pressing the Tab
-  key.
+  This test reports discrepancies between focusable and Tab-focused element counts. The test first
+  counts all the visible focusable (i.e. with tabIndex 0) elements (except counting each group of
+  radio buttons as only one focusable element). Then it repeatedly presses the Tab (or Option-Tab
+  in webkit) key until it has reached all the elements it can and counts those elements. If the
+  two counts differ, navigation can be made more difficult. The cause may be surprising changes in
+  content during navigation with the Tab key, or inability to reach every focusable element (or
+  widget, such as one radio button or tab in each group) merely by pressing the Tab key.
 */
 exports.reporter = async page => {
   // Identify the count of visible focusable elements.

package/tests/focInd.js

@@ -5,10 +5,13 @@
   line thickness and non-transparent color. The test is based on the assumption that outlines are
   the standard and thus most familiar focus indicator. Other focus indicators are assumed better
   than none, but more likely to be misunderstood. For example, underlines may be mistaken for
-  selection indicators. Some pages delay the appearance of focus indicators. This test waits for
-  focus indicators to appear if specified and, if there is a delay, reports on its magnitude.
+  selection indicators. Some pages delay the appearance of focus indicators. If a wait is
+  specified, the test checks every 100 ms for an outline until the allow wait time expires,
+  and once more after it expires. If no outline appears by then, the test checks for other focus
+  indicators. If an outline does not appear immediately but appears on a subsequent check, the test
+  reports the amount of the delay.
 
-  Bug: This test fails to recognize outlines when run with firefox.
+  WARNING: This test fails to recognize outlines when run with firefox.
 */
 exports.reporter = async (page, revealAll, allowedDelay, withItems) => {
   // If required, make all elements visible.

package/tests/ibm.js

@@ -19,12 +19,13 @@
 */
 // Import required modules.
 const fs = require('fs').promises;
-const {getCompliance} = require('accessibility-checker');
+const {getCompliance, close} = require('accessibility-checker');
 // Runs the IBM test.
 const run = async content => {
   const nowLabel = (new Date()).toISOString().slice(0, 19);
   // Return the result of a test.
   const ibmReport = await getCompliance(content, nowLabel);
+  await close();
   return ibmReport;
 };
 // Trims an IBM report.
@@ -67,11 +68,11 @@ const doTest = async (content, withItems, timeLimit) => {
       resolve({});
     }, 1000 * timeLimit);
   });
-  // Conduct and report the test.
+  // Conduct the test and get a Promise of the report.
   const ibmReport = run(content);
-  // Wait for the report until the time limit expires.
+  // Wait for completion or until the time limit expires.
   const ibmReportIfFast = await Promise.race([ibmReport, wait]);
-  // Delete the report files.
+  // Delete existing report files.
   try {
     const reportNames = await fs.readdir('results');
     for (const reportName of reportNames) {
@@ -88,7 +89,6 @@ const doTest = async (content, withItems, timeLimit) => {
     return ibmTypeReport;
   }
   else {
-    console.log('ERROR: getting ibm test report took too long');
     return {
       prevented: true,
       error: 'ERROR: getting ibm test report took too long'
@@ -105,6 +105,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
     result.content = await doTest(typeContent, withItems, timeLimit);
     if (result.content.prevented) {
       result.prevented = true;
+      console.log('ERROR: Getting ibm test report from page took too long');
     }
   }
   // If a test with new content is to be performed:
@@ -112,8 +113,9 @@ exports.reporter = async (page, withItems, withNewContent) => {
     const timeLimit = 20;
     const typeContent = page.url();
     result.url = await doTest(typeContent, withItems, timeLimit);
-    if (result.content.prevented) {
+    if (result.url.prevented) {
       result.prevented = true;
+      console.log('ERROR: Getting ibm test report from URL took too long');
     }
   }
   return {result};

package/tests/tabNav.js

@@ -1,6 +1,6 @@
 /*
   tabNav
-  This test reports nonstandard keyboard navigation among tab elements in tab lists.
+  This test reports nonstandard keyboard navigation among tab elements in visible tab lists.
   Standards are based on https://www.w3.org/TR/wai-aria-practices-1.1/#tabpanel.
 */
 
@@ -73,8 +73,8 @@ exports.reporter = async (page, withItems) => {
       correct: []
     };
   }
-  // Identify an array of the tablists.
-  const tabLists = await page.$$('[role=tablist]');
+  // Identify an array of the visible tablists.
+  const tabLists = await page.$$('[role=tablist]:visible');
   if (tabLists.length) {
     // FUNCTION DEFINITIONS START
     // Returns text associated with an element.
@@ -94,22 +94,34 @@ exports.reporter = async (page, withItems) => {
     ) => {
       let pressed = true;
       // Click the tab element, to make the focus on it effective.
-      await tabElement.click({timeout: 1500})
+      await tabElement.click({
+        timeout: 500
+      })
+      .catch(async error => {
+        await tabElement.click({
+          force: true
+        });
+      })
       .catch(error => {
         console.log(`ERROR: could not click tab element ${itemData.text} (${error.message})`);
-        pressed = false;
+      pressed = false;
       });
+      // Increment the counts of navigations and key navigations.
+      data.totals.navigations.all.total++;
+      data.totals.navigations.specific[keyProp].total++;
+      const {navigationErrors} = itemData;
+      // If the click succeeded:
       if (pressed) {
         // Refocus the tab element and press the specified key (page.keyboard.press may fail).
-        await tabElement.press(keyName)
+        await tabElement.press(keyName, {
+          timeout: 1000
+        })
         .catch(error => {
           console.log(`ERROR: could not press ${keyName} (${error.message})`);
           pressed = false;
         });
+        // If the refocus and keypress succeeded:
         if (pressed) {
-          // Increment the counts of navigations and key navigations.
-          data.totals.navigations.all.total++;
-          data.totals.navigations.specific[keyProp].total++;
           // Identify which tab element is now focused, if any.
           const focusIndex = await focusedTab(tabs);
           // If the focus is correct:
@@ -128,15 +140,36 @@ exports.reporter = async (page, withItems) => {
             // If itemization is required:
             if (withItems) {
               // Update the element report.
-              itemData.navigationErrors.push(keyName);
+              navigationErrors.push(keyName);
             }
           }
           return elementIsCorrect;
         }
+        // Otherwise, i.e. if the refocus or keypress failed:
         else {
+          // Increment the counts of incorrect navigations and incorrect key navigations.
+          data.totals.navigations.all.incorrect++;
+          data.totals.navigations.specific[keyProp].incorrect++;
+          // If itemization is required and a focus failure has not yet been reported:
+          if (withItems && ! navigationErrors.includes('focus')) {
+            // Update the element report.
+            navigationErrors.push('focus');
+          }
           return false;
         }
       }
+      // Otherwise, i.e. if the click failed:
+      else {
+        // Increment the counts of incorrect navigations and incorrect key navigations.
+        data.totals.navigations.all.incorrect++;
+        data.totals.navigations.specific[keyProp].incorrect++;
+        // If itemization is required and a click failure has not yet been reported:
+        if (withItems && ! navigationErrors.includes('click')) {
+          // Update the element report.
+          navigationErrors.push('click');
+        }
+        return false;
+      }
     };
     // Returns the index to which an arrow key should move the focus.
     const arrowTarget = (startIndex, tabCount, orientation, direction) => {