testaro 4.11.0 → 4.12.1

package/commands.js

@@ -153,7 +153,7 @@ exports.commands = {
     axe: [
       'Perform an Axe test',
       {
-        detailLevel: [true, 'number', '', 'count to include of: violations, incomplete, passes, inapplicable'],
+        detailLevel: [true, 'number', '', '0 = least, 4 = most'],
         rules: [true, 'array', 'areStrings', 'rule names, or empty if all']
       }
     ],
@@ -206,7 +206,7 @@ exports.commands = {
       }
     ],
     menuNav: [
-      'Perform a tabNav test',
+      'Perform a menuNav test',
       {
         withItems: [true, 'boolean']
       }

package/high.js

@@ -1,7 +1,7 @@
 /*
   high.js
   Invokes Testaro with the high-level method.
-  Usage example: node high tp10 weborgs
+  Usage example: node high tp11 weborgs
 */
 
 const {runJob} = require('./create');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.11.0",
+  "version": "4.12.1",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -239,6 +239,7 @@ const isValidReport = async report => {
 const closeBrowser = async () => {
   const browser = browserContext && browserContext.browser();
   if (browser) {
+    console.log(`Closing browser, version ${browser.version()}`);
     await browser.close();
   }
 };

package/tests/aatt.js

@@ -65,9 +65,12 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
       const issueArray = JSON.parse(reportJSON);
       // Remove the notices from the array.
       const nonNotices = issueArray.filter(issue => issue.type !== 'notice');
-      // Convert the technique property from a string to an array of strings.
+      // For each non-notice issue:
+      let errors = 0;
+      let warnings = 0;
       nonNotices.forEach(issue => {
         if (issue.type) {
+          // Convert the technique property from a string to an array of strings.
           const longTech = issue.techniques;
           issue.techniques = longTech.replace(/a><a/g, 'a>%<a').split('%');
           issue.id = issue
@@ -75,12 +78,24 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
           .map(technique => technique.replace(/^.+?>|<\/a>$/g, ''))
           .sort()
           .join('+');
+          // Add the issue to the totals.
+          if (issue.type === 'error') {
+            errors++;
+          }
+          else if (issue.type === 'warning') {
+            warnings++;
+          }
         }
       });
+      // Return the result.
       return {
         result: {
+          totals: {
+            errors,
+            warnings
+          },
           report: nonNotices,
-          triesLeft
+          preventionCount: tryLimit - triesLeft - 1
         }
       };
     }

package/tests/alfa.js

@@ -44,7 +44,13 @@ exports.reporter = async page => {
       });
       await rulePage.close();
     }
-    const data = [];
+    const data = {
+      totals: {
+        failures: 0,
+        warnings: 0
+      },
+      items: []
+    };
     await Scraper.with(async scraper => {
       for (const input of await scraper.scrape(page.url())) {
         const audit = Audit.of(input, alfaRules.default);
@@ -96,7 +102,13 @@ exports.reporter = async page => {
               if (etcTags.length) {
                 outcomeData.etcTags = etcTags;
               }
-              data.push(outcomeData);
+              if (outcomeData.verdict === 'failed') {
+                data.totals.failures++;
+              }
+              else if (outcomeData.verdict === 'cantTell') {
+                data.totals.warnings++;
+              }
+              data.items.push(outcomeData);
             }
           }
         });

package/tests/focAll.js

@@ -1,17 +1,12 @@
 /*
   focAll
-  This test reports discrepancies between focusable and Tab-focused element counts.
-  The test first counts all the visible focusable (i.e. with tabIndex 0) elements
-  (except counting each group of radio buttons as only one focusable element). Then
-  it repeatedly presses the Tab (or Option-Tab in webkit) key until it has reached
-  all the elements it can and counts those elements. If the latter are more than the
-  former, Tab-key navigation made more focusable elements visible. If the latter are
-  fewer than the former, the page manages focus so as to prevent Tab-key navigation
-  from reaching all focusable elements. Either of these can complicate navigation for
-  users. It may disappoint the expectation that the content will remain stable as they
-  move with the Tab key, or the expectation that they can reach every focusable element
-  (or widget, such as one radio button or tab in each group) merely by pressing the Tab
-  key.
+  This test reports discrepancies between focusable and Tab-focused element counts. The test first
+  counts all the visible focusable (i.e. with tabIndex 0) elements (except counting each group of
+  radio buttons as only one focusable element). Then it repeatedly presses the Tab (or Option-Tab
+  in webkit) key until it has reached all the elements it can and counts those elements. If the
+  two counts differ, navigation can be made more difficult. The cause may be surprising changes in
+  content during navigation with the Tab key, or inability to reach every focusable element (or
+  widget, such as one radio button or tab in each group) merely by pressing the Tab key.
 */
 exports.reporter = async page => {
   // Identify the count of visible focusable elements.

package/tests/focInd.js

@@ -5,10 +5,13 @@
   line thickness and non-transparent color. The test is based on the assumption that outlines are
   the standard and thus most familiar focus indicator. Other focus indicators are assumed better
   than none, but more likely to be misunderstood. For example, underlines may be mistaken for
-  selection indicators. Some pages delay the appearance of focus indicators. This test waits for
-  focus indicators to appear if specified and, if there is a delay, reports on its magnitude.
+  selection indicators. Some pages delay the appearance of focus indicators. If a wait is
+  specified, the test checks every 100 ms for an outline until the allow wait time expires,
+  and once more after it expires. If no outline appears by then, the test checks for other focus
+  indicators. If an outline does not appear immediately but appears on a subsequent check, the test
+  reports the amount of the delay.
 
-  Bug: This test fails to recognize outlines when run with firefox.
+  WARNING: This test fails to recognize outlines when run with firefox.
 */
 exports.reporter = async (page, revealAll, allowedDelay, withItems) => {
   // If required, make all elements visible.

package/tests/ibm.js

@@ -88,7 +88,6 @@ const doTest = async (content, withItems, timeLimit) => {
     return ibmTypeReport;
   }
   else {
-    console.log('ERROR: getting ibm test report took too long');
     return {
       prevented: true,
       error: 'ERROR: getting ibm test report took too long'
@@ -105,6 +104,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
     result.content = await doTest(typeContent, withItems, timeLimit);
     if (result.content.prevented) {
       result.prevented = true;
+      console.log('ERROR: Getting ibm test report from page took too long');
     }
   }
   // If a test with new content is to be performed:
@@ -112,8 +112,9 @@ exports.reporter = async (page, withItems, withNewContent) => {
     const timeLimit = 20;
     const typeContent = page.url();
     result.url = await doTest(typeContent, withItems, timeLimit);
-    if (result.content.prevented) {
+    if (result.url.prevented) {
       result.prevented = true;
+      console.log('ERROR: Getting ibm test report from URL took too long');
     }
   }
   return {result};

package/tests/motion.js

@@ -9,7 +9,8 @@
   between screen shots (interval), and how many screen shots to make (count). The test compares the
   screen shots and reports 9 statistics:
     0. bytes: an array of the sizes of the screen shots, in bytes
-    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs of screen shots
+    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs
+      of screen shots
     2. meanLocalRatio: the mean of the ratios in the localRatios array
     3. maxLocalRatio: the greatest of the ratios in the localRatios array
     4. globalRatio: the ratio of bytes of the largest to the smallest screen shot
@@ -17,6 +18,10 @@
     6. meanPixelChange: the mean of the counts in the pixelChanges array
     7. maxPixelChange: the greatest of the counts in the pixelChanges array
     8. changeFrequency: what fraction of the adjacent pairs of screen shots has pixel differences
+  
+  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
+  when the browser type is chromium and is not implemented for the firefox browser type. The only
+  browser type usable with this test is webkit.
 */
 const pixelmatch = require('pixelmatch');
 const {PNG} = require('pngjs');

package/tests/tabNav.js

@@ -1,6 +1,6 @@
 /*
   tabNav
-  This test reports nonstandard keyboard navigation among tab elements in tab lists.
+  This test reports nonstandard keyboard navigation among tab elements in visible tab lists.
   Standards are based on https://www.w3.org/TR/wai-aria-practices-1.1/#tabpanel.
 */
 
@@ -73,8 +73,8 @@ exports.reporter = async (page, withItems) => {
       correct: []
     };
   }
-  // Identify an array of the tablists.
-  const tabLists = await page.$$('[role=tablist]');
+  // Identify an array of the visible tablists.
+  const tabLists = await page.$$('[role=tablist]:visible');
   if (tabLists.length) {
     // FUNCTION DEFINITIONS START
     // Returns text associated with an element.
@@ -94,22 +94,34 @@ exports.reporter = async (page, withItems) => {
     ) => {
       let pressed = true;
       // Click the tab element, to make the focus on it effective.
-      await tabElement.click({timeout: 1500})
+      await tabElement.click({
+        timeout: 500
+      })
+      .catch(async error => {
+        await tabElement.click({
+          force: true
+        });
+      })
       .catch(error => {
         console.log(`ERROR: could not click tab element ${itemData.text} (${error.message})`);
-        pressed = false;
+      pressed = false;
       });
+      // Increment the counts of navigations and key navigations.
+      data.totals.navigations.all.total++;
+      data.totals.navigations.specific[keyProp].total++;
+      const {navigationErrors} = itemData;
+      // If the click succeeded:
       if (pressed) {
         // Refocus the tab element and press the specified key (page.keyboard.press may fail).
-        await tabElement.press(keyName)
+        await tabElement.press(keyName, {
+          timeout: 1000
+        })
         .catch(error => {
           console.log(`ERROR: could not press ${keyName} (${error.message})`);
           pressed = false;
         });
+        // If the refocus and keypress succeeded:
         if (pressed) {
-          // Increment the counts of navigations and key navigations.
-          data.totals.navigations.all.total++;
-          data.totals.navigations.specific[keyProp].total++;
           // Identify which tab element is now focused, if any.
           const focusIndex = await focusedTab(tabs);
           // If the focus is correct:
@@ -128,15 +140,36 @@ exports.reporter = async (page, withItems) => {
             // If itemization is required:
             if (withItems) {
               // Update the element report.
-              itemData.navigationErrors.push(keyName);
+              navigationErrors.push(keyName);
             }
           }
           return elementIsCorrect;
         }
+        // Otherwise, i.e. if the refocus or keypress failed:
         else {
+          // Increment the counts of incorrect navigations and incorrect key navigations.
+          data.totals.navigations.all.incorrect++;
+          data.totals.navigations.specific[keyProp].incorrect++;
+          // If itemization is required and a focus failure has not yet been reported:
+          if (withItems && ! navigationErrors.includes('focus')) {
+            // Update the element report.
+            navigationErrors.push('focus');
+          }
           return false;
         }
       }
+      // Otherwise, i.e. if the click failed:
+      else {
+        // Increment the counts of incorrect navigations and incorrect key navigations.
+        data.totals.navigations.all.incorrect++;
+        data.totals.navigations.specific[keyProp].incorrect++;
+        // If itemization is required and a click failure has not yet been reported:
+        if (withItems && ! navigationErrors.includes('click')) {
+          // Update the element report.
+          navigationErrors.push('click');
+        }
+        return false;
+      }
     };
     // Returns the index to which an arrow key should move the focus.
     const arrowTarget = (startIndex, tabCount, orientation, direction) => {