testaro 4.4.0 → 4.5.2

package/commands.js

@@ -100,8 +100,8 @@ exports.commands = {
     state: [
       'Wait until the page reaches a load state',
       {
-        which: [true, 'string', 'hasLength', '“loaded” or “idle”'],
-        what: [false, 'string', 'isState', 'comment']
+        which: [true, 'string', 'isState', '“loaded” or “idle”'],
+        what: [false, 'string', 'hasLength', 'comment']
       }
     ],
     tenonRequest: [
@@ -146,7 +146,8 @@ exports.commands = {
     aatt: [
       'Perform an AATT test with HTML CodeSniffer',
       {
-        waitLong: [false, 'boolean', '', 'whether to wait 20 instead of 10 seconds for the result']
+        waitLong: [false, 'boolean', '', 'whether to wait 12 instead of 6 seconds for a result'],
+        tryLimit: [false, 'number', '', 'times to try the test before giving up; default 4']
       }
     ],
     axe: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.4.0",
+  "version": "4.5.2",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -256,7 +256,7 @@ const launch = async typeName => {
       // Make its console messages appear in the Playwright console.
       page.on('console', msg => {
         const msgText = msg.text();
-        console.log(msgText);
+        console.log(`[${msgText}]`);
         logCount++;
         logSize += msgText.length;
         const msgLC = msgText.toLowerCase();
@@ -291,7 +291,7 @@ const textOf = async (page, element) => {
     if (['A', 'BUTTON', 'INPUT', 'SELECT'].includes(tagName)) {
       // Return its visible labels, descriptions, and legend if the first input in a fieldset.
       totalText = await page.evaluate(element => {
-        const {tagName} = element;
+        const {tagName, ariaLabel} = element;
         let ownText = '';
         if (['A', 'BUTTON'].includes(tagName)) {
           ownText = element.textContent;
@@ -302,6 +302,9 @@ const textOf = async (page, element) => {
         // HTML link elements have no labels property.
         const labels = tagName !== 'A' ? Array.from(element.labels) : [];
         const labelTexts = labels.map(label => label.textContent);
+        if (ariaLabel) {
+          labelTexts.push(ariaLabel);
+        }
         const refIDs = new Set([
           element.getAttribute('aria-labelledby') || '',
           element.getAttribute('aria-describedby') || ''
@@ -369,69 +372,79 @@ const textOf = async (page, element) => {
     return null;
   }
 };
-// Returns an element case-insensitively matching a text.
+// Returns an element of a type case-insensitively matching a text.
 const matchElement = async (page, selector, matchText, index = 0) => {
-  // If the page still exists:
-  if (page) {
-    // Wait 2 seconds until the body contains any text to be matched.
-    const slimText = debloat(matchText);
-    const bodyText = await page.textContent('body');
-    const slimBody = debloat(bodyText);
-    const textInBodyJSHandle = await page.waitForFunction(
-      args => {
-        const matchText = args[0];
-        const bodyText = args[1];
-        return ! matchText || bodyText.includes(matchText);
-      },
-      [slimText, slimBody],
-      {timeout: 4000}
-    )
-    .catch(async error => {
-      console.log(`ERROR: text “${matchText}” not in body (${error.message})`);
-    });
-    // If there is no text to be matched or the body contained it:
-    if (textInBodyJSHandle) {
-      const lcText = matchText ? matchText.toLowerCase() : '';
-      // Identify the selected elements.
-      const selections = await page.$$(`body ${selector}`);
+  if (matchText) {
+    // If the page still exists:
+    if (page) {
+      const slimText = debloat(matchText);
+      // Identify the elements of the specified type.
+      const selections = await page.$$(selector);
       // If there are any:
       if (selections.length) {
         // If there are enough to make a match possible:
         if (index < selections.length) {
-          // Return the nth one including any specified text, or the count of candidates if none.
-          const elementTexts = [];
-          let nth = 0;
-          for (const element of selections) {
-            const elementText = await textOf(page, element);
-            elementTexts.push(elementText);
-            if ((! lcText || elementText.includes(lcText)) && nth++ === index) {
-              return element;
+          // Return the specified one, if any.
+          let matchCount = 0;
+          const selectionTexts = [];
+          for (const selection of selections) {
+            const selectionText = await textOf(page, selection);
+            selectionTexts.push(selectionText);
+            if (selectionText.includes(slimText)) {
+              if (matchCount++ === index) {
+                return {
+                  success: true,
+                  matchingElement: selection
+                };
+              }
             }
           }
-          return elementTexts;
+          // None satisfied the specifications, so return a failure.
+          return {
+            success: false,
+            error: 'exhausted',
+            message: `Text found in only ${matchCount} (not ${index + 1}) of ${selections.length}`,
+            candidateTexts: selectionTexts
+          };
         }
-        // Otherwise, i.e. if there are too few to make a match possible:
+        // Otherwise, i.e. if there are too few such elements to make a match possible:
         else {
-          // Return the count of candidates.
-          return selections.length;
+          // Return a failure.
+          return {
+            success: false,
+            error: 'fewer',
+            message: `Count of '${selector}' elements only ${selections.length}`
+          };
         }
       }
-      // Otherwise, i.e. if there are no selected elements, return 0.
+      // Otherwise, i.e. if there are no elements of the specified type:
       else {
-        return 0;
+        // Return a failure.
+        return {
+          success: false,
+          error: 'none',
+          message: `No '${selector}' elements found`
+        };
       }
     }
-    // Otherwise, i.e. if the body did not contain it:
+    // Otherwise, i.e. if the page no longer exists:
     else {
-      // Return the failure.
-      return -1;
+      // Return a failure.
+      return {
+        success: false,
+        error: 'gone',
+        message: 'Page gone'
+      };
     }
   }
-  // Otherwise, i.e. if the page no longer exists:
+  // Otherwise, i.e. if no text was specified:
   else {
-    // Return null.
-    console.log('ERROR: Page gone');
-    return null;
+    // Return a failure.
+    return {
+      success: false,
+      error: 'text',
+      message: 'No text specified'
+    };
   }
 };
 // Returns a string with any final slash removed.
@@ -502,8 +515,9 @@ const visit = async (act, page, isStrict) => {
           // If the visit fails:
           if (response === 'error') {
             // Give up.
-            console.log(`ERROR: Visits to ${requestedURL} failed`);
-            act.result = `ERROR: Visit to ${requestedURL} failed`;
+            const errorMsg = `ERROR: Attemts to visit ${requestedURL} failed`;
+            console.log(errorMsg);
+            act.result = errorMsg;
             await page.goto('about:blank')
             .catch(error => {
               console.log(`ERROR: Navigation to blank page failed (${error.message})`);
@@ -563,6 +577,14 @@ const waitError = (page, act, error, what) => {
   act.result.error = `ERROR waiting for ${what}`;
   return false;
 };
+// Waits.
+const wait = ms => {
+  return new Promise(resolve => {
+    setTimeout(() => {
+      resolve('');
+    }, ms);
+  });
+};
 // Recursively performs the acts in a report.
 const doActs = async (report, actIndex, page) => {
   const {acts} = report;
@@ -597,12 +619,12 @@ const doActs = async (report, actIndex, page) => {
         if (truth[1]) {
           // If the performance of commands is to stop:
           if (act.jump === 0) {
-            // Set the command index to cause a stop.
+            // Stop.
             actIndex = -2;
           }
           // Otherwise, if there is a numerical jump:
           else if (act.jump) {
-            // Set the command index accordingly.
+            // Set the act index accordingly.
             actIndex += act.jump - 1;
           }
           // Otherwise, if there is a named next command:
@@ -630,76 +652,65 @@ const doActs = async (report, actIndex, page) => {
         else if (act.type === 'wait') {
           const {what, which} = act;
           console.log(`>> for ${what} to include “${which}”`);
-          // Wait 5 or 10 seconds for the specified text, and quit if it does not.
-          let successJSHandle;
+          // Wait 5 or 10 seconds for the specified text, and quit if it does not appear.
           if (act.what === 'url') {
-            successJSHandle = await page.waitForFunction(
-              text => document.URL.includes(text), act.which, {timeout: 5000}
-            )
+            await page.waitForURL(act.which, {timeout: 15000})
             .catch(error => {
-              actIndex = acts.length;
-              return waitError(page, act, error, 'URL');
+              actIndex = -2;
+              waitError(page, act, error, 'URL');
             });
           }
           else if (act.what === 'title') {
-            successJSHandle = await page.waitForFunction(
-              text => document.title.includes(text), act.which, {timeout: 5000}
+            await page.waitForFunction(
+              text => document && document.title && document.title.includes(text),
+              act.which,
+              {
+                polling: 1000,
+                timeout: 5000
+              }
             )
             .catch(error => {
-              actIndex = acts.length;
-              return waitError(page, act, error, 'title');
+              actIndex = -2;
+              waitError(page, act, error, 'title');
             });
           }
           else if (act.what === 'body') {
-            successJSHandle = await page.waitForFunction(
-              matchText => {
-                const innerText = document && document.body && document.body.innerText;
-                if (innerText) {
-                  return innerText.includes(matchText);
-                }
-                else {
-                  actIndex = acts.length;
-                  console.log('ERROR finding document body');
-                  return false;
-                }
-              }, which, {timeout: 20000}
+            await page.waitForFunction(
+              text => document && document.body && document.body.innerText.includes(text),
+              act.which,
+              {
+                polling: 2000,
+                timeout: 10000
+              }
             )
-            .catch(error => {
-              actIndex = acts.length;
-              return waitError(page, act, error, 'body');
+            .catch(async error => {
+              actIndex = -2;
+              waitError(page, act, error, 'body');
             });
           }
-          if (successJSHandle) {
+          // If the text was found:
+          if (actIndex > -2) {
+            // Add this to the report.
             act.result = {url: page.url()};
             if (act.what === 'title') {
               act.result.title = await page.title();
             }
-            await page.waitForLoadState('networkidle', {timeout: 10000})
-            .catch(error => {
-              console.log(`ERROR waiting for stability after ${act.what} (${error.message})`);
-              act.result.error = `ERROR waiting for stability after ${act.what}`;
-            });
           }
         }
         // Otherwise, if the act is a wait for a state:
         else if (act.type === 'state') {
-          // If the state is valid:
+          // Wait for it, and quit if it does not appear.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
-          if (stateIndex !== -1) {
-            // Wait for it, and quit if it does not appear.
-            await page.waitForLoadState(
-              ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 5000][stateIndex]}
-            )
-            .catch(error => {
-              console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
-              act.result = `ERROR waiting for page to be ${act.which}`;
-              actIndex = acts.length;
-            });
-          }
-          else {
-            console.log('ERROR: invalid state');
-            act.result = 'ERROR: invalid state';
-            actIndex = acts.length;
+          await page.waitForLoadState(
+            ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 5000][stateIndex]}
+          )
+          .catch(error => {
+            console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
+            act.result = `ERROR waiting for page to be ${act.which}`;
+            actIndex = -2;
+          });
+          if (actIndex > -2) {
+            act.result = `Page became ${act.which}`;
           }
         }
         // Otherwise, if the act is a page switch:
@@ -889,48 +900,34 @@ const doActs = async (report, actIndex, page) => {
             // Otherwise, if the act is a move:
             else if (moves[act.type]) {
               const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
-              // Identify the element to perform the move on.
-              const whichElement = await matchElement(page, selector, act.which || '', act.index);
-              // If there were enough candidates but no text match:
-              if (Array.isArray(whichElement)) {
-                // Add the result to the act.
-                act.result = {
-                  candidateCount: whichElement.length,
-                  error: 'ERROR: no element with matching text found',
-                  candidateTexts: whichElement
-                };
-              }
-              // Otherwise, if the body did not contain the text:
-              else if (whichElement === -1) {
-                // Add the failure to the act.
-                act.result = 'ERROR: body did not contain text to match';
-              }
-              // Otherwise, if there were not enough candidates:
-              else if (typeof whichElement === 'number') {
-                // Add the failure to the act.
-                act.result = {
-                  candidateCount: whichElement,
-                  error: 'ERROR: too few such elements to allow a match'
-                };
+              // Try up to 5 times, every 2 seconds, to identify the element to perform the move on.
+              let matchResult = {success: false};
+              let tries = 0;
+              while (tries++ < 5 && ! matchResult.success) {
+                matchResult = await matchElement(page, selector, act.which || '', act.index);
+                if (! matchResult.success) {
+                  await wait(2000);
+                }
               }
-              // Otherwise, if a match was found:
-              else if (whichElement !== null) {
+              // If a match was found:
+              if (matchResult.success) {
+                const {matchingElement} = matchResult;
                 // If the move is a button click, perform it.
                 if (act.type === 'button') {
-                  await whichElement.click({timeout: 3000});
+                  await matchingElement.click({timeout: 3000});
                   act.result = 'clicked';
                 }
                 // Otherwise, if it is checking a radio button or checkbox, perform it.
                 else if (['checkbox', 'radio'].includes(act.type)) {
-                  await whichElement.waitForElementState('stable', {timeout: 2000})
+                  await matchingElement.waitForElementState('stable', {timeout: 2000})
                   .catch(error => {
                     console.log(`ERROR waiting for stable ${act.type} (${error.message})`);
                     act.result = `ERROR waiting for stable ${act.type}`;
                   });
                   if (! act.result) {
-                    const isEnabled = await whichElement.isEnabled();
+                    const isEnabled = await matchingElement.isEnabled();
                     if (isEnabled) {
-                      await whichElement.check({
+                      await matchingElement.check({
                         force: true,
                         timeout: 2000
                       })
@@ -951,29 +948,37 @@ const doActs = async (report, actIndex, page) => {
                 }
                 // Otherwise, if it is focusing the element, perform it.
                 else if (act.type === 'focus') {
-                  await whichElement.focus({timeout: 2000});
+                  await matchingElement.focus({timeout: 2000});
                   act.result = 'focused';
                 }
                 // Otherwise, if it is clicking a link, perform it.
                 else if (act.type === 'link') {
-                  const href = await whichElement.getAttribute('href');
-                  const target = await whichElement.getAttribute('target');
-                  await whichElement.click({timeout: 2000})
+                  const href = await matchingElement.getAttribute('href');
+                  const target = await matchingElement.getAttribute('target');
+                  await matchingElement.click({timeout: 2000})
                   .catch(async () => {
-                    await whichElement.click({
+                    console.log('ERROR: First attempt to click link timed out');
+                    await matchingElement.click({
                       force: true,
                       timeout: 10000
+                    })
+                    .catch(() => {
+                      actIndex = -2;
+                      console.log('ERROR: Second (forced) attempt to click link timed out');
+                      act.result = 'ERROR: Normal and forced click attempts timed out';
                     });
                   });
-                  act.result = {
-                    href: href || 'NONE',
-                    target: target || 'NONE',
-                    move: 'clicked'
-                  };
+                  if (actIndex > -2) {
+                    act.result = {
+                      href: href || 'NONE',
+                      target: target || 'NONE',
+                      move: 'clicked'
+                    };
+                  }
                 }
                 // Otherwise, if it is selecting an option in a select list, perform it.
                 else if (act.type === 'select') {
-                  const options = await whichElement.$$('option');
+                  const options = await matchingElement.$$('option');
                   let optionText = '';
                   if (options && Array.isArray(options) && options.length) {
                     const optionTexts = [];
@@ -984,12 +989,12 @@ const doActs = async (report, actIndex, page) => {
                     const matchTexts = optionTexts.map((text, index) => text.includes(act.what) ? index : -1);
                     const index = matchTexts.filter(text => text > -1)[act.index || 0];
                     if (index !== undefined) {
-                      await whichElement.selectOption({index});
+                      await matchingElement.selectOption({index});
                       optionText = optionTexts[index];
                     }
                   }
                   act.result = optionText
-                    ? `&ldquo;${optionText}}&rdquo; selected`
+                    ? `“${optionText}” selected`
                     : 'ERROR: option not found';
                 }
                 // Otherwise, if it is entering text on the element:
@@ -1003,19 +1008,24 @@ const doActs = async (report, actIndex, page) => {
                     what = what.replace(/__[A-Z]+__/, envValue);
                   }
                   // Enter the text.
-                  await whichElement.type(act.what);
+                  await matchingElement.type(act.what);
                   report.presses += act.what.length;
                   act.result = 'entered';
                 }
                 // Otherwise, i.e. if the move is unknown, add the failure to the act.
                 else {
                   // Report the error.
-                  act.result = 'ERROR: move unknown';
+                  const report = 'ERROR: move unknown';
+                  act.result = report;
+                  console.log(report);
                 }
               }
-              // Otherwise, i.e. if the page was gone:
+              // Otherwise, i.e. if no match was found:
               else {
-                act.result = 'ERROR: page gone, so matching element not found';
+                // Stop.
+                act.result = matchResult;
+                console.log('ERROR: Specified element not found');
+                actIndex = -2;
               }
             }
             // Otherwise, if the act is a keypress:
@@ -1203,7 +1213,9 @@ const doActs = async (report, actIndex, page) => {
     // Otherwise, i.e. if the command is invalid:
     else {
       // Add an error result to the act.
-      act.result = `ERROR: Invalid command of type ${act.type}`;
+      const errorMsg = `ERROR: Invalid command of type ${act.type}`;
+      act.result = errorMsg;
+      console.log(errorMsg);
     }
     // Perform the remaining acts.
     await doActs(report, actIndex + 1, page);

package/tests/aatt.js

@@ -7,31 +7,59 @@
 const {evaluate} = require('aatt');
 
 // FUNCTIONS
-exports.reporter = async (page, waitLong) => {
-  // Set the limit in seconds on the wait for the result.
-  const timeLimit = waitLong ? 20 : 10;
-  // Get the HTML of the document body.
-  const source = await page.content();
-  // Return the result of a test with the HTML CodeSniffer WCAG 2.1 AA ruleset as a string.
-  const report = evaluate({
-    source,
-    output: 'json',
-    engine: 'htmlcs',
-    level: 'WCAG2AA'
-  });
-  // Wait for it until the time limit expires.
-  let timeoutID;
-  const wait = new Promise(resolve => {
-    timeoutID = setTimeout(() => {
-      resolve('');
-    }, 1000 * timeLimit);
-  });
-  const resultIfFast = await Promise.race([report, wait]);
-  // If it arrived within the time limit:
-  if (resultIfFast) {
-    clearTimeout(timeoutID);
-    // Remove the non-JSON prefix and (if any) suffix from the string.
-    const reportJSON = resultIfFast.replace(/^.+?Object]\s+|\s+done\s*$/sg, '');
+// Recursively test a page with HTML CodeSniffer for WCAC 2.1 AAA.
+const retest = async (page, waitLong, triesLeft) => {
+  // If the limit on tries has not been exhausted:
+  if (triesLeft) {
+    // Set the limit in seconds on the wait for the result.
+    const timeLimit = waitLong ? 12 : 6;
+    // Get the HTML of the document body.
+    const source = await page.content();
+    // Return the result of a test with the HTML CodeSniffer WCAG 2.1 AA ruleset as a string.
+    const report = evaluate({
+      source,
+      output: 'json',
+      engine: 'htmlcs',
+      level: 'WCAG2AAA'
+    });
+    // Wait for it until the time limit expires.
+    let timeoutID;
+    const wait = new Promise(resolve => {
+      timeoutID = setTimeout(() => {
+        resolve('');
+      }, 1000 * timeLimit);
+    });
+    const result = await Promise.race([report, wait]);
+    // If it arrived within the time limit:
+    if (result) {
+      clearTimeout(timeoutID);
+      // Return the result as JSON.
+      const reportJSON = result.replace(/^.+?Object.?\s+|\s+done\s*$/sg, '');
+      return {
+        triesLeft,
+        reportJSON
+      };
+    }
+    else {
+      console.log(`ERROR: Test aatt timed out at ${timeLimit} seconds; tries left: ${triesLeft - 1}`);
+      return retest(page, waitLong, --triesLeft);
+    }
+  }
+  // Otherwise, i.e. if the limit on tries has been exhausted:
+  else {
+    // Return this.
+    return {
+      triesLeft,
+      reportJSON: ''
+    };
+  }
+};
+exports.reporter = async (page, waitLong, tryLimit = 4) => {
+  // Try the test up to the limit on tries.
+  const result = await retest(page, waitLong, tryLimit);
+  const {triesLeft, reportJSON} = result;
+  // If any try succeeded:
+  if (reportJSON) {
     try {
       // Convert the JSON string to an array.
       const issueArray = JSON.parse(reportJSON);
@@ -49,10 +77,18 @@ exports.reporter = async (page, waitLong) => {
           .join('+');
         }
       });
-      return {result: nonNotices};
+      return {
+        result: {
+          report: nonNotices,
+          triesLeft
+        }
+      };
     }
     catch (error) {
       console.log(`ERROR processing AATT report (${error.message})`);
+      console.log(
+        `JSON report starts with ${reportJSON.slice(0, 50)} and ends with ${reportJSON.slice(-50)}`
+      );
       return {
         result: {
           prevented: true,
@@ -61,14 +97,15 @@ exports.reporter = async (page, waitLong) => {
       };
     }
   }
-  // Otherwise, i.e. if the result did not arrive within the time limit:
+  // Otherwise, i.e. if the limit on tries was exhausted:
   else {
     // Report the failure.
-    console.log('ERROR: getting report took too long');
+    const error = 'ERROR: Getting AATT report took too long';
+    console.log(error);
     return {
       result: {
         prevented: true,
-        error: 'ERROR: getting AATT report took too long'
+        error
       }
     };
   }