testaro 4.3.2 → 4.5.0

package/commands.js

@@ -1,7 +1,7 @@
 exports.commands = {
   etc: {
     button: [
-      'Click a button',
+      'Click a button or submit input',
       {
         which: [true, 'string', 'hasLength', 'substring of button text'],
         index: [false, 'number', '', 'index among matches if not 0'],
@@ -94,14 +94,14 @@ exports.commands = {
       {
         which: [true, 'string', 'hasLength', 'substring of select-list text'],
         index: [false, 'number', '', 'index among matches if not 0'],
-        what: [true, 'string', 'hasLength', 'substring of option text']
+        what: [true, 'string', 'hasLength', 'substring of option text content']
       }
     ],
     state: [
       'Wait until the page reaches a load state',
       {
-        which: [true, 'string', 'hasLength', '“loaded” or “idle”'],
-        what: [false, 'string', 'isState', 'comment']
+        which: [true, 'string', 'isState', '“loaded” or “idle”'],
+        what: [false, 'string', 'hasLength', 'comment']
       }
     ],
     tenonRequest: [

package/high.js

@@ -1,5 +1,8 @@
-// high.js
-// Invokes Testaro with the high-level method.
+/*
+  high.js
+  Invokes Testaro with the high-level method.
+  Usage example: node high tp10 weborgs
+*/
 
 const {runJob} = require('./create');
 const scriptID = process.argv[2];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.3.2",
+  "version": "4.5.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -12,12 +12,13 @@ const {commands} = require('./commands');
 const debug = process.env.DEBUG === 'true';
 // Set WAITS environment variable to a positive number to insert delays (in ms).
 const waits = Number.parseInt(process.env.WAITS) || 0;
+const urlInject = process.env.URL_INJECT || 'yes';
 // CSS selectors for targets of moves.
 const moves = {
-  button: 'button',
+  button: 'button, [role=button], input[type=submit]',
   checkbox: 'input[type=checkbox]',
   focus: true,
-  link: 'a',
+  link: 'a, [role=link]',
   radio: 'input[type=radio]',
   select: 'select',
   text: 'input[type=text]'
@@ -255,7 +256,7 @@ const launch = async typeName => {
       // Make its console messages appear in the Playwright console.
       page.on('console', msg => {
         const msgText = msg.text();
-        console.log(msgText);
+        console.log(`[${msgText}]`);
         logCount++;
         logSize += msgText.length;
         const msgLC = msgText.toLowerCase();
@@ -291,7 +292,13 @@ const textOf = async (page, element) => {
       // Return its visible labels, descriptions, and legend if the first input in a fieldset.
       totalText = await page.evaluate(element => {
         const {tagName} = element;
-        const ownText = ['A', 'BUTTON'].includes(tagName) ? element.textContent : '';
+        let ownText = '';
+        if (['A', 'BUTTON'].includes(tagName)) {
+          ownText = element.textContent;
+        }
+        else if (tagName === 'INPUT' && element.type === 'submit') {
+          ownText = element.value;
+        }
         // HTML link elements have no labels property.
         const labels = tagName !== 'A' ? Array.from(element.labels) : [];
         const labelTexts = labels.map(label => label.textContent);
@@ -362,69 +369,79 @@ const textOf = async (page, element) => {
     return null;
   }
 };
-// Returns an element case-insensitively matching a text.
+// Returns an element of a type case-insensitively matching a text.
 const matchElement = async (page, selector, matchText, index = 0) => {
-  // If the page still exists:
-  if (page) {
-    // Wait 2 seconds until the body contains any text to be matched.
-    const slimText = debloat(matchText);
-    const bodyText = await page.textContent('body');
-    const slimBody = debloat(bodyText);
-    const textInBodyJSHandle = await page.waitForFunction(
-      args => {
-        const matchText = args[0];
-        const bodyText = args[1];
-        return ! matchText || bodyText.includes(matchText);
-      },
-      [slimText, slimBody],
-      {timeout: 2000}
-    )
-    .catch(async error => {
-      console.log(`ERROR: text to match not in body (${error.message})`);
-    });
-    // If there is no text to be matched or the body contained it:
-    if (textInBodyJSHandle) {
-      const lcText = matchText ? matchText.toLowerCase() : '';
-      // Identify the selected elements.
-      const selections = await page.$$(`body ${selector}`);
+  if (matchText) {
+    // If the page still exists:
+    if (page) {
+      const slimText = debloat(matchText);
+      // Identify the elements of the specified type.
+      const selections = await page.$$(selector);
       // If there are any:
       if (selections.length) {
         // If there are enough to make a match possible:
         if (index < selections.length) {
-          // Return the nth one including any specified text, or the count of candidates if none.
-          const elementTexts = [];
-          let nth = 0;
-          for (const element of selections) {
-            const elementText = await textOf(page, element);
-            elementTexts.push(elementText);
-            if ((! lcText || elementText.includes(lcText)) && nth++ === index) {
-              return element;
+          // Return the specified one, if any.
+          let matchCount = 0;
+          const selectionTexts = [];
+          for (const selection of selections) {
+            const selectionText = await textOf(page, selection);
+            selectionTexts.push(selectionText);
+            if (selectionText.includes(slimText)) {
+              if (matchCount++ === index) {
+                return {
+                  success: true,
+                  matchingElement: selection
+                };
+              }
             }
           }
-          return elementTexts;
+          // None satisfied the specifications, so return a failure.
+          return {
+            success: false,
+            error: 'exhausted',
+            message: `Text found in only ${matchCount} (not ${index + 1}) of ${selections.length}`,
+            candidateTexts: selectionTexts
+          };
         }
-        // Otherwise, i.e. if there are too few to make a match possible:
+        // Otherwise, i.e. if there are too few such elements to make a match possible:
         else {
-          // Return the count of candidates.
-          return selections.length;
+          // Return a failure.
+          return {
+            success: false,
+            error: 'fewer',
+            message: `Count of '${selector}' elements only ${selections.length}`
+          };
         }
       }
-      // Otherwise, i.e. if there are no selected elements, return 0.
+      // Otherwise, i.e. if there are no elements of the specified type:
       else {
-        return 0;
+        // Return a failure.
+        return {
+          success: false,
+          error: 'none',
+          message: `No '${selector}' elements found`
+        };
       }
     }
-    // Otherwise, i.e. if the body did not contain it:
+    // Otherwise, i.e. if the page no longer exists:
     else {
-      // Return the failure.
-      return -1;
+      // Return a failure.
+      return {
+        success: false,
+        error: 'gone',
+        message: 'Page gone'
+      };
     }
   }
-  // Otherwise, i.e. if the page no longer exists:
+  // Otherwise, i.e. if no text was specified:
   else {
-    // Return null.
-    console.log('ERROR: Page gone');
-    return null;
+    // Return a failure.
+    return {
+      success: false,
+      error: 'text',
+      message: 'No text specified'
+    };
   }
 };
 // Returns a string with any final slash removed.
@@ -495,8 +512,9 @@ const visit = async (act, page, isStrict) => {
           // If the visit fails:
           if (response === 'error') {
             // Give up.
-            console.log(`ERROR: Visits to ${requestedURL} failed`);
-            act.result = `ERROR: Visit to ${requestedURL} failed`;
+            const errorMsg = `ERROR: Attemts to visit ${requestedURL} failed`;
+            console.log(errorMsg);
+            act.result = errorMsg;
             await page.goto('about:blank')
             .catch(error => {
               console.log(`ERROR: Navigation to blank page failed (${error.message})`);
@@ -549,13 +567,21 @@ const isTrue = (object, specs) => {
   }
   return [actual, satisfied];
 };
-// Adds an error result to an act.
+// Adds a wait error result to an act.
 const waitError = (page, act, error, what) => {
   console.log(`ERROR waiting for ${what} (${error.message})`);
   act.result = {url: page.url()};
   act.result.error = `ERROR waiting for ${what}`;
   return false;
 };
+// Waits.
+const wait = ms => {
+  return new Promise(resolve => {
+    setTimeout(() => {
+      resolve('');
+    }, ms);
+  });
+};
 // Recursively performs the acts in a report.
 const doActs = async (report, actIndex, page) => {
   const {acts} = report;
@@ -590,12 +616,12 @@ const doActs = async (report, actIndex, page) => {
         if (truth[1]) {
           // If the performance of commands is to stop:
           if (act.jump === 0) {
-            // Set the command index to cause a stop.
+            // Set the act index to cause a stop.
             actIndex = -2;
           }
           // Otherwise, if there is a numerical jump:
           else if (act.jump) {
-            // Set the command index accordingly.
+            // Set the act index accordingly.
             actIndex += act.jump - 1;
           }
           // Otherwise, if there is a named next command:
@@ -623,58 +649,65 @@ const doActs = async (report, actIndex, page) => {
         else if (act.type === 'wait') {
           const {what, which} = act;
           console.log(`>> for ${what} to include “${which}”`);
-          // Wait 5 seconds for the specified text to appear in the specified place.
-          let successJSHandle;
+          // Wait 5 or 10 seconds for the specified text, and quit if it does not appear.
           if (act.what === 'url') {
-            successJSHandle = await page.waitForFunction(
-              text => document.URL.includes(text), act.which, {timeout: 5000}
-            )
-            .catch(error => waitError(page, act, error, 'URL'));
+            await page.waitForURL(act.which, {timeout: 15000})
+            .catch(error => {
+              actIndex = -2;
+              waitError(page, act, error, 'URL');
+            });
           }
           else if (act.what === 'title') {
-            successJSHandle = await page.waitForFunction(
-              text => document.title.includes(text), act.which, {timeout: 5000}
+            await page.waitForFunction(
+              text => document && document.title && document.title.includes(text),
+              act.which,
+              {
+                polling: 1000,
+                timeout: 5000
+              }
             )
-            .catch(error => waitError(page, act, error, 'title'));
+            .catch(error => {
+              actIndex = -2;
+              waitError(page, act, error, 'title');
+            });
           }
           else if (act.what === 'body') {
-            successJSHandle = await page.waitForFunction(
-              matchText => {
-                const innerText = document.body.innerText;
-                return innerText.includes(matchText);
-              }, which, {timeout: 5000}
+            await page.waitForFunction(
+              text => document && document.body && document.body.innerText.includes(text),
+              act.which,
+              {
+                polling: 2000,
+                timeout: 10000
+              }
             )
-            .catch(error => waitError(page, act, error, 'body'));
+            .catch(async error => {
+              actIndex = -2;
+              waitError(page, act, error, 'body');
+            });
           }
-          if (successJSHandle) {
+          // If the text was found:
+          if (actIndex > -2) {
+            // Add this to the report.
             act.result = {url: page.url()};
             if (act.what === 'title') {
               act.result.title = await page.title();
             }
-            await page.waitForLoadState('networkidle', {timeout: 10000})
-            .catch(error => {
-              console.log(`ERROR waiting for stability after ${act.what} (${error.message})`);
-              act.result.error = `ERROR waiting for stability after ${act.what}`;
-            });
           }
         }
         // Otherwise, if the act is a wait for a state:
         else if (act.type === 'state') {
-          // If the state is valid:
+          // Wait for it, and quit if it does not appear.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
-          if (stateIndex !== -1) {
-            // Wait for it.
-            await page.waitForLoadState(
-              ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 5000][stateIndex]}
-            )
-            .catch(error => {
-              console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
-              act.result = `ERROR waiting for page to be ${act.which}`;
-            });
-          }
-          else {
-            console.log('ERROR: invalid state');
-            act.result = 'ERROR: invalid state';
+          await page.waitForLoadState(
+            ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 5000][stateIndex]}
+          )
+          .catch(error => {
+            console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
+            act.result = `ERROR waiting for page to be ${act.which}`;
+            actIndex = -2;
+          });
+          if (actIndex > -2) {
+            act.result = `Page became ${act.which}`;
           }
         }
         // Otherwise, if the act is a page switch:
@@ -864,48 +897,34 @@ const doActs = async (report, actIndex, page) => {
             // Otherwise, if the act is a move:
             else if (moves[act.type]) {
               const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
-              // Identify the element to perform the move on.
-              const whichElement = await matchElement(page, selector, act.which || '', act.index);
-              // If there were enough candidates but no text match:
-              if (Array.isArray(whichElement)) {
-                // Add the result to the act.
-                act.result = {
-                  candidateCount: whichElement.length,
-                  error: 'ERROR: no element with matching text found',
-                  candidateTexts: whichElement
-                };
-              }
-              // Otherwise, if the body did not contain the text:
-              else if (whichElement === -1) {
-                // Add the failure to the act.
-                act.result = 'ERROR: body did not contain text to match';
-              }
-              // Otherwise, if there were not enough candidates:
-              else if (typeof whichElement === 'number') {
-                // Add the failure to the act.
-                act.result = {
-                  candidateCount: whichElement,
-                  error: 'ERROR: too few such elements to allow a match'
-                };
+              // Wait 10 seconds until the element to perform the move on is identified.
+              let matchResult = {success: false};
+              let tries = 0;
+              while (tries++ < 5 && ! matchResult.success) {
+                matchResult = await matchElement(page, selector, act.which || '', act.index);
+                if (! matchResult.success) {
+                  await wait(2000);
+                }
               }
-              // Otherwise, if a match was found:
-              else if (whichElement !== null) {
+              // If a match was found:
+              if (matchResult.success) {
+                const {matchingElement} = matchResult;
                 // If the move is a button click, perform it.
                 if (act.type === 'button') {
-                  await whichElement.click({timeout: 3000});
+                  await matchingElement.click({timeout: 3000});
                   act.result = 'clicked';
                 }
                 // Otherwise, if it is checking a radio button or checkbox, perform it.
                 else if (['checkbox', 'radio'].includes(act.type)) {
-                  await whichElement.waitForElementState('stable', {timeout: 2000})
+                  await matchingElement.waitForElementState('stable', {timeout: 2000})
                   .catch(error => {
                     console.log(`ERROR waiting for stable ${act.type} (${error.message})`);
                     act.result = `ERROR waiting for stable ${act.type}`;
                   });
                   if (! act.result) {
-                    const isEnabled = await whichElement.isEnabled();
+                    const isEnabled = await matchingElement.isEnabled();
                     if (isEnabled) {
-                      await whichElement.check({
+                      await matchingElement.check({
                         force: true,
                         timeout: 2000
                       })
@@ -926,28 +945,53 @@ const doActs = async (report, actIndex, page) => {
                 }
                 // Otherwise, if it is focusing the element, perform it.
                 else if (act.type === 'focus') {
-                  await whichElement.focus({timeout: 2000});
+                  await matchingElement.focus({timeout: 2000});
                   act.result = 'focused';
                 }
                 // Otherwise, if it is clicking a link, perform it.
                 else if (act.type === 'link') {
-                  const href = await whichElement.getAttribute('href');
-                  const target = await whichElement.getAttribute('target');
-                  await whichElement.click({timeout: 2000});
-                  act.result = {
-                    href: href || 'NONE',
-                    target: target || 'NONE',
-                    move: 'clicked'
-                  };
+                  const href = await matchingElement.getAttribute('href');
+                  const target = await matchingElement.getAttribute('target');
+                  await matchingElement.click({timeout: 2000})
+                  .catch(async () => {
+                    console.log('ERROR: First attempt to click link timed out');
+                    await matchingElement.click({
+                      force: true,
+                      timeout: 10000
+                    })
+                    .catch(() => {
+                      actIndex = -2;
+                      console.log('ERROR: Second (forced) attempt to click link timed out');
+                      act.result = 'ERROR: Normal and forced click attempts timed out';
+                    });
+                  });
+                  if (actIndex > -2) {
+                    act.result = {
+                      href: href || 'NONE',
+                      target: target || 'NONE',
+                      move: 'clicked'
+                    };
+                  }
                 }
                 // Otherwise, if it is selecting an option in a select list, perform it.
                 else if (act.type === 'select') {
-                  await whichElement.selectOption({what: act.what});
-                  const optionText = await whichElement.$eval(
-                    'option:selected', el => el.textContent
-                  );
+                  const options = await matchingElement.$$('option');
+                  let optionText = '';
+                  if (options && Array.isArray(options) && options.length) {
+                    const optionTexts = [];
+                    for (const option of options) {
+                      const optionText = await option.textContent();
+                      optionTexts.push(optionText);
+                    }
+                    const matchTexts = optionTexts.map((text, index) => text.includes(act.what) ? index : -1);
+                    const index = matchTexts.filter(text => text > -1)[act.index || 0];
+                    if (index !== undefined) {
+                      await matchingElement.selectOption({index});
+                      optionText = optionTexts[index];
+                    }
+                  }
                   act.result = optionText
-                    ? `&ldquo;${optionText}}&rdquo; selected`
+                    ? `“${optionText}” selected`
                     : 'ERROR: option not found';
                 }
                 // Otherwise, if it is entering text on the element:
@@ -961,19 +1005,23 @@ const doActs = async (report, actIndex, page) => {
                     what = what.replace(/__[A-Z]+__/, envValue);
                   }
                   // Enter the text.
-                  await whichElement.type(act.what);
+                  await matchingElement.type(act.what);
                   report.presses += act.what.length;
                   act.result = 'entered';
                 }
                 // Otherwise, i.e. if the move is unknown, add the failure to the act.
                 else {
                   // Report the error.
-                  act.result = 'ERROR: move unknown';
+                  const report = 'ERROR: move unknown';
+                  act.result = report;
+                  console.log(report);
                 }
               }
-              // Otherwise, i.e. if the page was gone:
+              // Otherwise, i.e. if no match was found:
               else {
-                act.result = 'ERROR: page gone, so matching element not found';
+                const report = 'ERROR: Specified element not found';
+                act.result = report;
+                console.log(report);
               }
             }
             // Otherwise, if the act is a keypress:
@@ -1161,7 +1209,9 @@ const doActs = async (report, actIndex, page) => {
     // Otherwise, i.e. if the command is invalid:
     else {
       // Add an error result to the act.
-      act.result = `ERROR: Invalid command of type ${act.type}`;
+      const errorMsg = `ERROR: Invalid command of type ${act.type}`;
+      act.result = errorMsg;
+      console.log(errorMsg);
     }
     // Perform the remaining acts.
     await doActs(report, actIndex + 1, page);
@@ -1247,8 +1297,10 @@ exports.handleRequest = async report => {
     report.timeStamp = report.id.replace(/-.+/, '');
     // Add the script commands to the report as its initial acts.
     report.acts = JSON.parse(JSON.stringify(report.script.commands));
-    // Inject url acts where necessary to undo DOM changes.
-    injectURLActs(report.acts);
+    // Inject url acts where necessary to undo DOM changes, if specified.
+    if (urlInject === 'yes') {
+      injectURLActs(report.acts);
+    }
     // Perform the acts, asynchronously adding to the log and report.
     await doScript(report);
   }

package/samples/batches/a11yorgs.json

@@ -1,6 +1,6 @@
 {
   "id": "a11yorgs",
-  "what": "Pages expected to pass all accessibility tests",
+  "what": "Home pages of accessibility organizations",
   "hosts": [
     {
       "id": "iaap",

package/samples/batches/eurail.json

@@ -0,0 +1,11 @@
+{
+  "id": "eurail",
+  "what": "Eurail",
+  "hosts": [
+    {
+      "id": "eurail",
+      "which": "https://www.eurail.com/en",
+      "what": "Eurail"
+    }
+  ]
+}

package/samples/scripts/digiCert.json

@@ -0,0 +1,63 @@
+{
+  "id": "digicert",
+  "what": "moves on DigiCert website",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chrome"
+    },
+    {
+      "type": "url",
+      "which": "https://order.digicert.com/",
+      "what": "DigiCert"
+    },
+    {
+      "type": "focus",
+      "what": "button",
+      "which": "Choose payment plan"
+    },
+    {
+      "type": "press",
+      "which": "ArrowDown",
+      "again": 2,
+      "what": "Choose 2-year plan"
+    },
+    {
+      "type": "press",
+      "which": "Enter",
+      "what": "Commit to choose 2-year plan"
+    },
+    {
+      "type": "button",
+      "which": "I don't have",
+      "what": "I don’t have my CSR"
+    },
+    {
+      "type": "select",
+      "which": "Server app details",
+      "what": "NGINX"
+    },
+    {
+      "type": "text",
+      "which": "main website",
+      "what": "jpdev.pro"
+    },
+    {
+      "type": "button",
+      "which": "Continue",
+      "what": "Continue"
+    },
+    {
+      "type": "wait",
+      "what": "url",
+      "which": "step2"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    }
+  ]
+}

package/tests/aatt.js

@@ -53,13 +53,23 @@ exports.reporter = async (page, waitLong) => {
     }
     catch (error) {
       console.log(`ERROR processing AATT report (${error.message})`);
-      return {result: 'ERROR processing AATT report'};
+      return {
+        result: {
+          prevented: true,
+          error: 'ERROR processing AATT report'
+        }
+      };
     }
   }
   // Otherwise, i.e. if the result did not arrive within the time limit:
   else {
     // Report the failure.
     console.log('ERROR: getting report took too long');
-    return {result: 'ERROR: getting AATT report took too long'};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: getting AATT report took too long'
+      }
+    };
   }
 };

package/tests/alfa.js

@@ -4,11 +4,13 @@
 */
 
 // IMPORTS
+
 const {Audit} = require('@siteimprove/alfa-act');
 const {Scraper} = require('@siteimprove/alfa-scraper');
 const alfaRules = require('@siteimprove/alfa-rules');
 
 // FUNCTIONS
+
 // Conducts and reports an alfa test.
 exports.reporter = async page => {
   // Get the document containing the summaries of the alfa rules.
@@ -104,6 +106,11 @@ exports.reporter = async page => {
   }
   catch(error) {
     console.log(`ERROR: navigation to URL timed out (${error})`);
-    return {result: {error: 'ERROR: navigation to URL timed out'}};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: navigation to URL timed out'
+      }
+    };
   }
 };

package/tests/axe.js

@@ -16,10 +16,11 @@ exports.reporter = async (page, withItems, rules = []) => {
   await injectAxe(page)
   .catch(error => {
     console.log(`ERROR: Axe injection failed (${error.message})`);
-    data.result = 'ERROR: axe injection failed';
+    data.prevented = true;
+    data.error = 'ERROR: axe injection failed';
   });
   // If the injection succeeded:
-  if (! data.result) {
+  if (! data.prevented) {
     // Get the data on the elements violating the specified axe-core rules.
     const axeOptions = {};
     if (rules.length) {
@@ -100,6 +101,7 @@ exports.reporter = async (page, withItems, rules = []) => {
     // Otherwise, i.e. if the test failed:
     else {
       // Report this.
+      data.prevented = true;
       data.error = 'ERROR: axe failed';
       console.log('ERROR: axe failed');
     }

package/tests/bulk.js

@@ -12,6 +12,7 @@ exports.reporter = async page => {
   await page.waitForSelector('body', {timeout: 10000})
   .catch(error => {
     console.log(`ERROR (${error.message})`);
+    data.prevented = true;
     data.error = 'ERROR: bulk timed out';
     return {result: data};
   });

package/tests/focOp.js

@@ -127,6 +127,7 @@ exports.reporter = async (page, withItems) => {
   }, withItems)
   .catch(error => {
     console.log(`ERROR getting focOp data (${error.message})`);
+    data.prevented = true;
   });
   return {result: data};
 };

package/tests/hover.js

@@ -212,6 +212,7 @@ exports.reporter = async (page, withItems) => {
   const triggers = await page.$$(selectors.join(', '))
   .catch(error => {
     console.log(`ERROR getting hover triggers (${error.message})`);
+    data.prevented = true;
     return [];
   });
   // If they number more than the sample size limit, sample them.

package/tests/ibm.js

@@ -48,10 +48,12 @@ const report = (ibmReport, withItems) => {
       }
     }
     else {
+      data.prevented = true;
       data.error = 'ERROR: ibm test delivered no totals';
     }
   }
   else {
+    data.prevented = true;
     data.error = 'ERROR: ibm test delivered no report summary';
   }
   return data;
@@ -88,6 +90,7 @@ const doTest = async (content, withItems, timeLimit) => {
   else {
     console.log('ERROR: getting ibm test report took too long');
     return {
+      prevented: true,
       error: 'ERROR: getting ibm test report took too long'
     };
   }
@@ -100,12 +103,18 @@ exports.reporter = async (page, withItems, withNewContent) => {
     const timeLimit = 15;
     const typeContent = await page.content();
     result.content = await doTest(typeContent, withItems, timeLimit);
+    if (result.content.prevented) {
+      result.prevented = true;
+    }
   }
   // If a test with new content is to be performed:
   if ([true, undefined].includes(withNewContent)) {
     const timeLimit = 20;
     const typeContent = page.url();
     result.url = await doTest(typeContent, withItems, timeLimit);
+    if (result.content.prevented) {
+      result.prevented = true;
+    }
   }
   return {result};
 };

package/tests/labClash.js

@@ -151,7 +151,10 @@ exports.reporter = async (page, withItems) => {
   }, withItems)
   .catch(error => {
     console.log(`ERROR: labClash failed (${error.message})`);
-    const data = {error: 'ERROR: labClash failed'};
+    const data = {
+      prevented: true,
+      error: 'ERROR: labClash failed'
+    };
     return {result: data};
   });
 };

package/tests/motion.js

@@ -110,6 +110,11 @@ exports.reporter = async (page, delay, interval, count) => {
   // Otherwise, i.e. if the shooting failed:
   else {
     // Return failure.
-    return {result: {error: 'ERROR: screenshots failed'}};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: screenshots failed'
+      }
+    };
   }
 };

package/tests/styleDiff.js

@@ -11,11 +11,6 @@ exports.reporter = async (page, withItems) => {
   return await page.$eval('body', (body, args) => {
     const withItems = args[0];
     const linkTypes = args[1];
-    // Initialize the data to be returned.
-    const data = {totals: {}};
-    if (withItems) {
-      data.items = {};
-    }
     // Identify the settable style properties to be compared for all tag names.
     const mainStyles = [
       'borderStyle',
@@ -39,6 +34,15 @@ exports.reporter = async (page, withItems) => {
     const headingStyles = [
       'fontSize'
     ];
+    // Initialize the data to be returned.
+    const data = {
+      mainStyles,
+      headingStyles,
+      totals: {}
+    };
+    if (withItems) {
+      data.items = {};
+    }
     // Identify the heading tag names to be analyzed.
     const headingNames = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'];
     // Identify the other nonlink tag names to be analyzed.

package/tests/tabNav.js

@@ -3,66 +3,71 @@
   This test reports nonstandard keyboard navigation among tab elements in tab lists.
   Standards are based on https://www.w3.org/TR/wai-aria-practices-1.1/#tabpanel.
 */
-exports.reporter = async (page, withItems) => {
-  // Initialize a report.
-  const data = {
-    totals: {
-      navigations: {
-        all: {
+
+// CONSTANTS
+
+const data = {
+  totals: {
+    navigations: {
+      all: {
+        total: 0,
+        correct: 0,
+        incorrect: 0
+      },
+      specific: {
+        tab: {
           total: 0,
           correct: 0,
           incorrect: 0
         },
-        specific: {
-          tab: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          left: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          right: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          up: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          down: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          home: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          end: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          }
+        left: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        right: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        up: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        down: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        home: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        end: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
         }
-      },
-      tabElements: {
-        total: 0,
-        correct: 0,
-        incorrect: 0
-      },
-      tabLists: {
-        total: 0,
-        correct: 0,
-        incorrect: 0
       }
+    },
+    tabElements: {
+      total: 0,
+      correct: 0,
+      incorrect: 0
+    },
+    tabLists: {
+      total: 0,
+      correct: 0,
+      incorrect: 0
     }
-  };
+  }
+};
+
+// FUNCTIONS
+
+exports.reporter = async (page, withItems) => {
   if (withItems) {
     data.tabElements = {
       incorrect: [],
@@ -182,6 +187,7 @@ exports.reporter = async (page, withItems) => {
           .catch(error => {
             console.log(`ERROR: could not get tag name (${error.message})`);
             found = false;
+            data.prevented = true;
             return 'ERROR: not found';
           });
           if (found) {
@@ -260,7 +266,10 @@ exports.reporter = async (page, withItems) => {
         if (! orientation) {
           orientation = 'horizontal';
         }
-        if (orientation !== 'ERROR') {
+        if (orientation === 'ERROR') {
+          data.prevented = true;
+        }
+        else {
           const tabs = await firstTabList.$$('[role=tab]');
           // If the tablist contains at least 2 tab elements:
           if (tabs.length > 1) {

package/tests/tenon.js

@@ -107,15 +107,19 @@ exports.reporter = async (tenonData, id) => {
     // Otherwise, if the test is still running after a wait for its status:
     else {
       // Report the test status.
-      return {result: {
-        error: 'ERROR: test status not completed',
-        testStatus
-      }};
+      return {
+        result: {
+          prevented: true,
+          error: 'ERROR: test status not completed',
+          testStatus
+        }
+      };
     }
   }
   else {
     return {
       result: {
+        prevented: true,
         error: 'ERROR: tenon authorization and test data incomplete'
       }
     };

package/tests/wave.js

@@ -32,6 +32,7 @@ exports.reporter = async (page, reportType) => {
           }
           catch (error) {
             return resolve({
+              prevented: true,
               error: 'WAVE did not return JSON.',
               report
             });