testaro 4.3.1 → 4.4.0

package/commands.js

@@ -1,7 +1,7 @@
 exports.commands = {
   etc: {
     button: [
-      'Click a button',
+      'Click a button or submit input',
       {
         which: [true, 'string', 'hasLength', 'substring of button text'],
         index: [false, 'number', '', 'index among matches if not 0'],
@@ -94,7 +94,7 @@ exports.commands = {
       {
         which: [true, 'string', 'hasLength', 'substring of select-list text'],
         index: [false, 'number', '', 'index among matches if not 0'],
-        what: [true, 'string', 'hasLength', 'substring of option text']
+        what: [true, 'string', 'hasLength', 'substring of option text content']
       }
     ],
     state: [

package/high.js

@@ -1,5 +1,8 @@
-// high.js
-// Invokes Testaro with the high-level method.
+/*
+  high.js
+  Invokes Testaro with the high-level method.
+  Usage example: node high tp10 weborgs
+*/
 
 const {runJob} = require('./create');
 const scriptID = process.argv[2];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.3.1",
+  "version": "4.4.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -12,12 +12,13 @@ const {commands} = require('./commands');
 const debug = process.env.DEBUG === 'true';
 // Set WAITS environment variable to a positive number to insert delays (in ms).
 const waits = Number.parseInt(process.env.WAITS) || 0;
+const urlInject = process.env.URL_INJECT || 'yes';
 // CSS selectors for targets of moves.
 const moves = {
-  button: 'button',
+  button: 'button, [role=button], input[type=submit]',
   checkbox: 'input[type=checkbox]',
   focus: true,
-  link: 'a',
+  link: 'a, [role=link]',
   radio: 'input[type=radio]',
   select: 'select',
   text: 'input[type=text]'
@@ -291,7 +292,13 @@ const textOf = async (page, element) => {
       // Return its visible labels, descriptions, and legend if the first input in a fieldset.
       totalText = await page.evaluate(element => {
         const {tagName} = element;
-        const ownText = ['A', 'BUTTON'].includes(tagName) ? element.textContent : '';
+        let ownText = '';
+        if (['A', 'BUTTON'].includes(tagName)) {
+          ownText = element.textContent;
+        }
+        else if (tagName === 'INPUT' && element.type === 'submit') {
+          ownText = element.value;
+        }
         // HTML link elements have no labels property.
         const labels = tagName !== 'A' ? Array.from(element.labels) : [];
         const labelTexts = labels.map(label => label.textContent);
@@ -377,10 +384,10 @@ const matchElement = async (page, selector, matchText, index = 0) => {
         return ! matchText || bodyText.includes(matchText);
       },
       [slimText, slimBody],
-      {timeout: 2000}
+      {timeout: 4000}
     )
     .catch(async error => {
-      console.log(`ERROR: text to match not in body (${error.message})`);
+      console.log(`ERROR: text “${matchText}” not in body (${error.message})`);
     });
     // If there is no text to be matched or the body contained it:
     if (textInBodyJSHandle) {
@@ -549,7 +556,7 @@ const isTrue = (object, specs) => {
   }
   return [actual, satisfied];
 };
-// Adds an error result to an act.
+// Adds a wait error result to an act.
 const waitError = (page, act, error, what) => {
   console.log(`ERROR waiting for ${what} (${error.message})`);
   act.result = {url: page.url()};
@@ -623,28 +630,44 @@ const doActs = async (report, actIndex, page) => {
         else if (act.type === 'wait') {
           const {what, which} = act;
           console.log(`>> for ${what} to include “${which}”`);
-          // Wait 5 seconds for the specified text to appear in the specified place.
+          // Wait 5 or 10 seconds for the specified text, and quit if it does not.
           let successJSHandle;
           if (act.what === 'url') {
             successJSHandle = await page.waitForFunction(
               text => document.URL.includes(text), act.which, {timeout: 5000}
             )
-            .catch(error => waitError(page, act, error, 'URL'));
+            .catch(error => {
+              actIndex = acts.length;
+              return waitError(page, act, error, 'URL');
+            });
           }
           else if (act.what === 'title') {
             successJSHandle = await page.waitForFunction(
               text => document.title.includes(text), act.which, {timeout: 5000}
             )
-            .catch(error => waitError(page, act, error, 'title'));
+            .catch(error => {
+              actIndex = acts.length;
+              return waitError(page, act, error, 'title');
+            });
           }
           else if (act.what === 'body') {
             successJSHandle = await page.waitForFunction(
               matchText => {
-                const innerText = document.body.innerText;
-                return innerText.includes(matchText);
-              }, which, {timeout: 5000}
+                const innerText = document && document.body && document.body.innerText;
+                if (innerText) {
+                  return innerText.includes(matchText);
+                }
+                else {
+                  actIndex = acts.length;
+                  console.log('ERROR finding document body');
+                  return false;
+                }
+              }, which, {timeout: 20000}
             )
-            .catch(error => waitError(page, act, error, 'body'));
+            .catch(error => {
+              actIndex = acts.length;
+              return waitError(page, act, error, 'body');
+            });
           }
           if (successJSHandle) {
             act.result = {url: page.url()};
@@ -663,18 +686,20 @@ const doActs = async (report, actIndex, page) => {
           // If the state is valid:
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
           if (stateIndex !== -1) {
-            // Wait for it.
+            // Wait for it, and quit if it does not appear.
             await page.waitForLoadState(
               ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 5000][stateIndex]}
             )
             .catch(error => {
               console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
               act.result = `ERROR waiting for page to be ${act.which}`;
+              actIndex = acts.length;
             });
           }
           else {
             console.log('ERROR: invalid state');
             act.result = 'ERROR: invalid state';
+            actIndex = acts.length;
           }
         }
         // Otherwise, if the act is a page switch:
@@ -933,7 +958,13 @@ const doActs = async (report, actIndex, page) => {
                 else if (act.type === 'link') {
                   const href = await whichElement.getAttribute('href');
                   const target = await whichElement.getAttribute('target');
-                  await whichElement.click({timeout: 2000});
+                  await whichElement.click({timeout: 2000})
+                  .catch(async () => {
+                    await whichElement.click({
+                      force: true,
+                      timeout: 10000
+                    });
+                  });
                   act.result = {
                     href: href || 'NONE',
                     target: target || 'NONE',
@@ -942,10 +973,21 @@ const doActs = async (report, actIndex, page) => {
                 }
                 // Otherwise, if it is selecting an option in a select list, perform it.
                 else if (act.type === 'select') {
-                  await whichElement.selectOption({what: act.what});
-                  const optionText = await whichElement.$eval(
-                    'option:selected', el => el.textContent
-                  );
+                  const options = await whichElement.$$('option');
+                  let optionText = '';
+                  if (options && Array.isArray(options) && options.length) {
+                    const optionTexts = [];
+                    for (const option of options) {
+                      const optionText = await option.textContent();
+                      optionTexts.push(optionText);
+                    }
+                    const matchTexts = optionTexts.map((text, index) => text.includes(act.what) ? index : -1);
+                    const index = matchTexts.filter(text => text > -1)[act.index || 0];
+                    if (index !== undefined) {
+                      await whichElement.selectOption({index});
+                      optionText = optionTexts[index];
+                    }
+                  }
                   act.result = optionText
                     ? `“${optionText}}” selected`
                     : 'ERROR: option not found';
@@ -1247,8 +1289,10 @@ exports.handleRequest = async report => {
     report.timeStamp = report.id.replace(/-.+/, '');
     // Add the script commands to the report as its initial acts.
     report.acts = JSON.parse(JSON.stringify(report.script.commands));
-    // Inject url acts where necessary to undo DOM changes.
-    injectURLActs(report.acts);
+    // Inject url acts where necessary to undo DOM changes, if specified.
+    if (urlInject === 'yes') {
+      injectURLActs(report.acts);
+    }
     // Perform the acts, asynchronously adding to the log and report.
     await doScript(report);
   }

package/samples/batches/a11yorgs.json

@@ -1,6 +1,6 @@
 {
   "id": "a11yorgs",
-  "what": "Pages expected to pass all accessibility tests",
+  "what": "Home pages of accessibility organizations",
   "hosts": [
     {
       "id": "iaap",

package/samples/batches/eurail.json

@@ -0,0 +1,11 @@
+{
+  "id": "eurail",
+  "what": "Eurail",
+  "hosts": [
+    {
+      "id": "eurail",
+      "which": "https://www.eurail.com/en",
+      "what": "Eurail"
+    }
+  ]
+}

package/samples/scripts/digiCert.json

@@ -0,0 +1,63 @@
+{
+  "id": "digicert",
+  "what": "moves on DigiCert website",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chrome"
+    },
+    {
+      "type": "url",
+      "which": "https://order.digicert.com/",
+      "what": "DigiCert"
+    },
+    {
+      "type": "focus",
+      "what": "button",
+      "which": "Choose payment plan"
+    },
+    {
+      "type": "press",
+      "which": "ArrowDown",
+      "again": 2,
+      "what": "Choose 2-year plan"
+    },
+    {
+      "type": "press",
+      "which": "Enter",
+      "what": "Commit to choose 2-year plan"
+    },
+    {
+      "type": "button",
+      "which": "I don't have",
+      "what": "I don’t have my CSR"
+    },
+    {
+      "type": "select",
+      "which": "Server app details",
+      "what": "NGINX"
+    },
+    {
+      "type": "text",
+      "which": "main website",
+      "what": "jpdev.pro"
+    },
+    {
+      "type": "button",
+      "which": "Continue",
+      "what": "Continue"
+    },
+    {
+      "type": "wait",
+      "what": "url",
+      "which": "step2"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    }
+  ]
+}

package/samples/scripts/tp10.json

@@ -1,5 +1,5 @@
 {
-  "id": "ttp10",
+  "id": "tp10",
   "what": "AATT, Alfa, Axe, IBM, Tenon, WAVE, and 16 custom tests",
   "strict": true,
   "commands": [

package/tests/aatt.js

@@ -53,13 +53,23 @@ exports.reporter = async (page, waitLong) => {
     }
     catch (error) {
       console.log(`ERROR processing AATT report (${error.message})`);
-      return {result: 'ERROR processing AATT report'};
+      return {
+        result: {
+          prevented: true,
+          error: 'ERROR processing AATT report'
+        }
+      };
     }
   }
   // Otherwise, i.e. if the result did not arrive within the time limit:
   else {
     // Report the failure.
     console.log('ERROR: getting report took too long');
-    return {result: 'ERROR: getting AATT report took too long'};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: getting AATT report took too long'
+      }
+    };
   }
 };

package/tests/alfa.js

@@ -4,11 +4,13 @@
 */
 
 // IMPORTS
+
 const {Audit} = require('@siteimprove/alfa-act');
 const {Scraper} = require('@siteimprove/alfa-scraper');
 const alfaRules = require('@siteimprove/alfa-rules');
 
 // FUNCTIONS
+
 // Conducts and reports an alfa test.
 exports.reporter = async page => {
   // Get the document containing the summaries of the alfa rules.
@@ -104,6 +106,11 @@ exports.reporter = async page => {
   }
   catch(error) {
     console.log(`ERROR: navigation to URL timed out (${error})`);
-    return {result: {error: 'ERROR: navigation to URL timed out'}};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: navigation to URL timed out'
+      }
+    };
   }
 };

package/tests/axe.js

@@ -16,10 +16,11 @@ exports.reporter = async (page, withItems, rules = []) => {
   await injectAxe(page)
   .catch(error => {
     console.log(`ERROR: Axe injection failed (${error.message})`);
-    data.result = 'ERROR: axe injection failed';
+    data.prevented = true;
+    data.error = 'ERROR: axe injection failed';
   });
   // If the injection succeeded:
-  if (! data.result) {
+  if (! data.prevented) {
     // Get the data on the elements violating the specified axe-core rules.
     const axeOptions = {};
     if (rules.length) {
@@ -100,6 +101,7 @@ exports.reporter = async (page, withItems, rules = []) => {
     // Otherwise, i.e. if the test failed:
     else {
       // Report this.
+      data.prevented = true;
       data.error = 'ERROR: axe failed';
       console.log('ERROR: axe failed');
     }

package/tests/bulk.js

@@ -12,6 +12,7 @@ exports.reporter = async page => {
   await page.waitForSelector('body', {timeout: 10000})
   .catch(error => {
     console.log(`ERROR (${error.message})`);
+    data.prevented = true;
     data.error = 'ERROR: bulk timed out';
     return {result: data};
   });

package/tests/focOp.js

@@ -127,6 +127,7 @@ exports.reporter = async (page, withItems) => {
   }, withItems)
   .catch(error => {
     console.log(`ERROR getting focOp data (${error.message})`);
+    data.prevented = true;
   });
   return {result: data};
 };

package/tests/hover.js

@@ -212,6 +212,7 @@ exports.reporter = async (page, withItems) => {
   const triggers = await page.$$(selectors.join(', '))
   .catch(error => {
     console.log(`ERROR getting hover triggers (${error.message})`);
+    data.prevented = true;
     return [];
   });
   // If they number more than the sample size limit, sample them.

package/tests/ibm.js

@@ -48,10 +48,12 @@ const report = (ibmReport, withItems) => {
       }
     }
     else {
+      data.prevented = true;
       data.error = 'ERROR: ibm test delivered no totals';
     }
   }
   else {
+    data.prevented = true;
     data.error = 'ERROR: ibm test delivered no report summary';
   }
   return data;
@@ -88,6 +90,7 @@ const doTest = async (content, withItems, timeLimit) => {
   else {
     console.log('ERROR: getting ibm test report took too long');
     return {
+      prevented: true,
       error: 'ERROR: getting ibm test report took too long'
     };
   }
@@ -100,12 +103,18 @@ exports.reporter = async (page, withItems, withNewContent) => {
     const timeLimit = 15;
     const typeContent = await page.content();
     result.content = await doTest(typeContent, withItems, timeLimit);
+    if (result.content.prevented) {
+      result.prevented = true;
+    }
   }
   // If a test with new content is to be performed:
   if ([true, undefined].includes(withNewContent)) {
     const timeLimit = 20;
     const typeContent = page.url();
     result.url = await doTest(typeContent, withItems, timeLimit);
+    if (result.content.prevented) {
+      result.prevented = true;
+    }
   }
   return {result};
 };

package/tests/labClash.js

@@ -151,7 +151,10 @@ exports.reporter = async (page, withItems) => {
   }, withItems)
   .catch(error => {
     console.log(`ERROR: labClash failed (${error.message})`);
-    const data = {error: 'ERROR: labClash failed'};
+    const data = {
+      prevented: true,
+      error: 'ERROR: labClash failed'
+    };
     return {result: data};
   });
 };

package/tests/motion.js

@@ -110,6 +110,11 @@ exports.reporter = async (page, delay, interval, count) => {
   // Otherwise, i.e. if the shooting failed:
   else {
     // Return failure.
-    return {result: {error: 'ERROR: screenshots failed'}};
+    return {
+      result: {
+        prevented: true,
+        error: 'ERROR: screenshots failed'
+      }
+    };
   }
 };

package/tests/styleDiff.js

@@ -11,11 +11,6 @@ exports.reporter = async (page, withItems) => {
   return await page.$eval('body', (body, args) => {
     const withItems = args[0];
     const linkTypes = args[1];
-    // Initialize the data to be returned.
-    const data = {totals: {}};
-    if (withItems) {
-      data.items = {};
-    }
     // Identify the settable style properties to be compared for all tag names.
     const mainStyles = [
       'borderStyle',
@@ -39,6 +34,15 @@ exports.reporter = async (page, withItems) => {
     const headingStyles = [
       'fontSize'
     ];
+    // Initialize the data to be returned.
+    const data = {
+      mainStyles,
+      headingStyles,
+      totals: {}
+    };
+    if (withItems) {
+      data.items = {};
+    }
     // Identify the heading tag names to be analyzed.
     const headingNames = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'];
     // Identify the other nonlink tag names to be analyzed.

package/tests/tabNav.js

@@ -3,66 +3,71 @@
   This test reports nonstandard keyboard navigation among tab elements in tab lists.
   Standards are based on https://www.w3.org/TR/wai-aria-practices-1.1/#tabpanel.
 */
-exports.reporter = async (page, withItems) => {
-  // Initialize a report.
-  const data = {
-    totals: {
-      navigations: {
-        all: {
+
+// CONSTANTS
+
+const data = {
+  totals: {
+    navigations: {
+      all: {
+        total: 0,
+        correct: 0,
+        incorrect: 0
+      },
+      specific: {
+        tab: {
           total: 0,
           correct: 0,
           incorrect: 0
         },
-        specific: {
-          tab: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          left: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          right: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          up: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          down: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          home: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          },
-          end: {
-            total: 0,
-            correct: 0,
-            incorrect: 0
-          }
+        left: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        right: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        up: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        down: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        home: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
+        },
+        end: {
+          total: 0,
+          correct: 0,
+          incorrect: 0
         }
-      },
-      tabElements: {
-        total: 0,
-        correct: 0,
-        incorrect: 0
-      },
-      tabLists: {
-        total: 0,
-        correct: 0,
-        incorrect: 0
       }
+    },
+    tabElements: {
+      total: 0,
+      correct: 0,
+      incorrect: 0
+    },
+    tabLists: {
+      total: 0,
+      correct: 0,
+      incorrect: 0
     }
-  };
+  }
+};
+
+// FUNCTIONS
+
+exports.reporter = async (page, withItems) => {
   if (withItems) {
     data.tabElements = {
       incorrect: [],
@@ -182,6 +187,7 @@ exports.reporter = async (page, withItems) => {
           .catch(error => {
             console.log(`ERROR: could not get tag name (${error.message})`);
             found = false;
+            data.prevented = true;
             return 'ERROR: not found';
           });
           if (found) {
@@ -260,7 +266,10 @@ exports.reporter = async (page, withItems) => {
         if (! orientation) {
           orientation = 'horizontal';
         }
-        if (orientation !== 'ERROR') {
+        if (orientation === 'ERROR') {
+          data.prevented = true;
+        }
+        else {
           const tabs = await firstTabList.$$('[role=tab]');
           // If the tablist contains at least 2 tab elements:
           if (tabs.length > 1) {

package/tests/tenon.js

@@ -107,15 +107,19 @@ exports.reporter = async (tenonData, id) => {
     // Otherwise, if the test is still running after a wait for its status:
     else {
       // Report the test status.
-      return {result: {
-        error: 'ERROR: test status not completed',
-        testStatus
-      }};
+      return {
+        result: {
+          prevented: true,
+          error: 'ERROR: test status not completed',
+          testStatus
+        }
+      };
     }
   }
   else {
     return {
       result: {
+        prevented: true,
         error: 'ERROR: tenon authorization and test data incomplete'
       }
     };

package/tests/wave.js

@@ -32,6 +32,7 @@ exports.reporter = async (page, reportType) => {
           }
           catch (error) {
             return resolve({
+              prevented: true,
               error: 'WAVE did not return JSON.',
               report
             });