testaro 2.2.0 → 2.2.3

package/README.md

@@ -377,6 +377,8 @@ The second item in each array, if there are 3 items in the array, is an operator
 - `>`: greater than
 - `!`: unequal to
 
+A typical use for an `expect` property is checking the correctness of a Testaro test. Thus, the validation scripts in the `validation/tests/scripts` directory all contain `test` commands with `expect` properties. See the “Validation” section below.
+
 ## Batches
 
 There are two ways to use a script to give instructions to Testaro:
@@ -447,6 +449,8 @@ The executors are:
 
 To execute any executor `xyz.js`, call it with the statement `node validation/executors/xyz`. The results will appear in the standard output.
 
+The `tests.js` executor makes use of the scripts in the `validation/tests/scripts` directory, and they, in turn, run tests on HTML files in the `validation/tests/targets` directory.
+
 ## Contribution
 
 You can define additional Testaro commands and functionality. Contributions are welcome.

package/index.js

@@ -1121,7 +1121,7 @@ const doBatch = async (options, reportTemplate, hostIndex = 0) => {
   const host = hosts[hostIndex];
   // If the specified host exists:
   if (host) {
-    // Copy the report for it.
+    // Create a report for it.
     const hostReport = JSON.parse(JSON.stringify(reportTemplate));
     // Copy the properties of the specified host to all url acts.
     hostReport.acts.forEach(act => {
@@ -1132,6 +1132,18 @@ const doBatch = async (options, reportTemplate, hostIndex = 0) => {
     });
     // Perform the commands on the host.
     await doScript(options, hostReport);
+    // Add the host’s ID to the host report.
+    hostReport.hostName = host.id;
+    // Add data from the template to the host report.
+    hostReport.orderName = reportTemplate.id;
+    hostReport.id = `${hostReport.orderName}-${host.id}`;
+    hostReport.orderUserName = reportTemplate.userName;
+    hostReport.orderTime = reportTemplate.orderTime;
+    hostReport.scriptName = reportTemplate.scriptName;
+    hostReport.batchName = reportTemplate.batchName;
+    hostReport.scriptIsValid = reportTemplate.scriptIsValid;
+    hostReport.batchIsValid = reportTemplate.batchIsValid;
+    hostReport.host = host;
     // Process the remaining hosts.
     await doBatch(options, reportTemplate, hostIndex + 1);
   }
@@ -1210,6 +1222,7 @@ exports.handleRequest = async options => {
     }
     // Create a report template, containing a copy of the commands as its acts.
     const reportTemplate = {
+      host: '',
       acts: JSON.parse(JSON.stringify(options.script.commands))
     };
     // Inject url acts where necessary to undo DOM changes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "2.2.0",
+  "version": "2.2.3",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/tests/hover.js

@@ -8,188 +8,226 @@
   examined are the descendants of the grandparent of the element hovered over if that element
   has the tag name 'A' or 'BUTTON' or otherwise the descendants of the element. The only
   elements counted as being made visible by hovering are those with tag names 'A', 'BUTTON',
-  'INPUT', and 'SPAN', and those with 'role="menuitem"' attributes.
+  'INPUT', and 'SPAN', and those with 'role="menuitem"' attributes. The test waits 700 ms after
+  each hover in case of delayed effects. Despite this delay, the test makes the execution time
+  practical by randomly sampling targets instead of hovering over all of them. Therefore, the
+  results may vary from one execution to another.
 */
-exports.reporter = async (page, withItems) => {
-  // Initialize a counter.
-  let elementsChecked = 0;
-  // Identify the elements that are likely to trigger disclosures on hover.
-  const triggers = await page.$$(
-    'body a:visible, body button:visible, body li:visible, body [onmouseenter]:visible, body [onmouseover]:visible'
-  )
-  .catch(error => {
-    console.log(`ERROR getting hover triggers (${error.message})`);
-    return [];
-  });
-  // Identify the selectors of active elements likely to be disclosed by a hover.
-  const targetSelectors = ['a', 'button', 'input', '[role=menuitem]', 'span']
-  .map(selector => `${selector}:visible`)
-  .join(', ');
-  // Initialize the result.
-  const data = {
-    totals: {
-      triggers: 0,
-      madeVisible: 0,
-      opacityChanged: 0,
-      opacityAffected: 0,
-      unhoverables: 0
+
+// CONSTANTS
+
+// Selectors of active elements likely to be disclosed by a hover.
+const targetSelectors = ['a', 'button', 'input', '[role=menuitem]', 'span']
+.map(selector => `${selector}:visible`)
+.join(', ');
+// Initialize the result.
+const data = {
+  totals: {
+    triggers: 0,
+    madeVisible: 0,
+    opacityChanged: 0,
+    opacityAffected: 0,
+    unhoverables: 0
+  }
+};
+
+// VARIABLES
+
+// Counter.
+let elementsChecked = 0;
+
+// FUNCTIONS
+
+// Samples a population and returns the sample.
+const getSample = (population, sampleSize) => {
+  const popSize = population.length;
+  if (sampleSize > 0 && sampleSize < popSize) {
+    const sample = new Set();
+    while (sample.size < sampleSize) {
+      const index = Math.floor(popSize * Math.random());
+      sample.add(population[index]);
     }
-  };
+    return Array.from(sample);
+  }
+  else {
+    return [];
+  }
+};
+// Recursively finds and reports triggers and targets.
+const find = async (withItems, page, triggers) => {
   if (withItems) {
     data.items = {
       triggers: [],
       unhoverables: []
     };
   }
-  let triggerTag = '';
-  // FUNCTION DEFINITION START
-  // Recursively finds and reports triggers and targets.
-  const find = async triggers => {
-    // If any potential disclosure triggers remain:
-    if (triggers.length) {
-      // Identify the first of them.
-      const firstTrigger = triggers[0];
-      const tagNameJSHandle = await firstTrigger.getProperty('tagName')
-      .catch(error => {
-        console.log(`ERROR getting trigger tag name (${error.message})`);
-        return '';
-      });
-      if (tagNameJSHandle) {
-        const tagName = await tagNameJSHandle.jsonValue();
-        // Identify the root of a subtree likely to contain disclosed elements.
-        let root = firstTrigger;
-        if (['A', 'BUTTON'].includes(tagName)) {
-          const rootJSHandle = await page.evaluateHandle(
-            firstTrigger => {
-              const parent = firstTrigger.parentElement;
-              if (parent) {
-                return parent.parentElement || parent;
-              }
-              else {
-                return firstTrigger;
-              }
-            },
-            firstTrigger
-          );
-          root = rootJSHandle.asElement();
-        }
-        // Identify the visible active descendants of the root before the hover.
-        const preVisibles = await root.$$(targetSelectors);
-        // Identify all the descendants of the root.
-        const descendants = await root.$$('*');
-        // Identify their opacities before the hover.
-        const preOpacities = await page.evaluate(
-          elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-        );
-        try {
-          // Hover over the potential trigger.
-          await firstTrigger.hover({timeout: 700});
-          // Identify whether it controls other elements.
-          const isController = await page.evaluate(
-            element => element.ariaHasPopup || element.hasAttribute('aria-controls'), firstTrigger
-          );
-          // Wait for any delayed and/or slowed hover reaction if likely.
-          await page.waitForTimeout(
-            elementsChecked++ < 10 || tagName !== triggerTag || isController ? 1200 : 200
-          );
-          await root.waitForElementState('stable');
-          // Identify the visible active descendants of the root during the hover.
-          const postVisibles = await root.$$(targetSelectors);
-          // Identify the opacities of the descendants of the root during the hover.
-          const postOpacities = await page.evaluate(
-            elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-          );
-          // Identify the elements with opacity changes.
-          const opacityTargets = descendants
-          .filter((descendant, index) => postOpacities[index] !== preOpacities[index]);
-          // Count them and their descendants.
-          const opacityAffected = opacityTargets.length
-            ? await page.evaluate(elements => elements.reduce(
-              (total, current) => total + 1 + current.querySelectorAll('*').length, 0
-            ), opacityTargets)
-            : 0;
-          // If hovering disclosed any element or changed any opacity:
-          if (postVisibles.length > preVisibles.length || opacityAffected) {
-            // Preserve the lengthened reaction wait, if any, for the next 5 tries.
-            if (elementsChecked < 11) {
-              elementsChecked = 5;
+  // If any potential disclosure triggers remain:
+  if (triggers.length) {
+    // Identify the first of them.
+    const firstTrigger = triggers[0];
+    const tagNameJSHandle = await firstTrigger.getProperty('tagName')
+    .catch(error => {
+      console.log(`ERROR getting trigger tag name (${error.message})`);
+      return '';
+    });
+    if (tagNameJSHandle) {
+      const tagName = await tagNameJSHandle.jsonValue();
+      // Identify the root of a subtree likely to contain disclosed elements.
+      let root = firstTrigger;
+      if (['A', 'BUTTON'].includes(tagName)) {
+        const rootJSHandle = await page.evaluateHandle(
+          firstTrigger => {
+            const parent = firstTrigger.parentElement;
+            if (parent) {
+              return parent.parentElement || parent;
             }
-            // Hover over the upper-left corner of the page, to undo any hover reactions.
-            await page.hover('body', {
-              position: {
-                x: 0,
-                y: 0
-              }
-            });
-            // Wait for any delayed and/or slowed hover reaction.
-            await page.waitForTimeout(200);
-            await root.waitForElementState('stable');
-            // Increment the counts of triggers and targets.
-            data.totals.triggers++;
-            const madeVisible = Math.max(0, postVisibles.length - preVisibles.length);
-            data.totals.madeVisible += madeVisible;
-            data.totals.opacityChanged += opacityTargets.length;
-            data.totals.opacityAffected += opacityAffected;
-            // If details are to be reported:
-            if (withItems) {
-              // Report them.
-              const triggerDataJSHandle = await page.evaluateHandle(args => {
-                // Returns the text of an element.
-                const textOf = (element, limit) => {
-                  const text = element.textContent.trim() || element.outerHTML.trim();
-                  return text.replace(/\s{2,}/sg, ' ').slice(0, limit);
-                };
-                const trigger = args[0];
-                const preVisibles = args[1];
-                const postVisibles = args[2];
-                const madeVisible = postVisibles
-                .filter(el => ! preVisibles.includes(el))
-                .map(el => ({
-                  tagName: el.tagName,
-                  text: textOf(el, 50)
-                }));
-                const opacityChanged = args[3].map(el => ({
-                  tagName: el.tagName,
-                  text: textOf(el, 50)
-                }));
-                return {
-                  tagName: trigger.tagName,
-                  id: trigger.id || '',
-                  text: textOf(trigger, 50),
-                  madeVisible,
-                  opacityChanged
-                };
-              }, [firstTrigger, preVisibles, postVisibles, opacityTargets]);
-              const triggerData = await triggerDataJSHandle.jsonValue();
-              data.items.triggers.push(triggerData);
+            else {
+              return firstTrigger;
             }
+          },
+          firstTrigger
+        );
+        root = rootJSHandle.asElement();
+      }
+      // Identify the visible active descendants of the root before the hover.
+      const preVisibles = await root.$$(targetSelectors);
+      // Identify all the descendants of the root.
+      const descendants = await root.$$('*');
+      // Identify their opacities before the hover.
+      const preOpacities = await page.evaluate(
+        elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
+      );
+      try {
+        // Hover over the potential trigger.
+        await firstTrigger.hover({timeout: 700});
+        // Identify whether it is coded as controlling other elements.
+        const isController = await page.evaluate(
+          element => element.ariaHasPopup || element.hasAttribute('aria-controls'), firstTrigger
+        );
+        // Wait for any delayed and/or slowed hover reaction, longer if coded as a controller.
+        await page.waitForTimeout(isController ? 1200 : 600);
+        await root.waitForElementState('stable');
+        // Identify the visible active descendants of the root during the hover.
+        const postVisibles = await root.$$(targetSelectors);
+        // Identify the opacities of the descendants of the root during the hover.
+        const postOpacities = await page.evaluate(
+          elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
+        );
+        // Identify the elements with opacity changes.
+        const opacityTargets = descendants
+        .filter((descendant, index) => postOpacities[index] !== preOpacities[index]);
+        // Count them and their descendants.
+        const opacityAffected = opacityTargets.length
+          ? await page.evaluate(elements => elements.reduce(
+            (total, current) => total + 1 + current.querySelectorAll('*').length, 0
+          ), opacityTargets)
+          : 0;
+        // If hovering disclosed any element or changed any opacity:
+        if (postVisibles.length > preVisibles.length || opacityAffected) {
+          // Preserve the lengthened reaction wait, if any, for the next 5 tries.
+          if (elementsChecked < 11) {
+            elementsChecked = 5;
           }
-        }
-        catch (error) {
-          console.log('ERROR hovering');
-          // Returns the text of an element.
-          const textOf = async (element, limit) => {
-            let text = await element.textContent();
-            text = text.trim() || await element.innerHTML();
-            return text.trim().replace(/\s*/sg, '').slice(0, limit);
-          };
-          data.totals.unhoverables++;
+          // Hover over the upper-left corner of the page, to undo any hover reactions.
+          await page.hover('body', {
+            position: {
+              x: 0,
+              y: 0
+            }
+          });
+          // Wait for any delayed and/or slowed hover reaction.
+          await page.waitForTimeout(200);
+          await root.waitForElementState('stable');
+          // Increment the counts of triggers and targets.
+          data.totals.triggers++;
+          const madeVisible = Math.max(0, postVisibles.length - preVisibles.length);
+          data.totals.madeVisible += madeVisible;
+          data.totals.opacityChanged += opacityTargets.length;
+          data.totals.opacityAffected += opacityAffected;
+          // If details are to be reported:
           if (withItems) {
-            data.items.unhoverables.push({
-              tagName: tagName,
-              id: firstTrigger.id || '',
-              text: await textOf(firstTrigger, 50)
-            });
+            // Report them.
+            const triggerDataJSHandle = await page.evaluateHandle(args => {
+              // Returns the text of an element.
+              const textOf = (element, limit) => {
+                const text = element.textContent.trim() || element.outerHTML.trim();
+                return text.replace(/\s{2,}/sg, ' ').slice(0, limit);
+              };
+              const trigger = args[0];
+              const preVisibles = args[1];
+              const postVisibles = args[2];
+              const madeVisible = postVisibles
+              .filter(el => ! preVisibles.includes(el))
+              .map(el => ({
+                tagName: el.tagName,
+                text: textOf(el, 50)
+              }));
+              const opacityChanged = args[3].map(el => ({
+                tagName: el.tagName,
+                text: textOf(el, 50)
+              }));
+              return {
+                tagName: trigger.tagName,
+                id: trigger.id || '',
+                text: textOf(trigger, 50),
+                madeVisible,
+                opacityChanged
+              };
+            }, [firstTrigger, preVisibles, postVisibles, opacityTargets]);
+            const triggerData = await triggerDataJSHandle.jsonValue();
+            data.items.triggers.push(triggerData);
           }
         }
-        triggerTag = tagName;
       }
-      // Process the remaining potential triggers.
-      await find(triggers.slice(1));
+      catch (error) {
+        console.log('ERROR hovering');
+        // Returns the text of an element.
+        const textOf = async (element, limit) => {
+          let text = await element.textContent();
+          text = text.trim() || await element.innerHTML();
+          return text.trim().replace(/\s*/sg, '').slice(0, limit);
+        };
+        data.totals.unhoverables++;
+        if (withItems) {
+          data.items.unhoverables.push({
+            tagName: tagName,
+            id: firstTrigger.id || '',
+            text: await textOf(firstTrigger, 50)
+          });
+        }
+      }
     }
-  };
+    // Process the remaining potential triggers.
+    await find(withItems, page, triggers.slice(1));
+  }
+};
+exports.reporter = async (page, withItems) => {
+  // Identify the triggers.
+  const selectors = [
+    'body a:visible',
+    'body button:visible',
+    'body li:visible, body [onmouseenter]:visible',
+    'body [onmouseover]:visible'
+  ];
+  const triggers = await page.$$(selectors.join(', '))
+  .catch(error => {
+    console.log(`ERROR getting hover triggers (${error.message})`);
+    return [];
+  });
+  // If they number more than the sample size limit, sample them.
+  const triggerCount = triggers.length;
+  const sampleSize = 15;
+  const triggerSample = triggerCount > sampleSize ? getSample(triggers, 15) : triggers;
   // Find and document the hover-triggered disclosures.
-  await find(triggers);
+  await find(withItems, page, triggerSample);
+  // If the triggers were sampled:
+  if (triggerCount > sampleSize) {
+    // Change the totals to population estimates.
+    const multiplier = triggerCount / sampleSize;
+    Object.keys(data.totals).forEach(key => {
+      data.totals[key] = Math.round(multiplier * data.totals[key]);
+    });
+  }
   // Return the result.
   return {result: data};
 };

package/validation/executors/tests.js

@@ -4,6 +4,10 @@
 const fs = require('fs').promises;
 const {handleRequest} = require(`${__dirname}/../../index`);
 const validateTests = async () => {
+  const totals = {
+    attempts: 0,
+    successes: 0
+  };
   const scriptFileNames = await fs.readdir(`${__dirname}/../tests/scripts`);
   for (const scriptFileName of scriptFileNames) {
     const rawScriptJSON = await fs
@@ -33,21 +37,28 @@ const validateTests = async () => {
           : true
       )
     ) {
+      totals.attempts++;
+      totals.successes++;
       console.log('Success: Reports have been correctly populated');
       if (reports[0].acts.every(
         act => act.type === 'test' ? act.result.failureCount === 0 : true
       )) {
+        totals.attempts++;
+        totals.successes++;
         console.log('Success: No failures');
       }
       else {
+        totals.attempts++;
         console.log('Failure: At least one test has at least one failure');
         console.log(JSON.stringify(reports, null, 2));
       }
     }
     else {
+      totals.attempts++;
       console.log('Failure: Reports empty or invalid');
       console.log(JSON.stringify(reports, null, 2));
     }
   }
+  console.log(`Grand totals: attempts ${totals.attempts}, successes ${totals.successes}`);
 };
 validateTests();