testaro 4.13.0 → 4.15.0

package/commands.js

@@ -180,7 +180,11 @@ exports.commands = {
     hover: [
       'Perform a hover test',
       {
-        sampleSize: [false, 'number', '', 'number of triggers to sample, if fewer than all'],
+        headSize: [false, 'number', '', 'count of first triggers to sample separately, if any'],
+        headSampleSize: [false, 'number', '', 'size of the head sample to be drawn, if any'],
+        tailSampleSize: [
+          false, 'number', '', 'size of the non-head sample to be drawn, if not all'
+        ],
         withItems: [true, 'boolean']
       }
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.13.0",
+  "version": "4.15.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -289,9 +289,12 @@ const launch = async typeName => {
     browserContext = await browser.newContext(viewport);
     // When a page is added to the browser context:
     browserContext.on('page', page => {
-      // Make its console messages get reported and appear in the Playwright console.
+      // Make abbreviations of its console messages get reported in the Playwright console.
       page.on('console', msg => {
-        const msgText = msg.text();
+        let msgText = msg.text();
+        if (msgText.length > 300) {
+          msgText = `${msgText.slice(0, 150)} ... ${msgText.slice(-150)}`;
+        }
         console.log(`[${msgText}]`);
         const msgTextLC = msgText.toLowerCase();
         const msgLength = msgText.length;

package/tests/aatt.js

@@ -95,7 +95,7 @@ exports.reporter = async (page, waitLong, tryLimit = 4) => {
             warnings
           },
           report: nonNotices,
-          preventionCount: tryLimit - triesLeft - 1
+          preventionCount: tryLimit - triesLeft
         }
       };
     }

package/tests/hover.js

@@ -1,63 +1,71 @@
 /*
   hover
-  This test reports unexpected effects of hovering. The effects include elements that are made
-  visible, elements whose opacities are changed, elements with ancestors whose opacities are
-  changed, and elements that cannot be hovered over. Only Playwright-visible elements in the
-  DOM that have 'A', 'BUTTON', and 'LI' tag names or have 'onmouseenter' or 'onmouseover' attributes
-  are considered as hovering targets. The elements considered when the effects of hovering are
-  examined are the descendants of the grandparent of the element hovered over if that element
-  has the tag name 'A' or 'BUTTON' or otherwise the descendants of the element. The only
-  elements counted as being made visible by hovering are those with tag names 'A', 'BUTTON',
-  'INPUT', and 'SPAN', and those with 'role="menuitem"' attributes. The test waits 700 ms after
-  each hover in case of delayed effects. Despite this delay, the test can make the execution
-  time practical by randomly sampling targets instead of hovering over all of them. When
-  sampling is performed, the results may vary from one execution to another. An element is
-  reported as unhoverable when it fails the Playwright actionability checks for hovering, i.e.
-  when it fails to be attached to the DOM, visible, stable (not or no longer animating), and
-  able to receive events. All target candidates satisfy the first two conditions, so only the
-  last two might fail. Playwright defines the ability to receive events as being the target of
-  an action on the location where the center of the element is, rather than some other element
-  with a higher zIndex value in the same location being the target.
+  This test reports unexpected impacts of hovering. The effects include additions and removals
+  of visible elements, opacity changes, and unhoverable elements. The elements that are
+  subjected to hovering (called “triggers”) are the Playwright-visible elements that have 'A',
+  'BUTTON', or 'LI' tag names or have 'onmouseenter' or 'onmouseover' attributes. When such an
+  element is hovered over, the test examines the impacts on descendants of the great grandparents
+  of the elements with tag names 'A' and 'BUTTON', grandparents of elements with tag name 'LI',
+  and otherwise the descendants of the elements themselves. Four impacts are counted: (1) an
+  element is added or becomes visible, (2) an element is removed or becomes invisible, (3) the
+  opacity of an element changes, and (4) the element is a descendant of an element whose opacity
+  changes. The test checks up to 4 times for hovering impacts at intervals of 0.3 second.
+
+  Despite this delay, the test can make the execution time practical by randomly sampling triggers
+  instead of hovering over all of them. When sampling is performed, the results may vary from one
+  execution to another. Because hover impacts typically occur near the beginning of a page,
+  sampling is governed by three optional parameters (defaults in parentheses):
+    headSize (0): the size of an initial subset of triggers (“head”)
+    headSampleSize (-1): the size of the sample to be drawn from the head
+    tailSampleSize (-1): the size of the sample to be drawn from the remainder of the page
+  A sample size of -1 means that there is no sampling, and the entire population is tested.
+
+  An element is reported as unhoverable when it fails the Playwright actionability checks for
+  hovering, i.e. fails to be attached to the DOM, visible, stable (not or no longer animating), and
+  able to receive events. All triggers satisfy the first two conditions, so only the last two might
+  fail. Playwright defines the ability to receive events as being the target of an action on the
+  location where the center of the element is, rather than some other element with a higher zIndex
+  value in the same location being the target.
 */
 
 // CONSTANTS
 
-// Selectors of active elements likely to be disclosed by a hover.
-const targetSelectors = ['a', 'button', 'input', '[role=menuitem]', 'span']
-.map(selector => `${selector}:visible`)
-.join(', ');
 // Initialize the result.
 const data = {
-  populationSize: 0,
   totals: {
     triggers: 0,
-    madeVisible: 0,
-    opacityChanged: 0,
-    opacityAffected: 0,
+    headTriggers: 0,
+    tailTriggers: 0,
+    impactTriggers: 0,
+    additions: 0,
+    removals: 0,
+    opacityChanges: 0,
+    opacityImpact: 0,
     unhoverables: 0
   }
 };
 
-// VARIABLES
-
-// Counter.
-let elementsChecked = 0;
-
 // FUNCTIONS
 
 // Samples a population and returns the sample.
 const getSample = (population, sampleSize) => {
   const popSize = population.length;
-  if (sampleSize > 0 && sampleSize < popSize) {
-    const sample = new Set();
-    while (sample.size < sampleSize) {
-      const index = Math.floor(popSize * Math.random());
-      sample.add(population[index]);
-    }
-    return Array.from(sample);
+  if (sampleSize === 0) {
+    return [];
+  }
+  else if (sampleSize > 0 && sampleSize < popSize) {
+    const popData = [];
+    for (const trigger of population) {
+      popData.push({
+        trigger,
+        sorter: Math.random()
+      });
+    };
+    popData.sort((a, b) => a.sorter - b.sorter);
+    return popData.slice(0, sampleSize).map(obj => obj.trigger);
   }
   else {
-    return [];
+    return population;
   }
 };
 // Returns the text of an element.
@@ -66,12 +74,12 @@ const textOf = async (element, limit) => {
   text = text.trim() || await element.innerHTML();
   return text.trim().replace(/\s*/sg, '').slice(0, limit);
 };
-// Recursively finds and reports triggers and targets.
-const find = async (withItems, page, triggers) => {
-  // If any potential disclosure triggers remain:
-  if (triggers.length) {
+// Recursively reports impacts of hovering over triggers.
+const find = async (withItems, page, region, sample, popRatio) => {
+  // If any potential triggers remain:
+  if (sample.length) {
     // Identify the first of them.
-    const firstTrigger = triggers[0];
+    const firstTrigger = sample[0];
     const tagNameJSHandle = await firstTrigger.getProperty('tagName')
     .catch(error => {
       console.log(`ERROR getting trigger tag name (${error.message})`);
@@ -79,59 +87,86 @@ const find = async (withItems, page, triggers) => {
     });
     if (tagNameJSHandle) {
       const tagName = await tagNameJSHandle.jsonValue();
-      // Identify the root of a subtree likely to contain disclosed elements.
+      // Identify the root of a subtree likely to contain impacted elements.
       let root = firstTrigger;
-      if (['A', 'BUTTON'].includes(tagName)) {
+      if (['A', 'BUTTON', 'LI'].includes(tagName)) {
         const rootJSHandle = await page.evaluateHandle(
           firstTrigger => {
             const parent = firstTrigger.parentElement || firstTrigger;
             const grandparent = parent.parentElement || parent;
-            return grandparent;
+            const greatGrandparent = grandparent.parentElement || parent;
+            return firstTrigger.tagName === 'LI' ? grandparent : greatGrandparent;
           },
           firstTrigger
         );
         root = rootJSHandle.asElement();
       }
-      // Identify the visible active descendants of the root before the hover.
-      const preVisibles = await root.$$(targetSelectors);
       // Identify all the descendants of the root.
-      const descendants = await root.$$('*');
-      // Identify their opacities before the hover.
-      const preOpacities = await page.evaluate(
-        elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-      );
+      const preDescendants = await root.$$('*');
+      // Identify their opacities.
+      const preOpacities = await page.evaluate(elements => elements.map(
+        element => window.getComputedStyle(element).opacity
+      ), preDescendants);
       try {
-        // Hover over the potential trigger.
-        await firstTrigger.hover({timeout: 700});
-        // Identify whether it is coded as controlling other elements.
-        const isController = await page.evaluate(
-          element => element.ariaHasPopup || element.hasAttribute('aria-controls'), firstTrigger
-        );
-        // Wait for any delayed and/or slowed hover reaction, longer if coded as a controller.
-        await page.waitForTimeout(isController ? 1200 : 600);
-        await root.waitForElementState('stable');
-        // Identify the visible active descendants of the root during the hover.
-        const postVisibles = await root.$$(targetSelectors);
-        // Identify the opacities of the descendants of the root during the hover.
-        const postOpacities = await page.evaluate(
-          elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-        );
-        // Identify the elements with opacity changes.
-        const opacityTargets = descendants
-        .filter((descendant, index) => postOpacities[index] !== preOpacities[index]);
-        // Count them and their descendants.
-        const opacityAffected = opacityTargets.length
-          ? await page.evaluate(elements => elements.reduce(
-            (total, current) => total + 1 + current.querySelectorAll('*').length, 0
-          ), opacityTargets)
-          : 0;
-        // If hovering disclosed any element or changed any opacity:
-        if (postVisibles.length > preVisibles.length || opacityAffected) {
-          // Preserve the lengthened reaction wait, if any, for the next 5 tries.
-          if (elementsChecked < 11) {
-            elementsChecked = 5;
+        // Hover over the trigger.
+        await firstTrigger.hover({
+          timeout: 500,
+          noWaitAfter: true
+        });
+        // Repeatedly seeks impacts.
+        const getImpacts = async (interval, triesLeft) => {
+          if (triesLeft--) {
+            const postDescendants = await root.$$('*');
+            const remainerIndexes = await page.evaluate(args => {
+              const preDescendants = args[0];
+              const postDescendants = args[1];
+              const remainerIndexes = preDescendants
+              .map((element, index) => postDescendants.includes(element) ? index : -1)
+              .filter(index => index > -1);
+              return remainerIndexes;
+            }, [preDescendants, postDescendants]);
+            const additionCount = postDescendants.length - remainerIndexes.length;
+            const removalCount = preDescendants.length - remainerIndexes.length;
+            const remainers = [];
+            for (const index of remainerIndexes) {
+              remainers.push({
+                element: preDescendants[index],
+                preOpacity: preOpacities[index],
+                postOpacity: await page.evaluate(
+                  element => window.getComputedStyle(element).opacity, preDescendants[index]
+                )
+              });
+            }
+            const opacityChangers = remainers
+            .filter(remainer => remainer.postOpacity !== remainer.preOpacity);
+            const opacityImpact = opacityChangers ? await page.evaluate(changers => changers.reduce(
+              (total, current) => total + current.element.querySelectorAll('*').length, 0
+            ), opacityChangers) : 0;
+            if (additionCount || removalCount || opacityChangers.length) {
+              return {
+                additionCount,
+                removalCount,
+                opacityChangers,
+                opacityImpact
+              };
+            }
+            else {
+              return await new Promise(resolve => {
+                setTimeout(() => {
+                  resolve(getImpacts(interval, triesLeft));
+                }, interval);
+              });
+            }
+          }
+          else {
+            return null;
           }
-          // Hover over the upper-left corner of the page, to undo any hover reactions.
+        };
+        // Repeatedly seek impacts of the hover at intervals.
+        const impacts = await getImpacts(300, 4);
+        // If there were any:
+        if (impacts) {
+          // Hover over the upper-left corner of the page, to undo any impacts.
           await page.hover('body', {
             position: {
               x: 0,
@@ -141,100 +176,92 @@ const find = async (withItems, page, triggers) => {
           // Wait for any delayed and/or slowed hover reaction.
           await page.waitForTimeout(200);
           await root.waitForElementState('stable');
-          // Increment the counts of triggers and targets.
-          data.totals.triggers++;
-          const madeVisible = Math.max(0, postVisibles.length - preVisibles.length);
-          data.totals.madeVisible += madeVisible;
-          data.totals.opacityChanged += opacityTargets.length;
-          data.totals.opacityAffected += opacityAffected;
+          // Increment the counts of triggers and impacts.
+          const {additionCount, removalCount, opacityChangers, opacityImpact} = impacts;
+          data.totals.impactTriggers += popRatio;
+          data.totals.additions += popRatio * additionCount;
+          data.totals.removals += popRatio * removalCount;
+          data.totals.opacityChanges += popRatio * opacityChangers.length;
+          data.totals.opacityImpact += popRatio * opacityImpact;
           // If details are to be reported:
           if (withItems) {
             // Report them.
-            const triggerDataJSHandle = await page.evaluateHandle(args => {
-              // Returns the text of an element.
-              const textOf = (element, limit) => {
-                const text = element.textContent.trim() || element.outerHTML.trim();
-                return text.replace(/\s{2,}/sg, ' ').slice(0, limit);
-              };
-              const trigger = args[0];
-              const preVisibles = args[1];
-              const postVisibles = args[2];
-              const madeVisible = postVisibles
-              .filter(el => ! preVisibles.includes(el))
-              .map(el => ({
-                tagName: el.tagName,
-                text: textOf(el, 50)
-              }));
-              const opacityChanged = args[3].map(el => ({
-                tagName: el.tagName,
-                text: textOf(el, 50)
-              }));
-              return {
-                tagName: trigger.tagName,
-                id: trigger.id || '',
-                text: textOf(trigger, 50),
-                madeVisible,
-                opacityChanged
-              };
-            }, [firstTrigger, preVisibles, postVisibles, opacityTargets]);
-            const triggerData = await triggerDataJSHandle.jsonValue();
-            data.items.triggers.push(triggerData);
+            data.items[region].impactTriggers.push({
+              tagName,
+              text: await textOf(firstTrigger, 50),
+              additions: additionCount,
+              removals: removalCount,
+              opacityChanges: opacityChangers.length,
+              opacityImpact
+            });
           }
         }
       }
       catch (error) {
-        console.log('ERROR hovering');
+        console.log(`ERROR hovering (${error.message})`);
         data.totals.unhoverables++;
         if (withItems) {
-          data.items.unhoverables.push({
-            tagName: tagName,
-            id: firstTrigger.id || '',
+          const id = await firstTrigger.getAttribute('id');
+          data.items[region].unhoverables.push({
+            tagName,
+            id: id || '',
             text: await textOf(firstTrigger, 50)
           });
         }
       }
     }
     // Process the remaining potential triggers.
-    await find(withItems, page, triggers.slice(1));
+    await find(withItems, page, region, sample.slice(1), popRatio);
   }
 };
-// Performs hover test and reports results.
-exports.reporter = async (page, sampleSize = Infinity, withItems) => {
+// Performs the hover test and reports results.
+exports.reporter = async (
+  page, headSize = 0, headSampleSize = -1, tailSampleSize = -1, withItems
+) => {
   // If details are to be reported:
   if (withItems) {
     // Add properties for details to the initialized result.
     data.items = {
-      triggers: [],
-      unhoverables: []
+      head: {
+        impactTriggers: [],
+        unhoverables: []
+      },
+      tail: {
+        impactTriggers: [],
+        unhoverables: []
+      }
     };
   }
   // Identify the triggers.
-  const selectors = [
-    'body a:visible',
-    'body button:visible',
-    'body li:visible, body [onmouseenter]:visible',
-    'body [onmouseover]:visible'
-  ];
-  const triggers = await page.$$(selectors.join(', '))
+  const selectors = ['a', 'button', 'li', '[onmouseenter]', '[onmouseover]'];
+  const triggers = await page.$$(selectors.map(selector => `body ${selector}:visible`).join(', '))
   .catch(error => {
     console.log(`ERROR getting hover triggers (${error.message})`);
     data.prevented = true;
     return [];
   });
-  // If they number more than the sample size limit, sample them.
-  const triggerCount = triggers.length;
-  data.populationSize = triggerCount;
-  const triggerSample = triggerCount > sampleSize ? getSample(triggers, sampleSize) : triggers;
-  // Find and document the hover-triggered disclosures.
-  await find(withItems, page, triggerSample);
-  // If the triggers were sampled:
-  if (triggerCount > sampleSize) {
-    // Change the totals to population estimates.
-    const multiplier = triggerCount / sampleSize;
-    Object.keys(data.totals).forEach(key => {
-      data.totals[key] = Math.round(multiplier * data.totals[key]);
-    });
+  // Classify them into head and tail triggers.
+  const headTriggers = triggers.slice(0, headSize);
+  const tailTriggers = triggers.slice(headSize);
+  const headTriggerCount = headTriggers.length;
+  const tailTriggerCount = tailTriggers.length;
+  data.totals.triggers = headTriggerCount + tailTriggerCount;
+  data.totals.headTriggers = headTriggerCount;
+  data.totals.tailTriggers = tailTriggerCount;
+  // Get the head and tail samples.
+  const headSample = getSample(headTriggers, headSampleSize);
+  const tailSample = tailSampleSize === -1 ? tailTriggers : getSample(tailTriggers, tailSampleSize);
+  // Find and document the impacts.
+  if (headSample.length) {
+    await find(withItems, page, 'head', headSample, headTriggerCount / headSample.length);
+  }
+  if (tailSample.length) {
+    await find(withItems, page, 'tail', tailSample, tailTriggerCount / tailSample.length);
   }
+  // Round the reported totals.
+  Object.keys(data.totals).forEach(key => {
+    data.totals[key] = Math.round(data.totals[key]);
+  });
   // Return the result.
   return {result: data};
 };

package/tests/ibm.js

@@ -25,7 +25,6 @@ const run = async content => {
   const nowLabel = (new Date()).toISOString().slice(0, 19);
   // Return the result of a test.
   const ibmReport = await getCompliance(content, nowLabel);
-  await close();
   return ibmReport;
 };
 // Trims an IBM report.
@@ -100,7 +99,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
   // If a test with existing content is to be performed:
   const result = {};
   if (! withNewContent) {
-    const timeLimit = 15;
+    const timeLimit = 20;
     const typeContent = await page.content();
     result.content = await doTest(typeContent, withItems, timeLimit);
     if (result.content.prevented) {
@@ -118,5 +117,6 @@ exports.reporter = async (page, withItems, withNewContent) => {
       console.log('ERROR: Getting ibm test report from URL took too long');
     }
   }
+  await close();
   return {result};
 };