testaro 4.14.1 → 5.0.1

package/htmlcs/licence.txt

@@ -0,0 +1,24 @@
+Copyright (c) 2012, Squiz Pty Ltd (ABN 77 084 670 600)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of Squiz Pty Ltd nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.14.1",
+  "version": "5.0.1",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {
@@ -27,7 +27,6 @@
     "@siteimprove/alfa-playwright": "*",
     "@siteimprove/alfa-rules": "*",
     "@siteimprove/alfa-scraper": "*",
-    "aatt": "*",
     "accessibility-checker": "*",
     "axe-playwright": "*",
     "dotenv": "*",

package/run.js

@@ -29,7 +29,6 @@ const moves = {
 };
 // Names and descriptions of tests.
 const tests = {
-  aatt: 'AATT with HTML CodeSniffer WCAG 2.1 AA ruleset',
   alfa: 'alfa',
   axe: 'Axe',
   bulk: 'count of visible elements',
@@ -38,6 +37,7 @@ const tests = {
   focInd: 'focus indicators',
   focOp: 'focusability and operability',
   hover: 'hover-caused content additions',
+  htmlcs: 'HTML CodeSniffer WCAG 2.1 A, AA, and AAA rulesets',
   ibm: 'IBM Accessibility Checker',
   labClash: 'labeling inconsistencies',
   linkUl: 'adjacent-link underlining',
@@ -53,7 +53,7 @@ const tests = {
 };
 // Tests that may change the DOM.
 const domChangers = new Set([
-  'axe', 'focAll', 'focInd', 'focOp', 'hover', 'ibm', 'menuNav', 'wave'
+  'axe', 'focAll', 'focInd', 'focOp', 'hover', 'htmlcs', 'ibm', 'menuNav', 'wave'
 ]);
 // Browser types available in PlayWright.
 const browserTypeNames = {

package/samples/scripts/tp11.json

@@ -0,0 +1,164 @@
+{
+  "id": "tp11",
+  "what": "AATT, Alfa, Axe, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 20,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "aatt",
+      "waitLong": true,
+      "tryLimit": 2,
+      "what": "AATT with HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    }
+  ]
+}

package/samples/scripts/tp12.json

@@ -0,0 +1,162 @@
+{
+  "id": "tp12",
+  "what": "Alfa, Axe, HTML CodeSniffer, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 20,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "htmlcs",
+      "what": "HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    }
+  ]
+}

package/tests/hover.js

@@ -11,9 +11,14 @@
   opacity of an element changes, and (4) the element is a descendant of an element whose opacity
   changes. The test checks up to 4 times for hovering impacts at intervals of 0.3 second.
 
-  Despite this delay, the test can make the execution time practical by randomly sampling targets
+  Despite this delay, the test can make the execution time practical by randomly sampling triggers
   instead of hovering over all of them. When sampling is performed, the results may vary from one
-  execution to another.
+  execution to another. Because hover impacts typically occur near the beginning of a page,
+  sampling is governed by three optional parameters (defaults in parentheses):
+    headSize (0): the size of an initial subset of triggers (“head”)
+    headSampleSize (-1): the size of the sample to be drawn from the head
+    tailSampleSize (-1): the size of the sample to be drawn from the remainder of the page
+  A sample size of -1 means that there is no sampling, and the entire population is tested.
 
   An element is reported as unhoverable when it fails the Playwright actionability checks for
   hovering, i.e. fails to be attached to the DOM, visible, stable (not or no longer animating), and
@@ -27,13 +32,15 @@
 
 // Initialize the result.
 const data = {
-  populationSize: 0,
   totals: {
+    triggers: 0,
+    headTriggers: 0,
+    tailTriggers: 0,
     impactTriggers: 0,
     additions: 0,
     removals: 0,
     opacityChanges: 0,
-    opacityEffects: 0,
+    opacityImpact: 0,
     unhoverables: 0
   }
 };
@@ -43,16 +50,22 @@ const data = {
 // Samples a population and returns the sample.
 const getSample = (population, sampleSize) => {
   const popSize = population.length;
-  if (sampleSize > 0 && sampleSize < popSize) {
-    const sample = new Set();
-    while (sample.size < sampleSize) {
-      const index = Math.floor(popSize * Math.random());
-      sample.add(population[index]);
-    }
-    return Array.from(sample);
+  if (sampleSize === 0) {
+    return [];
+  }
+  else if (sampleSize > 0 && sampleSize < popSize) {
+    const popData = [];
+    for (const trigger of population) {
+      popData.push({
+        trigger,
+        sorter: Math.random()
+      });
+    };
+    popData.sort((a, b) => a.sorter - b.sorter);
+    return popData.slice(0, sampleSize).map(obj => obj.trigger);
   }
   else {
-    return [];
+    return population;
   }
 };
 // Returns the text of an element.
@@ -62,11 +75,11 @@ const textOf = async (element, limit) => {
   return text.trim().replace(/\s*/sg, '').slice(0, limit);
 };
 // Recursively reports impacts of hovering over triggers.
-const find = async (withItems, page, triggers) => {
+const find = async (withItems, page, region, sample, popRatio) => {
   // If any potential triggers remain:
-  if (triggers.length) {
+  if (sample.length) {
     // Identify the first of them.
-    const firstTrigger = triggers[0];
+    const firstTrigger = sample[0];
     const tagNameJSHandle = await firstTrigger.getProperty('tagName')
     .catch(error => {
       console.log(`ERROR getting trigger tag name (${error.message})`);
@@ -126,9 +139,9 @@ const find = async (withItems, page, triggers) => {
             }
             const opacityChangers = remainers
             .filter(remainer => remainer.postOpacity !== remainer.preOpacity);
-            const opacityImpact = await page.evaluate(changers => changers.reduce(
+            const opacityImpact = opacityChangers ? await page.evaluate(changers => changers.reduce(
               (total, current) => total + current.element.querySelectorAll('*').length, 0
-            ), opacityChangers);
+            ), opacityChangers) : 0;
             if (additionCount || removalCount || opacityChangers.length) {
               return {
                 additionCount,
@@ -165,15 +178,15 @@ const find = async (withItems, page, triggers) => {
           await root.waitForElementState('stable');
           // Increment the counts of triggers and impacts.
           const {additionCount, removalCount, opacityChangers, opacityImpact} = impacts;
-          data.totals.impactTriggers++;
-          data.totals.additions += additionCount;
-          data.totals.removals += removalCount;
-          data.totals.opacityChanges += opacityChangers.length;
-          data.totals.opacityImpact += opacityImpact;
+          data.totals.impactTriggers += popRatio;
+          data.totals.additions += popRatio * additionCount;
+          data.totals.removals += popRatio * removalCount;
+          data.totals.opacityChanges += popRatio * opacityChangers.length;
+          data.totals.opacityImpact += popRatio * opacityImpact;
           // If details are to be reported:
           if (withItems) {
             // Report them.
-            data.items.impactTriggers.push({
+            data.items[region].impactTriggers.push({
               tagName,
               text: await textOf(firstTrigger, 50),
               additions: additionCount,
@@ -189,7 +202,7 @@ const find = async (withItems, page, triggers) => {
         data.totals.unhoverables++;
         if (withItems) {
           const id = await firstTrigger.getAttribute('id');
-          data.items.unhoverables.push({
+          data.items[region].unhoverables.push({
             tagName,
             id: id || '',
             text: await textOf(firstTrigger, 50)
@@ -198,17 +211,25 @@ const find = async (withItems, page, triggers) => {
       }
     }
     // Process the remaining potential triggers.
-    await find(withItems, page, triggers.slice(1));
+    await find(withItems, page, region, sample.slice(1), popRatio);
   }
 };
-// Performs hover test and reports results.
-exports.reporter = async (page, sampleSize = Infinity, withItems) => {
+// Performs the hover test and reports results.
+exports.reporter = async (
+  page, headSize = 0, headSampleSize = -1, tailSampleSize = -1, withItems
+) => {
   // If details are to be reported:
   if (withItems) {
     // Add properties for details to the initialized result.
     data.items = {
-      impactTriggers: [],
-      unhoverables: []
+      head: {
+        impactTriggers: [],
+        unhoverables: []
+      },
+      tail: {
+        impactTriggers: [],
+        unhoverables: []
+      }
     };
   }
   // Identify the triggers.
@@ -219,20 +240,28 @@ exports.reporter = async (page, sampleSize = Infinity, withItems) => {
     data.prevented = true;
     return [];
   });
-  // If they number more than the sample size limit, sample them.
-  const triggerCount = triggers.length;
-  data.populationSize = triggerCount;
-  const triggerSample = triggerCount > sampleSize ? getSample(triggers, sampleSize) : triggers;
-  // Find and document the hover-triggered impacts.
-  await find(withItems, page, triggerSample);
-  // If the triggers were sampled:
-  if (triggerCount > sampleSize) {
-    // Change the totals to population estimates.
-    const multiplier = triggerCount / sampleSize;
-    Object.keys(data.totals).forEach(key => {
-      data.totals[key] = Math.round(multiplier * data.totals[key]);
-    });
+  // Classify them into head and tail triggers.
+  const headTriggers = triggers.slice(0, headSize);
+  const tailTriggers = triggers.slice(headSize);
+  const headTriggerCount = headTriggers.length;
+  const tailTriggerCount = tailTriggers.length;
+  data.totals.triggers = headTriggerCount + tailTriggerCount;
+  data.totals.headTriggers = headTriggerCount;
+  data.totals.tailTriggers = tailTriggerCount;
+  // Get the head and tail samples.
+  const headSample = getSample(headTriggers, headSampleSize);
+  const tailSample = tailSampleSize === -1 ? tailTriggers : getSample(tailTriggers, tailSampleSize);
+  // Find and document the impacts.
+  if (headSample.length) {
+    await find(withItems, page, 'head', headSample, headTriggerCount / headSample.length);
   }
+  if (tailSample.length) {
+    await find(withItems, page, 'tail', tailSample, tailTriggerCount / tailSample.length);
+  }
+  // Round the reported totals.
+  Object.keys(data.totals).forEach(key => {
+    data.totals[key] = Math.round(data.totals[key]);
+  });
   // Return the result.
   return {result: data};
 };

package/tests/htmlcs.js

@@ -0,0 +1,55 @@
+/*
+  htmlcs
+  This test implements the HTML CodeSniffer ruleset.
+*/
+
+// FUNCTIONS
+// Runs HTML CodeSniffer on the page.
+exports.reporter = async page => {
+  await page.addScriptTag({
+    path: `${__dirname}/../htmlcs/HTMLCS.js`
+  });
+  let messageStrings = [];
+  for (const standard of ['WCAG2A', 'WCAG2AA', 'WCAG2AAA']) {
+    const nextIssues = await page.evaluate(standard => HTMLCS_RUNNER.run(standard), standard);
+    messageStrings.push(... nextIssues);
+  }
+  // Sort the issues by class and standard.
+  messageStrings.sort();
+  // Remove any duplicate issues.
+  messageStrings = [... new Set(messageStrings)];
+  // Initialize the result.
+  const result = {
+    Error: {},
+    Warning: {}
+  };
+  // For each issue:
+  messageStrings.forEach(string => {
+    const parts = string.split(/\|/g);
+    const partCount = parts.length;
+    if (partCount !== 6) {
+      console.log(`ERROR: Issue string ${string} has too few or too many parts`);
+    }
+    // If it is an error or a warning (not a notice):
+    else if (['Error', 'Warning'].includes(parts[0])) {
+      /*
+        Add the issue to an issueClass.issueCode.description array in the result.
+        This saves space, because, although some descriptions are issue-specific, such as
+        descriptions that state the contrast ratio of an element, most descriptions are
+        generic, so typically many issues share a description.
+      */
+      if (! result[parts[0]][parts[1]]) {
+        result[parts[0]][parts[1]] = {};
+      }
+      if (! result[parts[0]][parts[1]][parts[4]]) {
+        result[parts[0]][parts[1]][parts[4]] = [];
+      }
+      result[parts[0]][parts[1]][parts[4]].push({
+        tagName: parts[2],
+        id: parts[3],
+        code: parts[5]
+      });
+    }
+  });
+  return {result};
+};

package/tests/ibm.js

@@ -99,7 +99,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
   // If a test with existing content is to be performed:
   const result = {};
   if (! withNewContent) {
-    const timeLimit = 15;
+    const timeLimit = 20;
     const typeContent = await page.content();
     result.content = await doTest(typeContent, withItems, timeLimit);
     if (result.content.prevented) {