testaro 4.15.0 → 5.1.0

package/htmlcs/licence.txt

@@ -0,0 +1,24 @@
+Copyright (c) 2012, Squiz Pty Ltd (ABN 77 084 670 600)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of Squiz Pty Ltd nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "4.15.0",
+  "version": "5.1.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {
@@ -27,7 +27,6 @@
     "@siteimprove/alfa-playwright": "*",
     "@siteimprove/alfa-rules": "*",
     "@siteimprove/alfa-scraper": "*",
-    "aatt": "*",
     "accessibility-checker": "*",
     "axe-playwright": "*",
     "dotenv": "*",

package/run.js

@@ -29,7 +29,6 @@ const moves = {
 };
 // Names and descriptions of tests.
 const tests = {
-  aatt: 'AATT with HTML CodeSniffer WCAG 2.1 AA ruleset',
   alfa: 'alfa',
   axe: 'Axe',
   bulk: 'count of visible elements',
@@ -38,6 +37,7 @@ const tests = {
   focInd: 'focus indicators',
   focOp: 'focusability and operability',
   hover: 'hover-caused content additions',
+  htmlcs: 'HTML CodeSniffer WCAG 2.1 A, AA, and AAA rulesets',
   ibm: 'IBM Accessibility Checker',
   labClash: 'labeling inconsistencies',
   linkUl: 'adjacent-link underlining',
@@ -53,7 +53,7 @@ const tests = {
 };
 // Tests that may change the DOM.
 const domChangers = new Set([
-  'axe', 'focAll', 'focInd', 'focOp', 'hover', 'ibm', 'menuNav', 'wave'
+  'axe', 'focAll', 'focInd', 'focOp', 'hover', 'htmlcs', 'ibm', 'menuNav', 'wave'
 ]);
 // Browser types available in PlayWright.
 const browserTypeNames = {

package/samples/scripts/tp11.json

@@ -0,0 +1,164 @@
+{
+  "id": "tp11",
+  "what": "AATT, Alfa, Axe, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 20,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "aatt",
+      "waitLong": true,
+      "tryLimit": 2,
+      "what": "AATT with HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    }
+  ]
+}

package/samples/scripts/tp12.json

@@ -0,0 +1,162 @@
+{
+  "id": "tp12",
+  "what": "Alfa, Axe, HTML CodeSniffer, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 20,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "htmlcs",
+      "what": "HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    }
+  ]
+}

package/tests/htmlcs.js

@@ -0,0 +1,56 @@
+/*
+  htmlcs
+  This test implements the HTML CodeSniffer ruleset.
+*/
+
+// FUNCTIONS
+// Runs HTML CodeSniffer on the page.
+exports.reporter = async page => {
+  await page.addScriptTag({
+    path: `${__dirname}/../htmlcs/HTMLCS.js`
+  });
+  let messageStrings = [];
+  for (const standard of ['WCAG2AA']) {
+    const nextIssues = await page.evaluate(standard => HTMLCS_RUNNER.run(standard), standard);
+    messageStrings.push(... nextIssues);
+  }
+  // Sort the issues by class and standard.
+  messageStrings.sort();
+  // Remove any duplicate issues.
+  messageStrings = [... new Set(messageStrings)];
+  // Initialize the result.
+  const result = {
+    Error: {},
+    Warning: {}
+  };
+  // For each issue:
+  messageStrings.forEach(string => {
+    const parts = string.split(/\|/g);
+    const partCount = parts.length;
+    if (partCount !== 6) {
+      console.log(`ERROR: Issue string ${string} has too few or too many parts`);
+    }
+    // If it is an error or a warning (not a notice):
+    else if (['Error', 'Warning'].includes(parts[0])) {
+      /*
+        Add the issue to an issueClass.issueCode.description array in the result.
+        This saves space, because, although some descriptions are issue-specific, such as
+        descriptions that state the contrast ratio of an element, most descriptions are
+        generic, so typically many issues share a description.
+      */
+      const issueCode = parts[1].replace(/^WCAG2|\.Principle\d\.Guideline[\d_]+/g, '');
+      if (! result[parts[0]][issueCode]) {
+        result[parts[0]][issueCode] = {};
+      }
+      if (! result[parts[0]][issueCode][parts[4]]) {
+        result[parts[0]][issueCode][parts[4]] = [];
+      }
+      result[parts[0]][issueCode][parts[4]].push({
+        tagName: parts[2],
+        id: parts[3],
+        code: parts[5]
+      });
+    }
+  });
+  return {result};
+};