testaro 5.6.0 → 5.6.3

package/continuum/continuum.conf.js

@@ -0,0 +1,19 @@
+/* eslint-disable semi */
+/* eslint-disable quotes */
+window.LevelAccess_AccessContinuumConfiguration = {
+  "accessEngineType": "community",
+  "ampInstanceUrl": "https://amp.levelaccess.net",
+  "defaultStandardIds": [
+    610, /* WCAG 2.0 Level A */
+    611, /* WCAG 2.0 Level AA */
+    612, /* WCAG 2.0 Level AAA */
+    1387, /* WCAG 2.1 Level A */
+    1388, /* WCAG 2.1 Level AA */
+    1389, /* WCAG 2.1 Level AAA */
+    1140, /* Section 508 and 255 (Revised 2017) */
+    1471 /* WCAG 2.0 Level A & AA Baseline */
+  ],
+  // includePotentialAccessibilityConcerns may be true only in the Professional Edition.
+  "includePotentialAccessibilityConcerns": false,
+  "ampApiToken": null
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "5.6.0",
+  "version": "5.6.3",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -32,6 +32,7 @@ const tests = {
   alfa: 'alfa',
   axe: 'Axe',
   bulk: 'count of visible elements',
+  continuum: 'Level Access Continuum, community edition',
   embAc: 'active elements embedded in links or buttons',
   focAll: 'focusable and Tab-focused elements',
   focInd: 'focus indicators',
@@ -53,7 +54,7 @@ const tests = {
 };
 // Tests that may change the DOM.
 const domChangers = new Set([
-  'axe', 'focAll', 'focInd', 'focOp', 'hover', 'htmlcs', 'ibm', 'menuNav', 'wave'
+  'axe', 'continuum', 'focAll', 'focInd', 'focOp', 'hover', 'htmlcs', 'ibm', 'menuNav', 'wave'
 ]);
 // Browser types available in PlayWright.
 const browserTypeNames = {
@@ -74,12 +75,14 @@ const errorWords = [
   'deprecated',
   'error',
   'failed',
+  'invalid',
   'missing',
   'but not used',
   'refused',
   'requires',
   'suspicious',
   'unrecognized',
+  'violates',
   'warning'
 ];
 

package/samples/scripts/tp13.json

@@ -0,0 +1,163 @@
+{
+  "id": "tp12",
+  "what": "Alfa, Axe, HTML CodeSniffer, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "timeLimit": 250,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 40,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "htmlcs",
+      "what": "HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    }
+  ]
+}

package/samples/scripts/tp14.json

@@ -0,0 +1,168 @@
+{
+  "id": "tp12",
+  "what": "Alfa, Axe, Continuum, HTML CodeSniffer, IBM, Tenon, WAVE, and 16 custom tests",
+  "strict": true,
+  "timeLimit": 300,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "Webkit browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "tenonRequest",
+      "id": "a",
+      "withNewContent": true,
+      "what": "Tenon API version 2 test request"
+    },
+    {
+      "type": "test",
+      "which": "motion",
+      "what": "spontaneous change of content; requires webkit",
+      "delay": 2500,
+      "interval": 2500,
+      "count": 5
+    },
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "Chromium browser"
+    },
+    {
+      "type": "url",
+      "which": "https://*",
+      "what": "any page"
+    },
+    {
+      "type": "test",
+      "which": "bulk",
+      "what": "count of visible elements"
+    },
+    {
+      "type": "test",
+      "which": "embAc",
+      "withItems": true,
+      "what": "active elements incorrectly embedded in each other"
+    },
+    {
+      "type": "test",
+      "which": "focAll",
+      "what": "Tab-focusability"
+    },
+    {
+      "type": "test",
+      "which": "focInd",
+      "revealAll": false,
+      "allowedDelay": 250,
+      "withItems": true,
+      "what": "focus indicators"
+    },
+    {
+      "type": "test",
+      "which": "focOp",
+      "withItems": true,
+      "what": "focusability and operability of elements"
+    },
+    {
+      "type": "test",
+      "which": "hover",
+      "headSize": 40,
+      "headSampleSize": 20,
+      "tailSampleSize": 15,
+      "withItems": true,
+      "what": "hover impacts"
+    },
+    {
+      "type": "test",
+      "which": "labClash",
+      "withItems": true,
+      "what": "unlabeled and mislabeled form controls"
+    },
+    {
+      "type": "test",
+      "which": "linkUl",
+      "withItems": true,
+      "what": "underlining of inline links"
+    },
+    {
+      "type": "test",
+      "which": "menuNav",
+      "withItems": true,
+      "what": "keyboard navigation within true-focus menus"
+    },
+    {
+      "type": "test",
+      "which": "radioSet",
+      "withItems": true,
+      "what": "grouping of radio buttons in fieldsets"
+    },
+    {
+      "type": "test",
+      "which": "role",
+      "what": "validity and necessity of role assignments"
+    },
+    {
+      "type": "test",
+      "which": "styleDiff",
+      "withItems": true,
+      "what": "style consistency of headings, buttons, and links"
+    },
+    {
+      "type": "test",
+      "which": "tabNav",
+      "withItems": true,
+      "what": "keyboard navigation within tab lists"
+    },
+    {
+      "type": "test",
+      "which": "zIndex",
+      "withItems": true,
+      "what": "elements with non-auto z indexes"
+    },
+    {
+      "type": "test",
+      "which": "alfa",
+      "what": "Siteimprove alfa"
+    },
+    {
+      "type": "test",
+      "which": "axe",
+      "detailLevel": 2,
+      "rules": [],
+      "what": "Axe core, all rules"
+    },
+    {
+      "type": "test",
+      "which": "continuum",
+      "what": "Continuum"
+    },
+    {
+      "type": "test",
+      "which": "htmlcs",
+      "what": "HTML CodeSniffer"
+    },
+    {
+      "type": "test",
+      "which": "ibm",
+      "withItems": true,
+      "what": "IBM Accessibility Checker, with page content and again with URL"
+    },
+    {
+      "type": "test",
+      "which": "wave",
+      "reportType": 4,
+      "what": "WAVE, report-type 4"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
+    }
+  ]
+}

package/tests/continuum.js

@@ -0,0 +1,60 @@
+/* eslint-disable no-undef */
+/*
+  continuum
+  This test implements the Continuum ruleset.
+*/
+
+// FUNCTIONS
+// Runs Continuum on the page.
+exports.reporter = async page => {
+  let result = {};
+  // Inject the continuum scripts into the page, exposing the continuum object.
+  for (const fileName of ['continuum.conf', 'AccessEngine.community', 'Continuum.community']) {
+    if (! result.prevented) {
+      await page.addScriptTag({
+        path: `${__dirname}/../continuum/${fileName}.js`
+      })
+      .catch(error => {
+        console.log(`ERROR adding the ${fileName} script to the page (${error.message})`);
+        result.prevented = true;
+        result.error = `ERROR adding the ${fileName} script to the page`;
+      });
+    }
+  }
+  if (! result.prevented) {
+    // Run the Continuum ruleset and get the result, failing if none within 20 seconds.
+    result = await page.evaluate(async () => {
+      continuum.setUp(null, null, window);
+      const bigResultPromise = continuum.runAllTests();
+      const deadlinePromise = new Promise(resolve => {
+        setTimeout(() => {
+          resolve('timeout');
+        }, 20000);
+      });
+      const bigResult = await Promise.race([bigResultPromise, deadlinePromise]);
+      if (Array.isArray(bigResult)) {
+        return bigResult.map(bigItem => {
+          const item = bigItem._rawEngineJsonObject;
+          delete item.fingerprint.encoding;
+          if (item.element.length > 200) {
+            item.element = `${item.element.slice(0, 100)} ... ${item.element.slice(-100)}`;
+          }
+          return item;
+        });
+      }
+      else if (bigResult === 'timeout') {
+        return {
+          prevented: true,
+          error: 'ERROR: Running all tests timed out'
+        };
+      }
+      else {
+        return {
+          prevented: true,
+          error: 'ERROR: Invalid result'
+        };
+      }
+    });
+  }
+  return {result};
+};