testaro 14.3.0 → 14.4.0

package/README.md

@@ -204,11 +204,13 @@ If the `STANDARD` environment variable has the value `also` (which it has by def
 
 The standard format of each tool report has two properties:
 - `totals`: an array of 4 integers, representing the counts of issue instances classified by the tool into 4 ordinal degrees of severity. For example, `[2, 13, 0, 5]` would mean that the tool reported 2 instances at the lowest severity, 13 at the next-lowest, none at the third-lowest, and 5 at the highest.
-- `instances`: an array of objects describing facts about issue instances reported by the tool. Insofar as each tool permits, this object has these properties:
+- `instances`: an array of objects describing facts about issue instances reported by the tool. This object has these properties, some of which have empty strings as values when the tool does not provide values:
     - `issueID`: a code identifying the issue
     - `what`: a description of the issue
     - `count` (optional): the count of instances if this instance represents multiple instances
     - `ordinalSeverity`: how the tool ranks the severity of the instance, on a 4-point ordinal scale
+    - `tagName`: upper-case tagName of the affected element
+    - `id`: value of the `id` property of that element
     - `location`: an object with three properties:
         - `doc`: whether the source (`source`) or the browser rendition (`dom`) was tested
         - `type`: the type of location information provided by the tool (`line`, `selector`, or `xpath`)
@@ -225,6 +227,8 @@ standardResult: {
       issueID: 'rule01',
       what: 'Button type invalid',
       ordinalSeverity: 0,
+      tagName: 'BUTTON'
+      id: '',
       location: {
         doc: 'dom',
         type: 'line',
@@ -236,6 +240,8 @@ standardResult: {
       issueID: 'rule01',
       what: 'Button type invalid',
       ordinalSeverity: 1,
+      tagName: 'BUTTON',
+      id: 'submitbutton',
       location: {
         doc: 'dom',
         type: 'line',
@@ -248,6 +254,8 @@ standardResult: {
       what: 'Links have empty href attributes',
       count: 17,
       ordinalSeverity: 3,
+      tagName: 'A',
+      id: '',
       location: {
         doc: '',
         type: '',

package/actSpecs.js

@@ -200,7 +200,7 @@ exports.actSpecs = {
           false,
           'array',
           'areStrings',
-          'message specifications (e.g., error:Bad value), if not all'
+          'message specifications (type and start of message, e.g., error:Bad value), if not all'
         ]
       }
     ],

package/call.js

@@ -71,6 +71,7 @@ if (fn === 'run' && fnArgs.length === 1) {
   callRun(fnArgs)
   .then(() => {
     console.log('Execution completed');
+    process.exit(0);
   });
 }
 else if (fn === 'watch' && fnArgs.length === 3) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "14.3.0",
+  "version": "14.4.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -994,31 +994,43 @@ const doActs = async (report, actIndex, page) => {
               }
               // Conduct, report, and time the test.
               const startTime = Date.now();
-              const testReport = await require(`./tests/${act.which}`).reporter(...args);
-              const expectations = act.expect;
-              // If the test has expectations:
-              if (expectations) {
-                // Initialize whether they were fulfilled.
-                testReport.result.expectations = [];
-                let failureCount = 0;
-                // For each expectation:
-                expectations.forEach(spec => {
-                  const truth = isTrue(testReport, spec);
-                  testReport.result.expectations.push({
-                    property: spec[0],
-                    relation: spec[1],
-                    criterion: spec[2],
-                    actual: truth[0],
-                    passed: truth[1]
+              let testReport = {
+                result: {
+                  success: false
+                }
+              };
+              try {
+                testReport = await require(`./tests/${act.which}`).reporter(...args);
+                const expectations = act.expect;
+                // If the test has expectations:
+                if (expectations) {
+                  // Initialize whether they were fulfilled.
+                  testReport.result.expectations = [];
+                  let failureCount = 0;
+                  // For each expectation:
+                  expectations.forEach(spec => {
+                    const truth = isTrue(testReport, spec);
+                    testReport.result.expectations.push({
+                      property: spec[0],
+                      relation: spec[1],
+                      criterion: spec[2],
+                      actual: truth[0],
+                      passed: truth[1]
+                    });
+                    if (! truth[1]) {
+                      failureCount++;
+                    }
                   });
-                  if (! truth[1]) {
-                    failureCount++;
-                  }
-                });
-                testReport.result.failureCount = failureCount;
+                  testReport.result.failureCount = failureCount;
+                }
+                testReport.result.success = true;
               }
-              testReport.result.success = true;
-              report.jobData.testTimes.push([act.which, Math.round((Date.now() - startTime) / 1000)]);
+              catch(error) {
+                console.log(`ERROR: Test act ${act.which} failed (${error.message.slice(0, 400)})`);
+              }
+              report.jobData.testTimes.push(
+                [act.which, Math.round((Date.now() - startTime) / 1000)]
+              );
               report.jobData.testTimes.sort((a, b) => b[1] - a[1]);
               // Add the result object (possibly an array) to the act.
               const resultCount = Object.keys(testReport.result).length;

package/standardize.js

@@ -35,10 +35,23 @@ const doAxe = (result, standardResult, certainty) => {
           critical: 1
         };
         const ordinalSeverity = severityWeights[node.impact] + (certainty === 'violations' ? 2 : 0);
+        const tagName = node.html && node.html.replace(/^<|[ >].*$/sg, '').toUpperCase();
+        let id = '';
+        if (node.target && node.target.length && node.target[0].startsWith('#')) {
+          id = node.target[0].slice(1);
+        }
+        else if (node.html) {
+          const idArray = node.html.match(/\sid="([^"]+)"/);
+          if (idArray && idArray.length === 2) {
+            id = idArray[1];
+          }
+        }
         const instance = {
           issueID: rule.id,
           what: Array.from(whatSet.values()).join('; '), 
           ordinalSeverity,
+          tagName,
+          id,
           location: {
             doc: 'dom',
             type: 'selector',
@@ -58,17 +71,19 @@ const doHTMLCS = (result, standardResult, severity) => {
       const ruleData = result[severity][ruleID];
       Object.keys(ruleData).forEach(what => {
         ruleData[what].forEach(item => {
-          const {tagName, code} = item;
+          const {tagName, id, code} = item;
           const instance = {
             issueID: ruleID,
             what,
             ordinalSeverity: ['Warning', '', '', 'Error'].indexOf(severity),
+            tagName: tagName.toUpperCase(),
+            id: id.slice(1),
             location: {
               doc: 'dom',
               type: '',
               spec: ''
             },
-            excerpt: cap(`${tagName ? tagName + ': ' : ''}${code || ''}`)
+            excerpt: cap(code)
           };
           standardResult.instances.push(instance);
         });
@@ -81,10 +96,27 @@ const doNuVal = (result, standardResult, docType) => {
   const items = result[docType] && result[docType].messages;
   if (items && items.length) {
     items.forEach(item => {
+      let tagName = '';
+      let id = '';
+      if (item.extract) {
+        const tagNameLCArray = item.extract.match(
+          /^Element ([^ ]+)|^An (img) element| (meta|script) element| element (script)| tag (script)/
+        );
+        if (tagNameLCArray && tagNameLCArray.length > 1) {
+          tagName = tagNameLCArray[1].toUpperCase();
+        }
+        const idArray = item.extract.match(/^.+\sid="([^"]+)"/);
+        if (idArray && idArray.length === 2) {
+          id = idArray[1];
+        }
+      }
+      // Include the message twice, because in scoring it is likely to be replaced by a pattern.
       const instance = {
         issueID: item.message,
         what: item.message,
         ordinalSeverity: -1,
+        tagName,
+        id,
         location: {
           doc: docType === 'pageContent' ? 'dom' : 'source',
           type: 'line',
@@ -125,16 +157,31 @@ const doQualWeb = (result, standardResult, ruleClassName) => {
       const ruleResult = ruleClass.assertions[rule];
       ruleResult.results.forEach(item => {
         item.elements.forEach(element => {
+          const {htmlCode} = element;
+          let tagName = '';
+          let id = '';
+          if (htmlCode) {
+            const tagNameArray = htmlCode.match(/^<([^ >]+)/);
+            if (tagNameArray && tagNameArray.length === 2) {
+              tagName = tagNameArray[1].toUpperCase();
+            }
+            const idArray = htmlCode.match(/\sid="([^"]+)"/);
+            if (idArray && idArray.length === 2) {
+              id = idArray[1];
+            }
+          }
           const instance = {
             issueID: rule,
             what: ruleResult.description,
             ordinalSeverity: severities[ruleClassName][item.verdict],
+            tagName,
+            id,
             location: {
               doc: 'dom',
               type: 'selector',
               spec: element.pointer
             },
-            excerpt: cap(element.htmlCode)
+            excerpt: cap(htmlCode)
           };
           standardResult.instances.push(instance);
           standardResult.totals[instance.ordinalSeverity]++;
@@ -150,10 +197,25 @@ const doWAVE = (result, standardResult, categoryName) => {
     const ordinalSeverity = categoryName === 'alert' ? 0 : 3;
     Object.keys(category.items).forEach(rule => {
       category.items[rule].selectors.forEach(selector => {
+        let tagName = '';
+        let id = '';
+        if (typeof selector === 'string') {
+          const finalTerm = selector.replace(/^.+\s/, '');
+          if (finalTerm.includes('#')) {
+            const finalArray = finalTerm.split('#');
+            tagName = finalArray[0].replace(/:.*/, '');
+            id = finalArray[1];
+          }
+          else {
+            tagName = finalTerm.replace(/:.*/, '');
+          }
+        }
         const instance = {
           issueID: rule,
           what: category.items[rule].description,
           ordinalSeverity,
+          tagName,
+          id,
           location: {
             doc: 'dom',
             type: 'selector',
@@ -167,28 +229,39 @@ const doWAVE = (result, standardResult, categoryName) => {
   }
 };
 // Converts a result.
-const convert = (testName, result, standardResult) => {
+const convert = (toolName, result, standardResult) => {
   // alfa
-  if (testName === 'alfa' && result.totals) {
+  if (toolName === 'alfa' && result.totals) {
     standardResult.totals = [result.totals.warnings, 0, 0, result.totals.failures];
     result.items.forEach(item => {
+      const {codeLines} = item.target;
+      let id = '';
+      if (codeLines && codeLines.length) {
+        const code = codeLines[0];
+        const idMatchArray = code.match(/\sid="([^"]+)"/);
+        if (idMatchArray && idMatchArray.length === 2) {
+          id = idMatchArray[1];
+        }
+      }
       const instance = {
         issueID: item.rule.ruleID,
         what: item.rule.ruleSummary,
         ordinalSeverity: ['cantTell', '', '', 'failed'].indexOf(item.verdict),
+        tagName: item.target.tagName.toUpperCase(),
+        id,
         location: {
           doc: 'dom',
           type: 'xpath',
           spec: item.target.path
         },
-        excerpt: cap(item.target.codeLines.join(' '))
+        excerpt: Array.isArray(codeLines) ? cap(codeLines.join(' ')) : ''
       };
       standardResult.instances.push(instance);
     });
   }
   // axe
   else if (
-    testName === 'axe'
+    toolName === 'axe'
     && result.totals
     && (result.totals.rulesWarned || result.totals.rulesViolated)
   ) {
@@ -203,13 +276,25 @@ const convert = (testName, result, standardResult) => {
     doAxe(result, standardResult, 'violations');
   }
   // continuum
-  else if (testName === 'continuum' && Array.isArray(result) && result.length) {
+  else if (toolName === 'continuum' && Array.isArray(result) && result.length) {
     standardResult.totals = [0, 0, 0, result.length];
     result.forEach(item => {
+      let tagName = '';
+      let id = '';
+      if (item.fingerprint && item.fingerprint.css) {
+        const {css} = item.fingerprint;
+        tagName = css.replace(/[^a-z].*$/, '').toUpperCase();
+        const idArray = css.match(/\[id="([^"]+)"\]/);
+        if (idArray && idArray.length) {
+          id = idArray[1];
+        }
+      }
       const instance = {
         issueID: item.engineTestId.toString(),
         what: item.attributeDetail,
-        ordinalSeverity: 0,
+        ordinalSeverity: 3,
+        tagName,
+        id,
         location: {
           doc: 'dom',
           type: 'selector',
@@ -221,7 +306,7 @@ const convert = (testName, result, standardResult) => {
     });
   }
   // htmlcs
-  else if (testName === 'htmlcs' && result) {
+  else if (toolName === 'htmlcs' && result) {
     doHTMLCS(result, standardResult, 'Warning');
     doHTMLCS(result, standardResult, 'Error');
     const {instances} = standardResult;
@@ -233,13 +318,29 @@ const convert = (testName, result, standardResult) => {
     ];
   }
   // ibm
-  else if (testName === 'ibm' && result.totals) {
+  else if (toolName === 'ibm' && result.totals) {
     standardResult.totals = [0, result.totals.recommendation, 0, result.totals.violation];
     result.items.forEach(item => {
+      let tagName = '';
+      let id = '';
+      if (item.path && item.path.dom) {
+        const tagNameArray = item.path.dom.match(/^.+\/([^/[]+)/s);
+        if (tagNameArray && tagNameArray.length === 2) {
+          tagName = tagNameArray[1];
+        }
+        if (item.snippet) {
+          const idArray = item.snippet.match(/^.+\sid="([^"]+)"/s);
+          if (idArray && idArray.length === 2) {
+            id = idArray[1];
+          }
+        }
+      }
       const instance = {
         issueID: item.ruleId,
         what: item.message,
         ordinalSeverity: ['', 'recommendation', '', 'violation'].indexOf(item.level),
+        tagName,
+        id,
         location: {
           doc: 'dom',
           type: 'xpath',
@@ -251,7 +352,7 @@ const convert = (testName, result, standardResult) => {
     });
   }
   // nuVal
-  else if (testName === 'nuVal' && (result.pageContent || result.rawPage)) {
+  else if (toolName === 'nuVal' && (result.pageContent || result.rawPage)) {
     if (result.pageContent) {
       doNuVal(result, standardResult, 'pageContent');
     }
@@ -268,7 +369,7 @@ const convert = (testName, result, standardResult) => {
   }
   // qualWeb
   else if (
-    testName === 'qualWeb'
+    toolName === 'qualWeb'
     && result.modules
     && (
       result.modules['act-rules']
@@ -288,14 +389,30 @@ const convert = (testName, result, standardResult) => {
     }
   }
   // tenon
-  else if (testName === 'tenon' && result.data && result.data.resultSet) {
+  else if (toolName === 'tenon' && result.data && result.data.resultSet) {
     result.data.resultSet.forEach(item => {
+      let tagName = '';
+      if (item.xpath) {
+        const tagNameArray = item.xpath.match(/^.+\/([^/[]+)/);
+        if (tagNameArray && tagNameArray.length === 2) {
+          tagName = tagNameArray[1].toUpperCase();
+        }
+      }
+      let id = '';
+      if (item.errorSnippet) {
+        const idArray = item.errorSnippet.match(/^.+\sid="([^"]+)"/);
+        if (idArray && idArray.length === 2) {
+          id = idArray[1];
+        }
+      }
       const instance = {
         issueID: item.tID ? item.tID.toString() : '',
         what: item.errorTitle || '',
         ordinalSeverity: Math.min(
           3, Math.max(0, Math.round((item.certainty || 0) * (item.priority || 0) / 3333))
         ),
+        tagName,
+        id,
         location: {
           doc: 'dom',
           type: 'xpath',
@@ -311,7 +428,7 @@ const convert = (testName, result, standardResult) => {
     });
   }
   // testaro
-  else if (testName === 'testaro') {
+  else if (toolName === 'testaro') {
     const rules = result.rules ? Object.keys(result.rules) : [];
     standardResult.totals = [0, 0, 0, 0];
     rules.forEach(rule => {
@@ -332,7 +449,7 @@ const convert = (testName, result, standardResult) => {
   }
   // wave
   else if (
-    testName === 'wave'
+    toolName === 'wave'
     && result.categories
     && (
       result.categories.error

package/testaro/allHidden.js

@@ -39,6 +39,16 @@ exports.reporter = async page => {
         document: document.documentElement.ariaHidden === 'true',
         body: body ? body.ariaHidden === 'true' : false,
         main: main ? main.ariaHidden === 'true' : false
+      },
+      tagName: {
+        document: 'HTML',
+        body: 'BODY',
+        main: main ? main.tagName : ''
+      },
+      id: {
+        document: document.id,
+        body: body.id,
+        main: main ? main.id : ''
       }
     };
     // Identify whether each region is really hidden.
@@ -69,6 +79,8 @@ exports.reporter = async page => {
           issueID: `allHidden-${hider}-${region}`,
           what: `${reportables[region][1]} ${reportables[hider][1]}`,
           ordinalSeverity: reportables[region][0] + reportables[hider][0] || 0,
+          tagName: region.tagName,
+          id: region.id,
           location: {
             doc: 'dom',
             type: 'selector',

package/testaro/attVal.js

@@ -32,6 +32,7 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
       });
       const report = badElements.map(el => ({
         tagName: el.tagName,
+        id: el.id,
         textStart: textOf(el, 70),
         attributeValue: el.getAttribute(attributeName)
       }));
@@ -51,6 +52,8 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
         what:
           `${item.tagName} element has attribute ${attributeName} with illicit value ${item.attributeValue}`,
         ordinalSeverity: 2,
+        tagName: item.tagName,
+        id: item.id,
         location: {
           doc: '',
           type: '',
@@ -64,7 +67,10 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
     standardInstances.push({
       issueID: 'attVal',
       what: 'Elements have attributes with illicit values',
+      count: data.total,
       ordinalSeverity: 2,
+      tagName: '',
+      id: '',
       location: {
         doc: '',
         type: '',

package/testaro/autocomplete.js

@@ -20,7 +20,7 @@ const addFailure = async (withItems, input, inputText, autocomplete, data) => {
     // If itemization is required:
     if (withItems) {
       // Add the item to the data.
-      data.items.push([autocomplete, inputText.slice(0, 100)]);
+      data.items.push([autocomplete, input.id || '', inputText.slice(0, 100)]);
     }
   }
 };
@@ -77,12 +77,14 @@ exports.reporter = async (page, withItems) => {
         issueID: `autocomplete-${item[0]}`,
         what: `Input is missing the required autocomplete attribute with value ${item[0]}`,
         ordinalSeverity: 2,
+        tagName: 'INPUT',
+        id: item[1] || '',
         location: {
           doc: '',
           type: '',
           spec: ''
         },
-        excerpt: item[1]
+        excerpt: item[2]
       });
     });
   }
@@ -91,6 +93,8 @@ exports.reporter = async (page, withItems) => {
       issueID: 'autocomplete',
       what: 'Inputs are missing required autocomplete attributes',
       ordinalSeverity: 2,
+      tagName: '',
+      id: '',
       location: {
         doc: '',
         type: '',

package/testaro/bulk.js

@@ -27,6 +27,8 @@ exports.reporter = async page => {
       what: 'Page contains a large number of visible elements',
       count,
       ordinalSeverity: 0,
+      tagName: 'HTML',
+      id: '',
       location: {
         doc: '',
         type: '',

package/testaro/docType.js

@@ -17,6 +17,8 @@ exports.reporter = async page => {
       issueID: 'docType',
       what: 'Document has no standard HTML doctype preamble',
       ordinalSeverity: 3,
+      tagName: 'HTML',
+      id: '',
       location: {
         doc: '',
         type: '',

package/testaro/dupAtt.js

@@ -53,11 +53,12 @@ exports.reporter = async (page, withItems) => {
   // Extract the opening tags of its elements.
   let elements = rawPage.match(/<[a-zA-Z][^<>]+>/g);
   // Delete their enclosing angle brackets and the values of any attributes in them.
-  elements = elements.map(el => el.replace(/^< *|=[^ ]*| *>$/g, ''));
+  const elementsWithVals = elements.map(el => el.replace(/^< *| *>$/g, ''));
+  const elementsWithoutVals = elementsWithVals.map(el => el.replace(/^ *|=[^ ]*$/g, ''));
   // For each element:
-  elements.forEach(element => {
+  elementsWithoutVals.forEach((elementWithoutVals, index) => {
     // Identify its tag name and attributes.
-    const terms = element.split(' ');
+    const terms = elementWithoutVals.split(' ');
     // If it has 2 or more attributes:
     if (terms.length > 2) {
       // If any is duplicated:
@@ -65,11 +66,14 @@ exports.reporter = async (page, withItems) => {
       const attributes = terms.slice(1);
       const attSet = new Set(attributes);
       if (attSet.size < attributes.length) {
-        // Add this to the data.
+        // Add it to the data.
         data.total++;
         if (withItems) {
           data.items.push({
             tagName,
+            id: terms.includes('id')
+              ? elementsWithVals[index].replace(/^.+id=/, '').replace(/ .*$/, '')
+              : '',
             attributes
           });
         }
@@ -83,6 +87,8 @@ exports.reporter = async (page, withItems) => {
         issueID: 'dupAtt',
         what: `Element ${item.tagName} has 2 attributes with the same name`,
         ordinalSeverity: 2,
+        tagName: item.tagName,
+        id: item.id,
         location: {
           doc: '',
           type: '',

package/testaro/embAc.js

@@ -32,6 +32,7 @@ exports.reporter = async (page, withItems) => await page.$$eval(
         }
         items.push({
           embeddedElement: bad.tagName,
+          embeddedID: bad.id,
           excerpt: compact(container.outerHTML)
         });
       }
@@ -45,8 +46,10 @@ exports.reporter = async (page, withItems) => await page.$$eval(
       items.forEach(item => {
         standardInstances.push({
           issueID: `embAc-${item.embeddedElement}`,
-          what: `Element ${item.embeddedElement} is embedded in a link or button`,
+          what: `${item.embeddedElement} element is embedded in a link or button`,
           ordinalSeverity: 2,
+          tagName: item.embeddedElement,
+          id: item.id,
           location: {
             doc: '',
             type: '',
@@ -62,6 +65,8 @@ exports.reporter = async (page, withItems) => await page.$$eval(
         what: 'Interactive elements are contained by links or buttons',
         count: total,
         ordinalSeverity: 2,
+        tagName: '',
+        id: '',
         location: {
           doc: '',
           type: '',

package/testaro/filter.js

@@ -40,6 +40,7 @@ exports.reporter = async (page, withItems) => {
       filterData.forEach(filterDatum => {
         data.items.push({
           tagName: filterDatum.element.tagName,
+          id: filterDatum.element.id,
           text: compact(filterDatum.element.textContent) || compact(filterDatum.element.outerHTML),
           impact: filterDatum.impact
         });
@@ -53,8 +54,10 @@ exports.reporter = async (page, withItems) => {
     data.items.forEach(item => {
       standardInstances.push({
         issueID: 'filterStyle',
-        what: `Element ${item.tagName} has a filter style that impacts ${item.impact} elements`,
+        what: `${item.tagName} element has a filter style that impacts ${item.impact} elements`,
         ordinalSeverity: 2,
+        tagName: item.toUpperCase(),
+        id: item.id,
         location: {
           doc: '',
           type: '',
@@ -69,6 +72,8 @@ exports.reporter = async (page, withItems) => {
       issueID: 'filterStyle',
       what: 'Elements have filter styles impacting other elements',
       ordinalSeverity: 2,
+      tagName: '',
+      id: '',
       location: {
         doc: '',
         type: '',

package/testaro/focAll.js

@@ -69,6 +69,8 @@ exports.reporter = async page => {
       what: 'Some focusable elements are not Tab-focusable or vice versa',
       count: Math.abs(data.discrepancy),
       ordinalSeverity: 2,
+      tagName: '',
+      id: '',
       location: {
         doc: '',
         type: '',