testaro 33.1.3 → 34.1.2

package/README.md

@@ -424,7 +424,7 @@ When you include a `rules` property, you limit the tests of the tool that are pe
 
 The `nuVal`, `qualWeb`, and `testaro` tools require specific formats for the `rules` property. Those formats are described below in the sections about those tools.
 
-The `aslint` and `ed11y` tools do not yet allow rule specification.
+The `ed11y` tool does not yet allow rule specification.
 
 ###### Examples
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "33.1.3",
+  "version": "34.1.2",
   "description": "Run 1000 web accessibility tests from 10 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/standardize.js

@@ -61,7 +61,12 @@ const getIdentifiers = code => {
   }
   return [tagName, id];
 };
-// Specifies conversions of rule IDs of aslint based on what substrings.
+/*
+  Differentiates some rule IDs of aslint.
+  If the purported rule ID is a key and the what property contains all of the strings except the
+  last of any array item of the value of that key, then the final rule ID is the last item of that
+  array item.
+*/
 const aslintData = {
   'misused_required_attribute': [
     ['not needed', 'misused_required_attributeR']
@@ -293,7 +298,6 @@ const doQualWeb = (result, standardResult, ruleClassName) => {
             excerpt: cap(htmlCode)
           };
           standardResult.instances.push(instance);
-          standardResult.totals[instance.ordinalSeverity]++;
         });
       });
     });
@@ -345,7 +349,6 @@ const convert = (toolName, data, result, standardResult) => {
   }
   // alfa
   else if (toolName === 'alfa' && result.totals) {
-    standardResult.totals = [result.totals.warnings, 0, 0, result.totals.failures];
     result.items.forEach(item => {
       const {codeLines} = item.target;
       const code = Array.isArray(codeLines) ? codeLines.join(' ') : '';
@@ -375,10 +378,6 @@ const convert = (toolName, data, result, standardResult) => {
   }
   // aslint
   else if (toolName === 'aslint' && result.summary && result.summary.byIssueType) {
-    // Get the totals.
-    standardResult.totals = [
-      result.summary.byIssueType.warning, 0, 0, result.summary.byIssueType.error
-    ];
     // For each rule:
     Object.keys(result.rules).forEach(ruleID => {
       // If it has a valid issue type:
@@ -396,7 +395,7 @@ const convert = (toolName, data, result, standardResult) => {
               && ruleResult.message.actual.description
             ) {
               const what = ruleResult.message.actual.description;
-              // Get its differentiated ID if any.
+              // Get the differentiated ID of the rule if any.
               const ruleData = aslintData[ruleID];
               let finalRuleID = ruleID;
               if (ruleData) {
@@ -407,6 +406,7 @@ const convert = (toolName, data, result, standardResult) => {
                   finalRuleID = changer[changer.length - 1];
                 }
               }
+              // Get the instance properties.
               const xpath = ruleResult.element && ruleResult.element.xpath || '';
               let tagName = xpath
               && xpath.replace(/^.*\//, '').replace(/[^-\w].*$/, '').toUpperCase()
@@ -449,13 +449,6 @@ const convert = (toolName, data, result, standardResult) => {
     && result.totals
     && (result.totals.rulesWarned || result.totals.rulesViolated)
   ) {
-    const {totals} = result;
-    standardResult.totals = [
-      totals.warnings.minor + totals.warnings.moderate,
-      totals.warnings.serious + totals.warnings.critical,
-      totals.violations.minor + totals.violations.moderate,
-      totals.violations.serious + totals.violations.critical
-    ];
     doAxe(result, standardResult, 'incomplete');
     doAxe(result, standardResult, 'violations');
   }
@@ -465,11 +458,8 @@ const convert = (toolName, data, result, standardResult) => {
     && result
     && ['results', 'errorCount', 'warningCount'].every(key => result[key] !== undefined)
   ) {
-    // Populate the totals of the standard result.
-    const {results, errorCount, warningCount} = result;
-    standardResult.totals = [warningCount, 0, errorCount, 0];
     // For each result:
-    results.forEach(result => {
+    result.results.forEach(result => {
       const {test, content, tagName, id, loc, excerpt} = result;
       if (['test', 'content'].every(key => result[key])) {
         // Standardize the what property.
@@ -501,17 +491,9 @@ const convert = (toolName, data, result, standardResult) => {
   else if (toolName === 'htmlcs' && result) {
     doHTMLCS(result, standardResult, 'Warning');
     doHTMLCS(result, standardResult, 'Error');
-    const {instances} = standardResult;
-    standardResult.totals = [
-      instances.filter(instance => instance.ordinalSeverity === 0).length,
-      0,
-      0,
-      instances.filter(instance => instance.ordinalSeverity === 3).length
-    ];
   }
   // ibm
   else if (toolName === 'ibm' && result.totals) {
-    standardResult.totals = [0, result.totals.recommendation, 0, result.totals.violation];
     if (result.items) {
       result.items.forEach(item => {
         const identifiers = getIdentifiers(item.snippet);
@@ -547,12 +529,6 @@ const convert = (toolName, data, result, standardResult) => {
       doNuVal(result, standardResult, 'rawPage');
     }
     const {instances} = standardResult;
-    standardResult.totals = [
-      instances.filter(instance => instance.ordinalSeverity === 0).length,
-      0,
-      instances.filter(instance => instance.ordinalSeverity === 2).length,
-      instances.filter(instance => instance.ordinalSeverity === 3).length,
-    ];
   }
   // qualWeb
   else if (
@@ -564,7 +540,6 @@ const convert = (toolName, data, result, standardResult) => {
       || result.modules['best-practices']
     )
   ) {
-    standardResult.totals = [0, 0, 0, 0];
     if (result.modules['act-rules']) {
       doQualWeb(result, standardResult, 'act-rules');
     }
@@ -577,24 +552,29 @@ const convert = (toolName, data, result, standardResult) => {
   }
   // testaro
   else if (toolName === 'testaro') {
+    // For each violated rule:
     const rules = result ? Object.keys(result) : [];
-    standardResult.totals = [0, 0, 0, 0];
     rules.forEach(rule => {
+      // Copy its instances to the standard result.
       const ruleResult = result[rule];
-      standardResult.totals.forEach((total, index) => {
-        standardResult.totals[index] += ruleResult
-        && ruleResult.totals ? ruleResult.totals[index] || 0 : 0;
-      });
       if (ruleResult.standardInstances) {
         standardResult.instances.push(... ruleResult.standardInstances);
       }
       else {
         console.log(`ERROR: Testaro rule ${rule} result has no standardInstances property`);
       }
+      // Add its (sample-ratio-weighted) totals to the totals of the standard result.
+      if (ruleResult.totals) {
+        for (const index in ruleResult.totals) {
+          standardResult.totals[index] += ruleResult.totals[index];
+        }
+      }
+      else {
+        console.log(`ERROR: Testaro rule ${rule} result has no totals property`);
+      }
     });
     const preventionCount = result.preventions && result.preventions.length;
     if (preventionCount) {
-      standardResult.totals[3] += preventionCount;
       standardResult.instances.push({
         ruleID: 'testPrevention',
         what: 'Page prevented tests from being performed',
@@ -606,7 +586,6 @@ const convert = (toolName, data, result, standardResult) => {
         excerpt: ''
       });
     }
-    standardResult.totals = standardResult.totals.map(total => Math.round(total));
   }
   // wave
   else if (
@@ -618,21 +597,18 @@ const convert = (toolName, data, result, standardResult) => {
       || result.categories.alert
     )
   ) {
-    const {categories} = result;
-    standardResult.totals = [
-      categories.alert.count || 0,
-      0,
-      0,
-      (categories.error.count || 0) + (categories.contrast.count || 0)
-    ];
     ['error', 'contrast', 'alert'].forEach(categoryName => {
       doWAVE(result, standardResult, categoryName);
     });
   }
-  // Any tool with no reported rule violations:
-  else {
-    standardResult.totals = [0, 0, 0, 0];
+  // Populate the totals of the standard result if the tool is not Testaro.
+  if (toolName !== 'testaro') {
+    standardResult.instances.forEach(instance => {
+      standardResult.totals[instance.ordinalSeverity] += instance.count || 1;
+    });
   }
+  // Round the totals of the standard result.
+  standardResult.totals = standardResult.totals.map(total => Math.round(total));
 };
 // Converts the results.
 exports.standardize = act => {

package/run.js

@@ -1088,7 +1088,7 @@ const doActs = async (report, actIndex, page) => {
             if (['also', 'only'].includes(standard)) {
               // Initialize it.
               act.standardResult = {
-                totals: [],
+                totals: [0, 0, 0, 0],
                 instances: []
               };
               // Populate it.

package/tests/aslint.js

@@ -43,7 +43,7 @@ exports.reporter = async (page, options) => {
     `${__dirname}/../node_modules/aslint-testaro/aslint.bundle.js`, 'utf8'
   );
   // Get the nonce, if any.
-  const {report} = options;
+  const {act, report} = options;
   const {jobData} = report;
   const scriptNonce = jobData && jobData.lastScriptNonce;
   // Inject the ASLint bundle and runner into the page.
@@ -94,10 +94,16 @@ exports.reporter = async (page, options) => {
     const actReport = await reportLoc.textContent();
     // Populate the act report.
     result = JSON.parse(actReport);
-    // Delete irrelevant properties from the tool report details.
+    // If any rules were reported violated:
     if (result.rules) {
+      // For each such rule:
       Object.keys(result.rules).forEach(ruleID => {
-        if (['passed', 'skipped'].includes(result.rules[ruleID].status.type)) {
+        // If the rule was passed or skipped or rules to be tested were specified and exclude it:
+        const excluded = act.rules && ! act.rules.includes(ruleID);
+        const instanceType = result.rules[ruleID].status.type;
+        // If rules to be tested were specified and exclude it or the rule was passed or skipped:
+        if (excluded || ['passed', 'skipped'].includes(instanceType)) {
+          // Delete the rule report.
           delete result.rules[ruleID];
         }
       });