testaro 59.2.10 → 59.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "59.2.10",
+  "version": "59.3.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -92,6 +92,8 @@ const timeLimits = {
   ibm: 30,
   testaro: 150 + Math.round(6 * process.env.WAITS / 1000)
 };
+// Timeout multiplier.
+const timeoutMultiplier = Number.parseFloat(process.env.TIMEOUT_MULTIPLIER) || 1;
 
 // Temporary directory
 const tmpDir = os.tmpdir();
@@ -721,7 +723,7 @@ const doActs = async (report, opts = {}) => {
         const actResult = await new Promise(resolve => {
           let closed = false;
           const child = fork(
-            `${__dirname}/procs/doTestAct`, [actIndex], {timeout: 1000 * timeLimits[act.which] || 15000}
+            `${__dirname}/procs/doTestAct`, [actIndex], {timeout: timeoutMultiplier * 1000 * (timeLimits[act.which] || 15)}
           );
           child.on('message', message => {
             if (! closed) {

package/tests/testaro.js

@@ -40,25 +40,6 @@ const fs = require('fs/promises');
 // CONSTANTS
 
 // The validation job data for the tests listed below are in the pending directory.
-/*
-const futureEvalRulesTraining = {
-  altScheme: 'img elements with alt attributes having URLs as their entire values',
-  captionLoc: 'caption elements that are not first children of table elements',
-  datalistRef: 'elements with ambiguous or missing referenced datalist elements',
-  secHeading: 'headings that violate the logical level order in their sectioning containers',
-  textSem: 'semantically vague elements i, b, and/or small'
-};
-const futureEvalRulesCleanRoom = {
-  adbID: 'elements with ambiguous or missing referenced descriptions',
-  imageLink: 'links with image files as their destinations',
-  legendLoc: 'legend elements that are not first children of fieldset elements',
-  optRoleSel: 'Non-option elements with option roles that have no aria-selected attributes',
-  phOnly: 'input elements with placeholders but no accessible names'
-};
-*/
-// The following were previously marked as future (clean-room) rules.
-// For local validation runs they are included in evalRules below. Remove from futureRules
-// when preparing clean-room submissions.
 const futureRules = new Set([]);
 const evalRules = {
   adbID: 'elements with ambiguous or missing referenced descriptions',
@@ -152,6 +133,7 @@ const slowTestLimits = {
   tabNav: 10,
   textSem: 10
 };
+const timeoutMultiplier = Number.parseFloat(process.env.TIMEOUT_MULTIPLIER) || 1;
 
 // ERROR HANDLER
 process.on('unhandledRejection', reason => {
@@ -296,7 +278,7 @@ exports.reporter = async (page, report, actIndex) => {
         while (testRetries > 0 && ! testSuccess) {
           try {
             // Apply a time limit to the test.
-            const timeLimit = 1000 * (slowTestLimits[rule] ?? 5);
+            const timeLimit = 1000 * timeoutMultiplier * (slowTestLimits[rule] ?? 5);
             // If the time limit expires during the test:
             const timer = new Promise(resolve => {
               timeout = setTimeout(() => {
@@ -304,6 +286,7 @@ exports.reporter = async (page, report, actIndex) => {
                 const endTime = Date.now();
                 testTimes.push([rule, Math.round((endTime - startTime) / 1000)]);
                 data.rulePreventions.push(rule);
+                data.rulePreventionMessages[rule] = 'Timeout';
                 result[rule].totals = [0, 0, 0, 0];
                 result[rule].standardInstances = [];
                 console.log(`ERROR: Test of testaro rule ${rule} timed out`);
@@ -333,6 +316,11 @@ exports.reporter = async (page, report, actIndex) => {
                 break;
               }
             }
+            // Otherwise, i.e. if the test timed out:
+            else {
+              // Stop retrying the test.
+              break;
+            }
           }
           // If an error is thrown by the test:
           catch(error) {
@@ -378,16 +366,17 @@ exports.reporter = async (page, report, actIndex) => {
             }
           }
           finally {
-            // Clear the timeout and the error listeners.
+            // Clear the timeout.
             clearTimeout(timeout);
-            if (page && ! page.isClosed() && crashHandler) {
-              page.off('crash', crashHandler);
-            }
-            if (browser && disconnectHandler) {
-              browser.off('disconnected', disconnectHandler);
-            }
           }
         }
+        // Clear the error listeners.
+        if (page && ! page.isClosed() && crashHandler) {
+          page.off('crash', crashHandler);
+        }
+        if (browser && disconnectHandler) {
+          browser.off('disconnected', disconnectHandler);
+        }
       }
       // Otherwise, i.e. if the rule is undefined or doubly defined:
       else {