testaro 43.0.2 → 44.0.0

package/run.js

@@ -76,6 +76,13 @@ const errorWords = [
   'violates',
   'warning'
 ];
+// Time limits on tools.
+const timeLimits = {
+  alfa: 20,
+  ed11y: 30,
+  ibm: 30,
+  testaro: 60
+};
 
 // ########## VARIABLES
 
@@ -502,7 +509,7 @@ const abortActs = async (report, actIndex) => {
   report.jobData.abortedAct = actIndex;
   report.jobData.aborted = true;
   // Report that the job is aborted.
-  console.log('ERROR: Job aborted');
+  console.log(`ERROR: Job aborted on act ${actIndex}`);
   // Return an abortive act index.
   return -2;
 };
@@ -527,7 +534,6 @@ const addError = async(alsoLog, alsoAbort, report, actIndex, message) => {
   }
   // If the job is to be aborted:
   if (alsoAbort) {
-    console.log(`report:\n${JSON.stringify(report, null, 2)}`);
     // Return an abortive act index.
     return await abortActs(report, actIndex);
   }
@@ -816,23 +822,46 @@ const doActs = async (report, actIndex, page) => {
           });
           // Get the start time of the act.
           const startTime = Date.now();
-          // Perform the specified tests of the tool and get a report.
+          let timer;
           try {
-            const actReport = await require(`./tests/${act.which}`).reporter(page, options);
-            // Import its test results and process data into the act.
-            act.result = actReport && actReport.result || {};
-            act.data = actReport && actReport.data || {};
-            // If the page prevented the tool from operating:
-            if (act.data.prevented) {
-              // Add prevention data to the job data.
-              report.jobData.preventions[act.which] = act.data.error;
+            // Impose a time limit on the act.
+            let timeoutReport = 'onTime';
+            const timeLimit = 1000 * timeLimits[act.which] || 15000;
+            timeoutReport = new Promise(resolve => {
+              timer = setTimeout(() => {
+                act.data = {
+                  prevented: true,
+                  error: `Act timed out at ${timeLimit / 1000} seconds`
+                };
+                console.log(`ERROR: Timed out at ${timeLimit / 1000} seconds`);
+                resolve('timedOut');
+              }, timeLimit);
+            });
+            // Try to perform the specified tests of the tool and get a report.
+            const actReport = require(`./tests/${act.which}`).reporter(page, options);
+            const raceReport = await Promise.race([timeoutReport, actReport]);
+            // If the act was finished without timing out:
+            if (raceReport !== 'timedOut') {
+              // Disable the timer.
+              clearTimeout(timer);
+              // Import the test results and process data into the act.
+              act.result = raceReport && raceReport.result || {};
+              act.data = raceReport && raceReport.data || {};
+              // If the page prevented the tool from operating:
+              if (act.data.prevented) {
+                // Add prevention data to the job data.
+                report.jobData.preventions[act.which] = act.data.error;
+              }
             }
           }
-          // If the testing failed:
+          // If the testing failed other than by timing out:
           catch(error) {
-            // Report this.
+            // Disable the timer.
+            clearTimeout(timer);
+            // Report the failure.
             const message = error.message.slice(0, 400);
             console.log(`ERROR: Test act ${act.which} failed (${message})`);
+            act.data.prevented = true;
             act.data.error = act.data.error ? `${act.data.error}; ${message}` : message;
           }
           // Add the elapsed time of the tool to the report.
@@ -842,9 +871,9 @@ const doActs = async (report, actIndex, page) => {
             toolTimes[act.which] = 0;
           }
           toolTimes[act.which] += time;
-          // If a standard-format result is to be included in the report:
+          // If the act was not prevented and standardization is required:
           const standard = report.standard || 'only';
-          if (['also', 'only'].includes(standard)) {
+          if (! act.data.prevented && ['also', 'only'].includes(standard)) {
             // Initialize it.
             act.standardResult = {
               totals: [0, 0, 0, 0],
@@ -868,9 +897,9 @@ const doActs = async (report, actIndex, page) => {
               delete act.result;
             }
           }
-          // If the act has expectations:
+          // If the act was not prevented and has expectations:
           const expectations = act.expect;
-          if (expectations) {
+          if (! act.data.prevented && expectations) {
             // Initialize whether they were fulfilled.
             act.expectations = [];
             let failureCount = 0;

package/tests/testaro.js

@@ -206,8 +206,8 @@ exports.reporter = async (page, options) => {
           result[rule] = {};
         }
         result[rule].what = what;
+        const startTime = Date.now();
         try {
-          const startTime = Date.now();
           // Apply a 15-second time limit to the test. If it expires:
           let timeout;
           const timer = new Promise(resolve => {