testaro 32.2.3 → 32.3.3

package/README.md

@@ -565,6 +565,8 @@ These changes were proposed as pull requests 1333 and 1334 (https://github.com/I
 
 The `ibm` tool is one of two tools (`testaro` is the other) with a `withItems` property. If you set `withItems` to `false`, the result includes the counts of “violations” and “recommendations”, but no information about the rules that gave rise to them.
 
+Experimentation indicates that the `ibm` tools emits untrappable errors for some targets when the content argument given to it is the page content rather than the page URL. Therefore, it is safer to use `true` as the value of `withNewContent` for the `ibm` tool.
+
 ###### Nu Html Checker
 
 The `nuVal` tool performs the tests of the Nu Html Checker.
@@ -876,7 +878,7 @@ The rationales motivating the Testaro-defined tests can be found in comments wit
 
 ### Abnormal termination
 
-On rare occasions a test throws an error that terminates the Node process and cannot be handled with a `try`-`catch` structure. It has been observed, for example, that the `ibm` test does this when run on the host at `https://zenythgroup.com/index` or `https://monsido.com`.
+On rare occasions a test throws an error that cannot be handled with a `try`-`catch` structure. It has been observed, for example, that the `ibm` test does this when the page content, rather than the page URL, is given to `getCompliance()` and the target is `https://globalsolutions.org` or `https://monsido.com`.
 
 ### Activation
 

package/actSpecs.js

@@ -184,7 +184,7 @@ exports.actSpecs = {
       {
         withItems: [true, 'boolean', '', 'itemize'],
         withNewContent: [
-          true, 'boolean', '', 'true: use a URL; false: use page content'
+          true, 'boolean', '', 'true: use a URL; false: use page content (risky)'
         ]
       }
     ],

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "testaro",
-  "version": "32.2.3",
-  "description": "Run 960 web accessibility tests from 9 tools and get a standardized report",
+  "version": "32.3.3",
+  "description": "Run 900 web accessibility tests from 9 tools and get a standardized report",
   "main": "index.js",
   "scripts": {
     "tests": "node validation/executors/tests",

package/run.js

@@ -422,10 +422,19 @@ const launch = async (report, typeName, url, debug, waits, isLowMotion = false)
     // Open a context (i.e. browser tab), with reduced motion if specified.
     const options = {reduceMotion: isLowMotion ? 'reduce' : 'no-preference'};
     const browserContext = await browser.newContext(options);
-    // Prevent default timeouts, including those arising from a Playwright bug.
+    // Prevent default timeouts.
     browserContext.setDefaultTimeout(0);
     // When a page (i.e. browser tab) is added to the browser context (i.e. browser window):
     browserContext.on('page', async page => {
+      page.on('crash', () => {
+        console.log('Page crashed');
+      });
+      page.on('pageerror', () => {
+        console.log('Page erred');
+      });
+      page.on('requestfailed', () => {
+        console.log('Request failed');
+      });
       // If it emits a message:
       page.on('console', msg => {
         const msgText = msg.text();

package/tests/ibm.js

@@ -41,27 +41,18 @@ const {getCompliance} = require('accessibility-checker');
 // FUNCTIONS
 
 // Runs the IBM test and returns the result.
-const run = async (content, timeLimit) => {
+const run = async content => {
   const nowLabel = (new Date()).toISOString().slice(0, 19);
-  // Start the timeout clock.
-  let timeoutID;
-  const timeout = new Promise(resolve => {
-    timeoutID = setTimeout(() => {
-      resolve(null);
-    }, 1000 * timeLimit);
-  });
-  // Return the result of the test, or null if it timed out.
   try {
-    const ibmReport = getCompliance(content, nowLabel);
-    const result = await Promise.race([ibmReport, timeout]);
-    clearTimeout(timeoutID);
-    return result;
+    const ibmReport = await getCompliance(content, nowLabel);
+    return ibmReport;
   }
   catch(error) {
-    console.log(
-      `ERROR: getCompliance failed (${error.message.replace(/\s+/g, ' ').slice(0, 200)}).`
-    );
-    return null;
+    console.log('ibm getCompliance failed');
+    return {
+      prevented: true,
+      error: error.message.slice(0, 200)
+    };
   }
 };
 // Revises act-report totals for any rule limitation.
@@ -114,8 +105,10 @@ const trimActReport = (data, actReport, withItems, rules) => {
     // Otherwise, i.e. if it excludes totals:
     else {
       // Report this.
+      const error = 'ERROR: No totals reported';
+      console.log(error);
       data.prevented = true;
-      data.error = 'ERROR: No totals reported';
+      data.error = error;
       // Return an empty act report.
       return {
         totals: {},
@@ -126,8 +119,10 @@ const trimActReport = (data, actReport, withItems, rules) => {
   // Otherwise, i.e. if it excludes a summary:
   else {
     // Report this.
+    const error = 'ERROR: No summary reported';
+    console.log(error);
     data.prevented = true;
-    data.error = 'ERROR: No summary reported';
+    data.error = error;
     // Return an empty act report.
     return {
       totals: {},
@@ -137,27 +132,38 @@ const trimActReport = (data, actReport, withItems, rules) => {
 };
 // Performs the IBM tests and returns an act report.
 const doTest = async (content, withItems, timeLimit, rules) => {
-  // Conduct the test and get the result.
+  // Conduct the tests.
   const data = {};
   try {
     const runReport = await run(content, timeLimit);
-    const actReport = runReport && runReport.report;
-    // Delete any report files.
-    try {
-      const reportNames = await fs.readdir('ibmOutput');
-      for (const reportName of reportNames) {
-        await fs.rm(`ibmOutput/${reportName}`);
+    // If there were results:
+    if (runReport.report) {
+      // Delete any report files.
+      try {
+        const reportNames = await fs.readdir('ibmOutput');
+        for (const reportName of reportNames) {
+          await fs.rm(`ibmOutput/${reportName}`);
+        }
       }
+      catch(error) {
+        console.log('No result files created');
+      };
+      // Return a trimmed act report.
+      const {report} = runReport;
+      const trimmedReport = trimActReport(data, report, withItems, rules);
+      return {
+        data,
+        result: trimmedReport
+      };
     }
-    catch(error) {
-      console.log('No result files created');
+    // Otherwise, i.e. if there were no results:
+    else {
+      // Return this.
+      return {
+        data: runReport,
+        result: {}
+      }
     }
-    // Return a trimmed act report.
-    const trimmedReport = trimActReport(data, actReport, withItems, rules);
-    return {
-      data,
-      result: trimmedReport
-    };
   }
   catch(error) {
     const message = `Act failed (${error.message.slice(0, 200)})`;
@@ -186,6 +192,7 @@ exports.reporter = async (page, options) => {
       const message = `ERROR: Act failed or timed out at ${timeLimit} seconds`;
       console.log(message);
       data.error = data.error ? `${data.error}; ${message}` : message;
+      // Return the failure.
       return {
         data,
         result: {}