testaro 47.2.1 → 48.0.2

package/README.md

@@ -10,7 +10,9 @@ The purposes of Testaro are to:
 - provide programmatic access to accessibility tests defined by several tools
 - facilitate the integration of the reports of the tools into a unified report
 
-Testaro is described in [Testaro: Efficient Ensemble Testing for Web Accessibility](https://arxiv.org/abs/2309.10167).
+Testaro is described in two papers:
+- [How to run a thousand accessibility tests](https://medium.com/cvs-health-tech-blog/how-to-run-a-thousand-accessibility-tests-63692ad120c3)
+- [Testaro: Efficient Ensemble Testing for Web Accessibility](https://arxiv.org/abs/2309.10167)
 
 The need for multi-tool integration, and its costs, are discussed in [Accessibility Metatesting: Comparing Nine Testing Tools](https://arxiv.org/abs/2304.07591).
 
@@ -729,13 +731,11 @@ A job can be immediately executed as follows:
 
 ```javascript
 const {doJob} = require('./run');
-doJob(report)
-.then(() => …);
+doJob(job)
+.then(report => …);
 ```
 
-The `report` variable here references a copy of a job to be used as a report.
-
-Testaro will run the job and modify the `report` object. When Testaro finishes, the `acts` and `jobData` properties of `report` will contain the results. The final statement can further process the `report` object as desired in the `then` callback.
+Testaro will run the job and return a modified `report` object. When Testaro finishes, the `acts` and `jobData` properties of `report` will contain the results. The final statement can further process the `report` object as desired in the `then` callback.
 
 The Testilo package contains functions that can create jobs from scripts and add scores and explanations to reports.
 
@@ -865,7 +865,7 @@ The rationales motivating the Testaro-defined tests can be found in comments wit
 
 On some occasions a test throws an error that cannot be handled with a `try`-`catch` structure. It has been observed, for example, that the `ibm` test does this when the page content, rather than the page URL, is given to `getCompliance()` and the target is `https://globalsolutions.org`, `https://monsido.com`, or `https://www.ambetterhealth.com/`.
 
-Some tools take apparently infinite time to perform their tests on some pages. To handle such stalling, Testaro subjects most tools to time limits. Further testing will be required before it can be determined whether this time limitation is robust. As currently implemented (without child processes), it may allow tool testing processes to continue and to write indefinitely to the response.
+Some tools take apparently infinite time to perform their tests on some pages. To handle such stalling, Testaro subjects all tools to time limits. The limitation is implemented with forked child processes. Specifically, the `procs/doTestAct.js` module is run as a forked process with a `timeout` option for each of the 11 tools.
 
 ### Activation
 
@@ -902,7 +902,7 @@ Testaro would become more reliable if the behavior of its tools were monitored f
 
 ## Repository exclusions
 
-The files in the `temp` directory are presumed ephemeral and are not tracked by `git`.
+The files in the `temp` directory are presumed ephemeral and are not tracked by `git`. The above-mentioned `procs/doTestAct.js` module stores temporary reports in that directory.
 
 ## Related packages
 

package/call.js

@@ -75,7 +75,7 @@ const callRun = async jobIDStart => {
     report.observe = false;
     report.sendReportTo = '';
     // Run it.
-    await doJob(report);
+    report = await doJob(report);
     // Archive it.
     await fs.rename(`${todoDir}/${jobFileName}`, `${jobDir}/done/${jobFileName}`);
     // Save the report.

package/dirWatch.js

@@ -45,8 +45,6 @@ const reportDir = process.env.REPORTDIR;
 
 // ########## FUNCTIONS
 
-// Gets a segment of a timestamp.
-const tsPart = (timeStamp, startIndex) => timeStamp.slice(startIndex, startIndex + 2);
 // Returns a string representing the date and time.
 const nowString = () => (new Date()).toISOString().slice(2, 16);
 // Writes a directory report.
@@ -115,7 +113,7 @@ exports.dirWatch = async (isForever, intervalInSeconds) => {
           const jobJSON = await fs.readFile(`${jobDir}/todo/${jobFileNames[0]}`, 'utf8');
           try {
             const job = JSON.parse(jobJSON);
-            const report = JSON.parse(jobJSON);
+            let report = JSON.parse(jobJSON);
             // Ensure it has no server properties.
             job.observe = false;
             job.sendReportTo = '';
@@ -123,12 +121,12 @@ exports.dirWatch = async (isForever, intervalInSeconds) => {
             report.sendReportTo = '';
             const {id} = job;
             console.log(`\n\nDirectory job ${id} ready to do (${nowString()})`);
-            // Perform it.
-            await doJob(report);
+            // Perform it, adding to the report.
+            report = await doJob(report);
             console.log(`Job ${id} finished (${nowString()})`);
-            // Report it.
+            // Save the report.
             await writeDirReport(report);
-            // Archive it.
+            // Archive the job.
             await archiveJob(job, true);
           }
           catch(error) {

package/netWatch.js

@@ -164,7 +164,7 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
                     // Perform the job, adding result data to it.
                     console.log(`${logStart}job ${id} (${nowString()})`);
                     console.log(`>> It will send report to ${sendReportTo}`);
-                    await doJob(contentObj);
+                    contentObj = await doJob(contentObj);
                     let reportJSON = JSON.stringify(contentObj, null, 2);
                     console.log(`Job ${id} finished (${nowString()})`);
                     // Send the report to the specified server.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "47.2.1",
+  "version": "48.0.2",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/doTestAct.js

@@ -0,0 +1,121 @@
+/*
+  © 2024 CVS Health and/or one of its affiliates. All rights reserved.
+
+  MIT License
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+*/
+
+/*
+  doTestAct
+  Performs the tests of an act.
+*/
+
+// IMPORTS
+
+// Module to perform file operations.
+const fs = require('fs/promises');
+const {launch} = require(`${__dirname}/../run`);
+
+// CONSTANTS
+
+// Set DEBUG environment variable to 'true' to add debugging features.
+const debug = process.env.DEBUG === 'true';
+// Set WAITS environment variable to a positive number to insert delays (in ms).
+const waits = Number.parseInt(process.env.WAITS) || 0;
+
+// VARIABLES
+
+const actIndex = Number.parseInt(process.argv[2]);
+
+const doTestAct = async () => {
+  const reportPath = `${__dirname}/../temp/report.json`;
+  // Get the saved report.
+  const reportJSON = await fs.readFile(reportPath, 'utf8');
+  const report = JSON.parse(reportJSON);
+  // Get the act.
+  const act = report.acts[actIndex];
+  // Get the tool name.
+  const {which} = act;
+  // Launch a browser, navigate to the URL, and redefine the page of the run module.
+  await launch(
+    report,
+    debug,
+    waits,
+    act.launch.browserID || report.browserID,
+    act.launch.target && act.launch.target.url || report.target.url
+  );
+  // If the launch aborted the job:
+  if (report.jobData && report.jobData.aborted) {
+    // Save the revised report.
+    const reportJSON = JSON.stringify(report);
+    await fs.writeFile(reportPath, reportJSON);
+    // Report this.
+    process.send('ERROR: Job aborted');
+  }
+  // Otherwise, i.e. if the launch did not abort the job:
+  else {
+    // Get the redefined page.
+    const {page} = require('../run');
+    // If it exists:
+    if (page) {
+      try {
+        // If the page prevents the tool from testing:
+        if (page.prevented) {
+          // Report this.
+          process.send('ERROR: Page prevented testing');
+        }
+        // Otherwise, i.e. if the page permits testing:
+        else {
+          // Wait for the act reporter to perform the specified tests of the tool.
+          const actReport = await require(`../tests/${which}`).reporter(page, report, actIndex, 65);
+          // Add the data and result to the act.
+          act.data = actReport.data;
+          act.result = actReport.result;
+          // If the tool reported that the page prevented testing:
+          if (actReport.data && actReport.data.prevented) {
+            // Add prevention data to the job data.
+            report.jobData.preventions[which] = act.data.error;
+          }
+          const reportJSON = JSON.stringify(report);
+          // Save the revised report.
+          await fs.writeFile(reportPath, reportJSON);
+          // Send a completion message.
+          process.send('Act completed');
+        }
+      }
+      // If the tool invocation failed:
+      catch(error) {
+        // Save the revised report.
+        const reportJSON = JSON.stringify(report);
+        await fs.writeFile(reportPath, reportJSON);
+        // Report the failure.
+        const message = error.message.slice(0, 400);
+        console.log(`ERROR: Test act ${act.which} failed (${message})`);
+        process.send('ERROR performing the act');
+      };
+    }
+    // Otherwise, i.e. if the page does not exist:
+    else {
+      process.send('ERROR: No page');
+    }
+  }
+};
+
+doTestAct();