testaro 8.4.6 → 8.5.2

package/README.md

@@ -362,9 +362,53 @@ An example of a **Testaro-defined** test is:
 
 In this case, Testaro runs the `motion` test with the specified parameters.
 
+###### Continuum
+
+The `continuum` packaged test makes use of the files in the `continuum` directory. The test inserts the contents of all three files into the page as scripts and then uses them to perform the tests of the Continuum package.
+
+Level Access on 22 August 2022 granted authorization for the copying of the `AccessEngine.community.js` file insofar as necessary for allowing Continuum community edition tests to be included in Testaro.
+
+###### IBM Equal Access
+
+The `ibm` packaged test requires the `aceconfig.js` file.
+
+If you choose to invoke this packaged test with the `withNewContent` property specified, you will choose whether the tested content is the content of the existing page or is retrieved anew with the document URL. Typically, both methods succeed and deliver similar results. However, sometimes one method succeeds and the other fails, or one method reports more violations than the other does. In those cases, most often the success, or the larger violation count, arises from the existing page (`withNewContent: false`).
+
+###### HTML CodeSniffer
+
+The `htmlcs` packaged test makes use of the`htmlcs/HTMLCS.js` file. That file was created, and can be recreated if necessary, as follows:
+
+1. Clone the (HTML CodeSniffer package)[https://github.com/squizlabs/HTML_CodeSniffer].
+1. Make that package’s directory the active directory.
+1. Install the HTML CodeSniffer dependencies by executing `npm install`.
+1. Build the HTML CodeSniffer auditor by executing `grunt build`.
+1. Copy the `build/HTMLCS.js` and `build/licence.txt` files into the `htmlcs` directory of Testaro.
+1. Edit the Testaro copy of `htmlcs/HTMLCS.js` to produce the changes shown below.
+
+The changes in `htmlcs/HTMLCS.js` are:
+
+```diff
+479a480
+>     '4_1_2_attribute': 'attribute',
+6482a6484
+>     var messageStrings = new Set();
+6496d6497
+<         console.log('done');
+6499d6499
+<         console.log('done');
+6500a6501
+>       return Array.from(messageStrings);
+6531c6532,6534
+<       console.log('[HTMLCS] ' + typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html);
+---
+>       messageStrings.add(
+>         typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html
+>       );
+```
+
 ###### Tenon
 
-Most packaged tests require only one command, but the `tenon` test requires two commands:
+Most packaged tests require only one command, but the `tenon` packaged test requires two commands:
 - A command of type `tenonRequest`.
 - A command of type `test` with `tenon` as the value of `which`.
 
@@ -404,44 +448,6 @@ Tenon recommends giving it a public URL rather than giving it the content of a p
 
 If a `tenon` test is included in a job, environment variables named `TENON_USER` and `TENON_PASSWORD` must exist, with your Tenon username and password, respectively, as their values. You can obtain those from [Tenon](https://tenon.io/documentation/overview).
 
-###### Continuum
-
-The `continuum` test makes use of the files in the `continuum` directory. The test inserts the contents of all three files into the page as scripts and then uses them to perform the tests of the Continuum package.
-
-Level Access on 22 August 2022 granted authorization for the copying of the `AccessEngine.community.js` file insofar as necessary for allowing Continuum community edition tests to be included in Testaro.
-
-###### HTML CodeSniffer
-
-The `htmlcs` test makes use of the`htmlcs/HTMLCS.js` file. That file was created, and can be recreated if necessary, as follows:
-
-1. Clone the (HTML CodeSniffer package)[https://github.com/squizlabs/HTML_CodeSniffer].
-1. Make that package’s directory the active directory.
-1. Install the HTML CodeSniffer dependencies by executing `npm install`.
-1. Build the HTML CodeSniffer auditor by executing `grunt build`.
-1. Copy the `build/HTMLCS.js` and `build/licence.txt` files into the `htmlcs` directory of Testaro.
-1. Edit the Testaro copy of `htmlcs/HTMLCS.js` to produce the changes shown below.
-
-The changes in `htmlcs/HTMLCS.js` are:
-
-```diff
-479a480
->     '4_1_2_attribute': 'attribute',
-6482a6484
->     var messageStrings = new Set();
-6496d6497
-<         console.log('done');
-6499d6499
-<         console.log('done');
-6500a6501
->       return Array.from(messageStrings);
-6531c6532,6534
-<       console.log('[HTMLCS] ' + typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html);
----
->       messageStrings.add(
->         typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html
->       );
-```
-
 ###### WAVE
 
 If a `wave` test is included in the job, an environment variable named `WAVE_KEY` must exist, with your WAVE API key as its value. You can get it from [WebAIM](https://wave.webaim.org/api/).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "8.4.6",
+  "version": "8.5.2",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -9,6 +9,8 @@
 require('dotenv').config();
 // Requirements for commands.
 const {commands} = require('./commands');
+// Playwright package.
+const playwright = require('playwright');
 
 // ########## CONSTANTS
 
@@ -295,7 +297,7 @@ const browserClose = async () => {
 const errorStart = error => error.message.replace(/\n.+/s, '');
 // Launches a browser.
 const launch = async (report, typeName, lowMotion = false) => {
-  const browserType = require('playwright')[typeName];
+  const browserType = playwright[typeName];
   // If the specified browser type exists:
   if (browserType) {
     // Close the current browser, if any.
@@ -604,6 +606,18 @@ const goTo = async (report, page, url, timeout, waitUntil, isStrict) => {
 };
 // Recursively performs the acts in a report.
 const doActs = async (report, actIndex, page) => {
+  // Quits and reports the performance being aborted.
+  const abortActs = async () => {
+    // Add data on the aborted command to the report.
+    report.jobData.abortTime = nowString();
+    report.jobData.abortedAct = actIndex;
+    // Prevent performance of additional commands.
+    actIndex = -2;
+    // Report this.
+    console.log('ERROR: Job aborted');
+    // Stop the action until the user resumes it.
+    await page.pause();
+  };
   process.on('message', message => {
     if (message === 'interrupt') {
       console.log('ERROR: Terminal interrupted doActs');
@@ -643,7 +657,7 @@ const doActs = async (report, actIndex, page) => {
         if (truth[1]) {
           // If the performance of commands is to stop:
           if (act.jump === 0) {
-            // Stop.
+            // Quit.
             actIndex = -2;
           }
           // Otherwise, if there is a numerical jump:
@@ -712,23 +726,17 @@ const doActs = async (report, actIndex, page) => {
           }
           // If none of the visits succeeded:
           if (response.error) {
-            // Report this.
-            report.jobData.aborted = true;
-            report.jobData.abortedAct = actIndex;
+            // Report this and quit.
             addError(act, 'failure', 'ERROR: Visits failed');
-            // Quit.
-            actIndex = -2;
+            await abortActs();
           }
           // Otherwise, i.e. if the last visit attempt succeeded:
           else {
             // If a prohibited redirection occurred:
             if (response.exception === 'badRedirection') {
-              // Report this.
-              report.jobData.aborted = true;
-              report.jobData.abortedAct = actIndex;
+              // Report this and quit.
               addError(act, 'badRedirection', 'ERROR: Navigation illicitly redirected');
-              // Quit.
-              actIndex = -2;
+              await abortActs();
             }
             // Add the resulting URL to the act.
             if (! act.result) {
@@ -744,20 +752,22 @@ const doActs = async (report, actIndex, page) => {
           const result = act.result = {};
           // If the text is to be the URL:
           if (what === 'url') {
-            // Wait for the URL to be the exact text and quit on failure.
+            // Wait for the URL to be the exact text.
             try {
               await page.waitForURL(which, {timeout: 15000});
               result.found = true;
               result.url = page.url();
             }
+            // If the wait times out:
             catch(error) {
-              actIndex = -2;
+              // Quit.
+              await abortActs();
               waitError(page, act, error, 'text in the URL');
             }
           }
           // Otherwise, if the text is to be a substring of the page title:
           else if (what === 'title') {
-            // Wait for the page title to include the text, case-insensitively, and quit on failure.
+            // Wait for the page title to include the text, case-insensitively.
             try {
               await page.waitForFunction(
                 text => document
@@ -772,14 +782,16 @@ const doActs = async (report, actIndex, page) => {
               result.found = true;
               result.title = await page.title();
             }
+            // If the wait times out:
             catch(error) {
-              actIndex = -2;
+              // Quit.
+              await abortActs();
               waitError(page, act, error, 'text in the title');
             }
           }
           // Otherwise, if the text is to be a substring of the text of the page body:
           else if (what === 'body') {
-            // Wait for the body to include the text, case-insensitively, and quit on failure.
+            // Wait for the body to include the text, case-insensitively.
             try {
               await page.waitForFunction(
                 text => document
@@ -793,25 +805,31 @@ const doActs = async (report, actIndex, page) => {
               );
               result.found = true;
             }
+            // If the wait times out:
             catch(error) {
-              actIndex = -2;
+              // Quit.
+              await abortActs();
               waitError(page, act, error, 'text in the body');
             }
           }
         }
         // Otherwise, if the act is a wait for a state:
         else if (act.type === 'state') {
-          // Wait for it and quit on failure.
+          // Wait for it.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
           await page.waitForLoadState(
             ['domcontentloaded', 'networkidle'][stateIndex], {timeout: [10000, 15000][stateIndex]}
           )
-          .catch(error => {
+          // If the wait times out:
+          .catch(async error => {
+            // Quit.
             console.log(`ERROR waiting for page to be ${act.which} (${error.message})`);
             addError(act, 'timeout', `ERROR waiting for page to be ${act.which}`);
-            actIndex = -2;
+            await abortActs();
           });
+          // If the wait succeeded:
           if (actIndex > -2) {
+            // Add state data to the report.
             act.result = {
               success: true,
               state: act.which
@@ -1085,8 +1103,9 @@ const doActs = async (report, actIndex, page) => {
               // If a match was found:
               if (act.result.found) {
                 // FUNCTION DEFINITION START
-                // Perform a click or Enter keypress and wait for the network to be idle.
+                // Performs a click or Enter keypress and waits for the network to be idle.
                 const doAndWait = async isClick => {
+                  // Perform and report the move.
                   const move = isClick ? 'click' : 'Enter keypress';
                   try {
                     await isClick
@@ -1095,12 +1114,14 @@ const doActs = async (report, actIndex, page) => {
                     act.result.success = true;
                     act.result.move = move;
                   }
+                  // If the move fails:
                   catch(error) {
+                    // Quit and add failure data to the report.
                     act.result.success = false;
                     act.result.error = 'moveFailure';
                     act.result.message = `ERROR: ${move} failed`;
                     console.log(`ERROR: ${move} failed (${errorStart(error)})`);
-                    actIndex = -2;
+                    await abortActs();
                   }
                   if (act.result.success) {
                     try {
@@ -1186,15 +1207,16 @@ const doActs = async (report, actIndex, page) => {
                     }
                     // If the click or load failed:
                     catch(error) {
-                      // Quit and report the failure.
+                      // Quit and add failure data to the report.
                       console.log(`ERROR clicking link (${errorStart(error)})`);
                       act.result.success = false;
                       act.result.error = 'unclickable';
                       act.result.message = 'ERROR: click or load timed out';
-                      actIndex = -2;
+                      await abortActs();
                     }
                     // If the link click succeeded:
                     if (! act.result.error) {
+                      // Add success data to the report.
                       act.result.success = true;
                       act.result.move = 'clicked';
                     }
@@ -1263,12 +1285,12 @@ const doActs = async (report, actIndex, page) => {
               }
               // Otherwise, i.e. if no match was found:
               else {
-                // Stop.
+                // Quit and add failure data to the report.
                 act.result.success = false;
                 act.result.error = 'absent';
                 act.result.message = 'ERROR: specified element not found';
                 console.log('ERROR: Specified element not found');
-                actIndex = -2;
+                await abortActs();
               }
             }
             // Otherwise, if the act is a keypress:
@@ -1460,18 +1482,18 @@ const doActs = async (report, actIndex, page) => {
     }
     // Otherwise, i.e. if the command is invalid:
     else {
-      // Add an error result to the act.
+      // Quit and add error data to the report.
       const errorMsg = `ERROR: Invalid command of type ${act.type}`;
       console.log(errorMsg);
       addError(act, 'badCommand', errorMsg);
-      // Quit.
-      actIndex = -2;
+      await abortActs();
     }
-    // Perform the remaining acts unless the performance of acts has been aborted.
+    // Perform any remaining acts if not aborted.
     await doActs(report, actIndex + 1, page);
   }
-  else {
-    console.log('No more acts to perform; closing the browser');
+  // Otherwise, if all commands have been performed and the job succeeded:
+  else if (! report.jobData.abortTime) {
+    console.log('Commands completed');
     await browserClose();
     console.log('Browser closed');
   }
@@ -1523,7 +1545,10 @@ const injectLaunches = acts => {
     }
   }
 };
-// Runs a job and adds the results to the report of the job.
+/*
+  Returns whether an initialized job report is valid and, if so, runs the job and adds the results
+  to the report.
+*/
 exports.doJob = async report => {
   // If the report is valid:
   if(isValidReport(report)) {
@@ -1554,16 +1579,14 @@ exports.doJob = async report => {
     report.jobData.presses = 0;
     report.jobData.amountRead = 0;
     report.jobData.testTimes = [];
-    // Recursively perform the specified acts.
+    // Recursively perform the acts and get any page if debugging is on and a job was aborted.
     await doActs(report, 0, null);
     // Add the end time and duration to the report.
     const endTime = new Date();
     report.jobData.endTime = nowString();
     report.jobData.elapsedSeconds =  Math.floor((endTime - startTime) / 1000);
-    return true;
   }
   else {
     console.log('ERROR: Initialized job report invalid');
-    return false;
   }
 };

package/watch.js

@@ -96,19 +96,16 @@ const writeDirReport = async report => {
       const reportName = `${jobID}.json`;
       await fs.writeFile(`${reportDir}/${reportName}`, reportJSON);
       console.log(`Report ${reportName} saved`);
-      return true;
     }
     catch(error) {
       console.log(`ERROR: Failed to write report (${error.message})`);
-      return false;
     }
   }
   else {
     console.log('ERROR: Job has no ID');
-    return false;
   }
 };
-// Submits a network report.
+// Submits a network report to a server and returns the server response.
 const writeNetReport = async report => {
   const ack = await new Promise(resolve => {
     const wholeURL = `${process.env.PROTOCOL}://${reportURL}`;
@@ -135,12 +132,17 @@ const writeNetReport = async report => {
     report.agent = agent;
     request.on('error', error => {
       console.log(`ERROR submitting job report (${error.message})`);
-      resolve({});
+      resolve({
+        error: 'ERROR: Job report submission failed',
+        message: error.message
+      });
     });
+    // Send the report to the server.
     request.write(JSON.stringify(report, null, 2));
     request.end();
     console.log(`Report ${report.job.id} submitted`);
   });
+  // Return the server response.
   return ack;
 };
 // Archives a job.
@@ -157,8 +159,9 @@ const wait = ms => {
     }, ms);
   });
 };
-// Runs a job and returns a report.
+// Runs a job and returns the page.
 const runJob = async (job, isDirWatch) => {
+  // If the job has an ID:
   const {id} = job;
   if (id) {
     try {
@@ -173,7 +176,7 @@ const runJob = async (job, isDirWatch) => {
       // If a directory was watched:
       if (isDirWatch) {
         // Save the report.
-        return await writeDirReport(report);
+        await writeDirReport(report);
       }
       // Otherwise, i.e. if the network was watched:
       else {
@@ -181,25 +184,19 @@ const runJob = async (job, isDirWatch) => {
         const ack = await writeNetReport(report);
         if (ack.error) {
           console.log(JSON.stringify(ack, null, 2));
-          return false;
-        }
-        else {
-          return true;
         }
       }
     }
+    // If the job failed:
     catch(error) {
+      // Report this.
       console.log(`ERROR: ${error.message}\n${error.stack}`);
-      return {
-        error: `ERROR: ${error.message}\n${error.stack}`
-      };
     }
   }
+  // Otherwise, i.e. if the job has no ID:
   else {
+    // Report this.
     console.log('ERROR: Job has no id');
-    return {
-      error: 'ERROR: Job has no id'
-    };
   }
 };
 // Checks for a job, performs it, and submits a report, once or repeatedly.
@@ -225,25 +222,24 @@ exports.cycle = async (isDirWatch, isForever, interval) => {
     if (job.id) {
       // Run it and save a report.
       console.log(`Running job ${job.id}`);
-      statusOK = await runJob(job, isDirWatch);
+      await runJob(job, isDirWatch);
       console.log(`Job ${job.id} finished`);
-      if (statusOK) {
-        // If a directory was watched:
-        if (isDirWatch) {
-          // Archive the job.
-          await archiveJob(job);
-          console.log(`Job ${job.id} archived`);
-        }
-        // If watching was specified for only 1 job, stop.
-        statusOK = isForever;
-        // Prevent a wait before the next iteration.
-        empty = false;
+      // If a directory was watched:
+      if (isDirWatch) {
+        // Archive the job.
+        await archiveJob(job);
+        console.log(`Job ${job.id} archived`);
       }
+      // If watching was specified for only 1 job, quit.
+      statusOK = isForever;
+      // Prevent a wait before the next iteration.
+      empty = false;
     }
+    // Otherwise, i.e. if no job was found:
     else {
+      // Cause a wait before the next check.
       empty = true;
     }
   }
   console.log('Watching ended');
-  return '';
 };