testaro 5.2.2 → 5.3.0

package/README.md

@@ -89,10 +89,13 @@ A script is a JSON file with the properties:
 {
   "what": "string: description of the script",
   "strict": "boolean: whether redirections should be treated as failures",
+  "timeLimit": "number: limit in seconds on the execution of this script",
   "commands": "array of objects: the commands to be performed"
 }
 ```
 
+The `timeLimit` property is optional. If it is omitted, a default of 300 seconds (5 minutes) is set.
+
 ### Example
 
 Here is an example of a script:
@@ -101,6 +104,7 @@ Here is an example of a script:
 {
   what: 'Test example.com with alfa',
   strict: true,
+  timeLimit: 15,
   commands: [
     {
       type: 'launch',
@@ -486,13 +490,13 @@ Relative paths must be relative to the Testaro project directory. For example, i
 
 Also ensure that Testaro can read all those directories and write to `REPORTDIR`.
 
-Place a script into `SCRIPTDIR` and, optionally, a batch into `BATCHDIR`. Each should be named `idValue.json`, where `idValue` is replaced with the value of its `id` property. That value must consist of only lower-case ASCII letters and digits.
+Place a script into `SCRIPTDIR` and, optionally, a batch into `BATCHDIR`. Each should be named `idvalue.json`, where `idvalue` is replaced with the value of its `id` property. That value must consist of only lower-case ASCII letters and digits.
 
 Then execute the statement `node high scriptID` or `node high scriptID batchID`, replacing `scriptID` and `batchID` with the `id` values of the script and the batch, respectively.
 
 The `high` module will call the `runJob` function of the `create` module, which in turn will call the `handleRequest` function of the `run` module. The results will be saved in report files in the `REPORTDIR` directory.
 
-If there is no batch, the report file will be named with a unique timestamp, suffixed with a `.json` extension. If there is a batch, then the base of each report file’s name will be the same timestamp, suffixed with `-hostID`, where `hostID` is the value of the `id` property of the `host` object in the batch file. For example, if you execute `node create script01 wikis`, you might get these report files deposited into `REPORTDIR`:
+If there is no batch, the report file will be named with a unique timestamp, suffixed with a `.json` extension. If there is a batch, then the base of each report file’s name will be the same timestamp, suffixed with `-hostid`, where `hostid` is the value of the `id` property of the `host` object in the batch file. For example, if you execute `node create script01 wikis`, you might get these report files deposited into `REPORTDIR`:
 - `enp46j-wikipedia.json`
 - `enp45j-wiktionary.json`
 - `enp45j-wikidata.json`
@@ -602,6 +606,16 @@ The rationales motivating the Testaro-defined tests can be found in comments wit
 
 ## Testing challenges
 
+### Abnormal termination
+
+On rare occasions a test throws an error that terminates the Node process and cannot be handled with a `try`-`catch` structure. It has been observed, for example, that the `ibm` test does this when run on the host at `https://zenythgroup.com/index` or `https://monsido.com`.
+
+If a single process performed all of the commands in a batch-based script, the process could perform tens of thousands of commands, and such an error could stop the process at any point.
+
+To handle this risk, Testaro processes batch-based jobs by forking a new process for each host. If such an error occurs, it crashes the child process, preventing a report for that host from being written. The parent process waits for the report to appear in the `REPORTDIR` directory until the time limit. When it fails to appear, the parent process continues to the next host.
+
+If you are using high-level invocation, your terminal will show the standard output of the parent process and, if there is a batch, the current child process, too. If you interrupt the process with `CTRL-c`, you will send a `SIGINT` signal to the parent process, which will handle it by sending a message to the child process telling it to terminate itself, and then the parent process will terminate by skipping the remaining hosts.
+
 ### Activation
 
 Testing to determine what happens when a control or link is activated is straightforward, except in the context of a comprehensive set of tests of a single page. There, activating a control or link can change the page or navigate away from it, interfering with the remaining planned tests of the page.

package/create.js

@@ -9,6 +9,7 @@
 require('dotenv').config();
 // Module to read and write files.
 const fs = require('fs/promises');
+const {fork} = require('child_process');
 const {handleRequest} = require('./run');
 // Module to convert a script and a batch to a batch-based array of scripts.
 const {batchify} = require('./batchify');
@@ -21,10 +22,10 @@ const reportDir = process.env.REPORTDIR;
 // ########## FUNCTIONS
 
 // Runs one script and writes a report file.
-const runHost = async (id, script, host = {}) => {
+const runHost = async (id, script) => {
   const report = {
     id,
-    host,
+    host: {},
     log: [],
     script,
     acts: []
@@ -35,10 +36,19 @@ const runHost = async (id, script, host = {}) => {
 };
 // Runs a file-based job and writes a report file for the script or each host.
 exports.runJob = async (scriptID, batchID) => {
+  let healthy = true;
+  let childAlive = true;
+  process.on('SIGINT', () => {
+    console.log('ERROR: Terminal interrupted runJob');
+    healthy = false;
+  });
   if (scriptID) {
     try {
       const scriptJSON = await fs.readFile(`${scriptDir}/${scriptID}.json`, 'utf8');
       const script = JSON.parse(scriptJSON);
+      // Get the time limit of the script or, if none, set it to 5 minutes.
+      let {timeLimit} = script;
+      timeLimit = timeLimit || 300;
       // Identify the start time and a timestamp.
       const timeStamp = Math.floor((Date.now() - Date.UTC(2022, 1)) / 2000).toString(36);
       // If there is a batch:
@@ -48,28 +58,89 @@ exports.runJob = async (scriptID, batchID) => {
         const batchJSON = await fs.readFile(`${batchDir}/${batchID}.json`, 'utf8');
         batch = JSON.parse(batchJSON);
         const specs = batchify(script, batch, timeStamp);
-        // For each host script:
-        while (specs.length) {
-          const spec = specs.shift();
-          const {id, host, script} = spec;
-          // Run it and save the result with a host-suffixed ID.
-          await runHost(id, script, host);
-        }
+        const batchSize = specs.length;
+        const sizedRep = `${batchSize} report${batchSize > 1 ? 's' : ''}`;
+        const timeoutHosts = [];
+        const crashHosts = [];
+        // FUNCTION DEFINITION START
+        // Recursively runs host scripts.
+        const runHosts = specs => {
+          // If any scripts remain to be run and the process has not been interrupted:
+          if (specs.length && healthy) {
+            childAlive = true;
+            // Run the first one and save the report with a host-suffixed ID.
+            const spec = specs.shift();
+            const {id, host, script} = spec;
+            const subprocess = fork(
+              'runHost', [id, JSON.stringify(script), JSON.stringify(host)],
+              {
+                detached: true,
+                stdio: [0, 1, 'ipc']
+              }
+            );
+            subprocess.on('exit', () => {
+              childAlive = false;
+            });
+            const startTime = Date.now();
+            // At 5-second intervals:
+            const reCheck = setInterval(async () => {
+              // If the user has not interrupted the process:
+              if (healthy) {
+                // If there is no need to keep checking:
+                const reportNames = await fs.readdir(reportDir);
+                const timedOut = Date.now() - startTime > 1000 * timeLimit;
+                if (timedOut || reportNames.includes(`${id}.json`) || ! childAlive) {
+                  // Stop checking.
+                  clearInterval(reCheck);
+                  // If the cause is a timeout:
+                  if (timedOut) {
+                    // Add the host to the array of timed-out hosts.
+                    timeoutHosts.push(id);
+                  }
+                  // Otherwise, if the cause is a child crash:
+                  else if (! childAlive) {
+                    // Add the host to the array of crashed hosts.
+                    crashHosts.push(id);
+                  }
+                  // Run the script of the next host.
+                  runHosts(specs);
+                }
+              }
+              // Otherwise, i.e. if the user has interrupted the process:
+              else {
+                // Tell the script run to quit.
+                subprocess.send('interrupt');
+                // Stop checking.
+                clearInterval(reCheck);
+              }
+            }, 5000);
+          }
+          else {
+            console.log(`${sizedRep} ${timeStamp}-....json in ${process.env.REPORTDIR}`);
+            if (timeoutHosts.length) {
+              console.log(`Reports not created:\n${JSON.stringify(timeoutHosts), null, 2}`);
+            }
+            if (crashHosts.length) {
+              console.log(`Hosts crashed:\n${JSON.stringify(crashHosts), null, 2}`);
+            }
+          }
+        };
+        // FUNCTION DEFINITION END
+        // Recursively run each host script and save the reports.
+        runHosts(specs);
       }
       // Otherwise, i.e. if there is no batch:
       else {
         // Run the script and save the result with a timestamp ID.
         await runHost(timeStamp, script);
+        console.log(`Report ${timeStamp}.json in ${process.env.REPORTDIR}`);
       }
-      return timeStamp;
     }
     catch(error) {
       console.log(`ERROR: ${error.message}\n${error.stack}`);
-      return null;
     }
   }
   else {
     console.log('ERROR: no script specified');
-    return null;
   }
 };

package/high.js

@@ -1,14 +1,10 @@
 /*
   high.js
   Invokes Testaro with the high-level method.
-  Usage example: node high tp11 weborgs
+  Usage example: node high tp12 weborgs
 */
 
 const {runJob} = require('./create');
 const scriptID = process.argv[2];
 const batchID = process.argv[3];
-const run = async (scriptID, batchID) => {
-  const timeStamp = await runJob(scriptID, batchID);
-  console.log(`Reports in ${process.env.REPORTDIR}; ID base ${timeStamp}`);
-};
-run(scriptID, batchID);
+runJob(scriptID, batchID);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "5.2.2",
+  "version": "5.3.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -633,6 +633,12 @@ const wait = ms => {
 };
 // Recursively performs the acts in a report.
 const doActs = async (report, actIndex, page) => {
+  process.on('message', message => {
+    if (message === 'interrupt') {
+      console.log('ERROR: Terminal interrupted doActs');
+      process.exit();
+    }
+  });
   const {acts} = report;
   // If any more commands are to be performed:
   if (actIndex > -1 && actIndex < acts.length) {
@@ -1310,8 +1316,7 @@ const doScript = async (report) => {
 const injectLaunches = acts => {
   let injectMore = true;
   while (injectMore) {
-    const injectIndex = acts.findIndex(
-      (act, index) =>
+    const injectIndex = acts.findIndex((act, index) =>
       index < acts.length - 1
       && act.type === 'test'
       && acts[index + 1].type === 'test'

package/runHost.js

@@ -0,0 +1,36 @@
+/*
+  runHost.js
+  Runs a host job and writes a report file.
+*/
+
+// ########## IMPORTS
+
+// Module to keep secrets.
+require('dotenv').config();
+// Module to read and write files.
+const fs = require('fs/promises');
+const {handleRequest} = require('./run');
+
+// ########## CONSTANTS
+const reportDir = process.env.REPORTDIR;
+
+// ########## FUNCTIONS
+
+// Runs one script and writes a report file.
+const runHost = async (id, scriptJSON, hostJSON) => {
+  const report = {
+    id,
+    host: JSON.parse(hostJSON),
+    log: [],
+    script: JSON.parse(scriptJSON),
+    acts: []
+  };
+  await handleRequest(report);
+  const reportJSON = JSON.stringify(report, null, 2);
+  await fs.writeFile(`${reportDir}/${id}.json`, reportJSON);
+  process.disconnect();
+  process.exit();
+};
+
+// ########## OPERATION
+runHost(... process.argv.slice(2));

package/tests/hover.js

@@ -158,7 +158,10 @@ const find = async (withItems, page, region, sample, popRatio) => {
             position: {
               x: 0,
               y: 0
-            }
+            },
+            timeout: 500,
+            force: true,
+            noWaitAfter: true
           });
           // Wait for any delayed and/or slowed hover reaction.
           await page.waitForTimeout(200);

package/tests/htmlcs.js

@@ -28,7 +28,7 @@ exports.reporter = async page => {
         }
         return issues;
       }, standard);
-      if (nextIssues) {
+      if (nextIssues && nextIssues.every(issue => typeof issue === 'string')) {
         messageStrings.push(... nextIssues);
       }
       else {

package/tests/ibm.js

@@ -104,7 +104,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
     result.content = await doTest(typeContent, withItems, timeLimit);
     if (result.content.prevented) {
       result.prevented = true;
-      console.log('ERROR: Getting ibm test report from page took too long');
+      console.log(`ERROR: Getting ibm test report from page timed out at ${timeLimit} seconds`);
     }
   }
   // If a test with new content is to be performed:
@@ -114,7 +114,7 @@ exports.reporter = async (page, withItems, withNewContent) => {
     result.url = await doTest(typeContent, withItems, timeLimit);
     if (result.url.prevented) {
       result.prevented = true;
-      console.log('ERROR: Getting ibm test report from URL took too long');
+      console.log(`ERROR: Getting ibm test report from URL timed out at ${timeLimit} seconds`);
     }
   }
   await close();

package/tests/tabNav.js

@@ -103,7 +103,9 @@ exports.reporter = async (page, withItems) => {
         });
       })
       .catch(error => {
-        console.log(`ERROR: could not click tab element ${itemData.text} (${error.message})`);
+        console.log(
+          `ERROR clicking tab element ${itemData.text} (${error.message.replace(/\n.+/s, '')})`
+        );
       pressed = false;
       });
       // Increment the counts of navigations and key navigations.