testaro 5.2.1 → 5.4.0

package/.eslintrc.json

@@ -0,0 +1,41 @@
+{
+  "env": {
+    "browser": true,
+    "commonjs": true,
+    "es2021": true,
+    "node": true
+  },
+  "extends": "eslint:recommended",
+  "parserOptions": {
+    "ecmaVersion": 2021
+  },
+  "rules": {
+    "indent": [
+      "error",
+      2,
+      {
+        "MemberExpression": 0,
+        "ObjectExpression": "first"
+      }
+    ],
+    "linebreak-style": [
+      "error",
+      "unix"
+    ],
+    "quotes": [
+      "error",
+      "single"
+    ],
+    "semi": [
+      "error",
+      "always"
+    ],
+    "no-use-before-define": [
+      "error"
+    ],
+    "brace-style": [
+      "error",
+      "stroustrup"
+    ]
+  }
+}

package/README.md

@@ -89,10 +89,13 @@ A script is a JSON file with the properties:
 {
   "what": "string: description of the script",
   "strict": "boolean: whether redirections should be treated as failures",
+  "timeLimit": "number: limit in seconds on the execution of this script",
   "commands": "array of objects: the commands to be performed"
 }
 ```
 
+The `timeLimit` property is optional. If it is omitted, a default of 300 seconds (5 minutes) is set.
+
 ### Example
 
 Here is an example of a script:
@@ -101,6 +104,7 @@ Here is an example of a script:
 {
   what: 'Test example.com with alfa',
   strict: true,
+  timeLimit: 15,
   commands: [
     {
       type: 'launch',
@@ -292,6 +296,38 @@ In case you want to perform more than one `tenon` test with the same script, you
 
 Tenon recommends giving it a public URL rather than giving it the content of a page, if possible. So, it is best to give the `withNewContent` property of the `tenonRequest` command the value `true`, unless the page is not public.
 
+###### HTML CodeSniffer
+
+The `htmlcs` test makes use of the`htmlcs/HTMLCS.js` file. That file was created, and can be recreated if necessary, as follows:
+
+1. Clone the (HTML CodeSniffer package)[https://github.com/squizlabs/HTML_CodeSniffer].
+1. Make that package’s directory the active directory.
+1. Install the HTML CodeSniffer dependencies by executing `npm install`.
+1. Build the HTML CodeSniffer auditor by executing `grunt build`.
+1. Copy the `build/HTMLCS.js` and `build/licence.txt` files into the `htmlcs` directory of Testaro.
+1. Edit the Testaro copy of `htmlcs/HTMLCS.js` to produce the changes shown below.
+
+The changes in `htmlcs/HTMLCS.js` are:
+
+```diff
+479a480
+>     '4_1_2_attribute': 'attribute',
+6482a6484
+>     var messageStrings = new Set();
+6496d6497
+<         console.log('done');
+6499d6499
+<         console.log('done');
+6500a6501
+>       return Array.from(messageStrings);
+6531c6532,6534
+<       console.log('[HTMLCS] ' + typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html);
+---
+>       messageStrings.add(
+>         typeName + '|' + msg.code + '|' + nodeName + '|' + elementId + '|' + msg.msg + '|' + html
+>       );
+```
+
 ##### Branching
 
 An example of a **branching** command is:
@@ -454,13 +490,13 @@ Relative paths must be relative to the Testaro project directory. For example, i
 
 Also ensure that Testaro can read all those directories and write to `REPORTDIR`.
 
-Place a script into `SCRIPTDIR` and, optionally, a batch into `BATCHDIR`. Each should be named `idValue.json`, where `idValue` is replaced with the value of its `id` property. That value must consist of only lower-case ASCII letters and digits.
+Place a script into `SCRIPTDIR` and, optionally, a batch into `BATCHDIR`. Each should be named `idvalue.json`, where `idvalue` is replaced with the value of its `id` property. That value must consist of only lower-case ASCII letters and digits.
 
 Then execute the statement `node high scriptID` or `node high scriptID batchID`, replacing `scriptID` and `batchID` with the `id` values of the script and the batch, respectively.
 
 The `high` module will call the `runJob` function of the `create` module, which in turn will call the `handleRequest` function of the `run` module. The results will be saved in report files in the `REPORTDIR` directory.
 
-If there is no batch, the report file will be named with a unique timestamp, suffixed with a `.json` extension. If there is a batch, then the base of each report file’s name will be the same timestamp, suffixed with `-hostID`, where `hostID` is the value of the `id` property of the `host` object in the batch file. For example, if you execute `node create script01 wikis`, you might get these report files deposited into `REPORTDIR`:
+If there is no batch, the report file will be named with a unique timestamp, suffixed with a `.json` extension. If there is a batch, then the base of each report file’s name will be the same timestamp, suffixed with `-hostid`, where `hostid` is the value of the `id` property of the `host` object in the batch file. For example, if you execute `node create script01 wikis`, you might get these report files deposited into `REPORTDIR`:
 - `enp46j-wikipedia.json`
 - `enp45j-wiktionary.json`
 - `enp45j-wikidata.json`
@@ -570,6 +606,16 @@ The rationales motivating the Testaro-defined tests can be found in comments wit
 
 ## Testing challenges
 
+### Abnormal termination
+
+On rare occasions a test throws an error that terminates the Node process and cannot be handled with a `try`-`catch` structure. It has been observed, for example, that the `ibm` test does this when run on the host at `https://zenythgroup.com/index` or `https://monsido.com`.
+
+If a single process performed all of the commands in a batch-based script, the process could perform tens of thousands of commands, and such an error could stop the process at any point.
+
+To handle this risk, Testaro processes batch-based jobs by forking a new process for each host. If such an error occurs, it crashes the child process, preventing a report for that host from being written. The parent process waits for the report to appear in the `REPORTDIR` directory until the time limit. When it fails to appear, the parent process continues to the next host.
+
+If you are using high-level invocation, your terminal will show the standard output of the parent process and, if there is a batch, the current child process, too. If you interrupt the process with `CTRL-c`, you will send a `SIGINT` signal to the parent process, which will handle it by sending a message to the child process telling it to terminate itself, and then the parent process will terminate by skipping the remaining hosts.
+
 ### Activation
 
 Testing to determine what happens when a control or link is activated is straightforward, except in the context of a comprehensive set of tests of a single page. There, activating a control or link can change the page or navigate away from it, interfering with the remaining planned tests of the page.
@@ -611,6 +657,10 @@ Testaro omits some functionalities of Autotest, such as:
 - file operations for score aggregation, report revision, and HTML reports
 - a web user interface
 
+## Code style
+
+The JavaScript code in this project generally conforms to the ESLint configuration file `.eslintrc`. However, the `htmlcs/HTMLCS.js` file implements an older version of JavaScript. Its style is regulated by the `htmlcs/.eslintrc.json` file.
+
 ## Origin
 
 Work on the custom tests in this package began in 2017, and work on the multi-package federation that Testaro implements began in early 2018. These two aspects were combined into the [Autotest](https://github.com/jrpool/autotest) package in early 2021 and into the more single-purpose packages, Testaro and Testilo, in January 2022.

package/create.js

@@ -9,6 +9,7 @@
 require('dotenv').config();
 // Module to read and write files.
 const fs = require('fs/promises');
+const {fork} = require('child_process');
 const {handleRequest} = require('./run');
 // Module to convert a script and a batch to a batch-based array of scripts.
 const {batchify} = require('./batchify');
@@ -21,10 +22,10 @@ const reportDir = process.env.REPORTDIR;
 // ########## FUNCTIONS
 
 // Runs one script and writes a report file.
-const runHost = async (id, script, host = {}) => {
+const runHost = async (id, script) => {
   const report = {
     id,
-    host,
+    host: {},
     log: [],
     script,
     acts: []
@@ -35,10 +36,19 @@ const runHost = async (id, script, host = {}) => {
 };
 // Runs a file-based job and writes a report file for the script or each host.
 exports.runJob = async (scriptID, batchID) => {
+  let healthy = true;
+  let childAlive = true;
+  process.on('SIGINT', () => {
+    console.log('ERROR: Terminal interrupted runJob');
+    healthy = false;
+  });
   if (scriptID) {
     try {
       const scriptJSON = await fs.readFile(`${scriptDir}/${scriptID}.json`, 'utf8');
       const script = JSON.parse(scriptJSON);
+      // Get the time limit of the script or, if none, set it to 5 minutes.
+      let {timeLimit} = script;
+      timeLimit = timeLimit || 300;
       // Identify the start time and a timestamp.
       const timeStamp = Math.floor((Date.now() - Date.UTC(2022, 1)) / 2000).toString(36);
       // If there is a batch:
@@ -48,28 +58,89 @@ exports.runJob = async (scriptID, batchID) => {
         const batchJSON = await fs.readFile(`${batchDir}/${batchID}.json`, 'utf8');
         batch = JSON.parse(batchJSON);
         const specs = batchify(script, batch, timeStamp);
-        // For each host script:
-        while (specs.length) {
-          const spec = specs.shift();
-          const {id, host, script} = spec;
-          // Run it and save the result with a host-suffixed ID.
-          await runHost(id, script, host);
-        }
+        const batchSize = specs.length;
+        const sizedRep = `${batchSize} report${batchSize > 1 ? 's' : ''}`;
+        const timeoutHosts = [];
+        const crashHosts = [];
+        // FUNCTION DEFINITION START
+        // Recursively runs host scripts.
+        const runHosts = specs => {
+          // If any scripts remain to be run and the process has not been interrupted:
+          if (specs.length && healthy) {
+            childAlive = true;
+            // Run the first one and save the report with a host-suffixed ID.
+            const spec = specs.shift();
+            const {id, host, script} = spec;
+            const subprocess = fork(
+              'runHost', [id, JSON.stringify(script), JSON.stringify(host)],
+              {
+                detached: true,
+                stdio: [0, 1, 'ipc']
+              }
+            );
+            subprocess.on('exit', () => {
+              childAlive = false;
+            });
+            const startTime = Date.now();
+            // At 5-second intervals:
+            const reCheck = setInterval(async () => {
+              // If the user has not interrupted the process:
+              if (healthy) {
+                // If there is no need to keep checking:
+                const reportNames = await fs.readdir(reportDir);
+                const timedOut = Date.now() - startTime > 1000 * timeLimit;
+                if (timedOut || reportNames.includes(`${id}.json`) || ! childAlive) {
+                  // Stop checking.
+                  clearInterval(reCheck);
+                  // If the cause is a timeout:
+                  if (timedOut) {
+                    // Add the host to the array of timed-out hosts.
+                    timeoutHosts.push(id);
+                  }
+                  // Otherwise, if the cause is a child crash:
+                  else if (! childAlive) {
+                    // Add the host to the array of crashed hosts.
+                    crashHosts.push(id);
+                  }
+                  // Run the script of the next host.
+                  runHosts(specs);
+                }
+              }
+              // Otherwise, i.e. if the user has interrupted the process:
+              else {
+                // Tell the script run to quit.
+                subprocess.send('interrupt');
+                // Stop checking.
+                clearInterval(reCheck);
+              }
+            }, 5000);
+          }
+          else {
+            console.log(`${sizedRep} ${timeStamp}-....json in ${process.env.REPORTDIR}`);
+            if (timeoutHosts.length) {
+              console.log(`Reports not created:\n${JSON.stringify(timeoutHosts), null, 2}`);
+            }
+            if (crashHosts.length) {
+              console.log(`Hosts crashed:\n${JSON.stringify(crashHosts), null, 2}`);
+            }
+          }
+        };
+        // FUNCTION DEFINITION END
+        // Recursively run each host script and save the reports.
+        runHosts(specs);
       }
       // Otherwise, i.e. if there is no batch:
       else {
         // Run the script and save the result with a timestamp ID.
         await runHost(timeStamp, script);
+        console.log(`Report ${timeStamp}.json in ${process.env.REPORTDIR}`);
       }
-      return timeStamp;
     }
     catch(error) {
       console.log(`ERROR: ${error.message}\n${error.stack}`);
-      return null;
     }
   }
   else {
     console.log('ERROR: no script specified');
-    return null;
   }
 };

package/high.js

@@ -1,14 +1,10 @@
 /*
   high.js
   Invokes Testaro with the high-level method.
-  Usage example: node high tp11 weborgs
+  Usage example: node high tp12 weborgs
 */
 
 const {runJob} = require('./create');
 const scriptID = process.argv[2];
 const batchID = process.argv[3];
-const run = async (scriptID, batchID) => {
-  const timeStamp = await runJob(scriptID, batchID);
-  console.log(`Reports in ${process.env.REPORTDIR}; ID base ${timeStamp}`);
-};
-run(scriptID, batchID);
+runJob(scriptID, batchID);

package/htmlcs/.eslintrc.json

@@ -0,0 +1,67 @@
+{
+  "root": true,
+  "env": {
+    "browser": true,
+    "commonjs": true,
+    "es2021": true,
+    "node": true
+  },
+  "extends": "eslint:recommended",
+  "parserOptions": {
+    "ecmaVersion": 5
+  },
+  "rules": {
+    "indent": [
+      "error",
+      2,
+      {
+        "MemberExpression": 0,
+        "ObjectExpression": "first"
+      }
+    ],
+    "linebreak-style": [
+      "error",
+      "unix"
+    ],
+    "quotes": [
+      "error",
+      "single"
+    ],
+    "semi": [
+      "error",
+      "always"
+    ],
+    "brace-style": [
+      "error",
+      "stroustrup"
+    ],
+    "no-undef": "off",
+    "no-redeclare": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off",
+    "no-prototype-builtins": "off",
+    "no-cond-assign": "off",
+    "no-fallthrough": "off",
+    "no-constant-condition": "off",
+    "block-spacing": [
+      "error",
+      "never"
+    ],
+    "array-bracket-spacing": [
+      "error",
+      "never"
+    ],
+    "space-before-function-paren": [
+      "error",
+      "always"
+    ],
+    "space-in-parens": [
+      "error",
+      "never"
+    ],
+    "object-curly-spacing": [
+      "error",
+      "never"
+    ]
+  }
+}