testaro 5.5.3 → 5.5.6

package/create.js

@@ -15,13 +15,23 @@ const {handleRequest} = require('./run');
 const {batchify} = require('./batchify');
 
 // ########## CONSTANTS
+
 const scriptDir = process.env.SCRIPTDIR;
 const batchDir = process.env.BATCHDIR;
 const reportDir = process.env.REPORTDIR;
+const successHosts = [];
+const crashHosts = [];
+const timeoutHosts = [];
+
+// ########## VARIABLES
+
+let healthy = true;
+let timeLimit = 300;
+let reportCount = 0;
 
 // ########## FUNCTIONS
 
-// Runs one script and writes a report file.
+// Runs one script with no batch and writes a report file.
 const runHost = async (id, script) => {
   const report = {
     id,
@@ -34,10 +44,69 @@ const runHost = async (id, script) => {
   const reportJSON = JSON.stringify(report, null, 2);
   await fs.writeFile(`${reportDir}/${id}.json`, reportJSON);
 };
+// Recursively runs host scripts.
+const runHosts = async (timeStamp, specs) => {
+  // If any scripts remain to be run and the process has not been interrupted:
+  if (specs.length && healthy) {
+    // Run the first one and save the report with a host-suffixed ID.
+    const spec = specs.shift();
+    const {id, host, script} = spec;
+    // Fork a child process to run the script for that host.
+    const subprocess = fork(
+      'runHost', [id, JSON.stringify(script), JSON.stringify(host)],
+      {
+        detached: true,
+        stdio: [0, 1, 'ignore', 'ipc']
+      }
+    );
+    // If the child process times out:
+    const timer = setTimeout(async () => {
+      clearTimeout(timer);
+      // Record the host as timed out.
+      timeoutHosts.push(id);
+      // Kill the child process.
+      subprocess.kill();
+      console.log(`Script for host ${id} exceeded ${timeLimit}-second time limit, so was killed`);
+      // Run the remaining host scripts.
+      await runHosts(timeStamp, specs);
+    }, 1000 * (script.timeLimit || timeLimit));
+    // If the child process succeeds:
+    subprocess.on('message', async message => {
+      clearTimeout(timer);
+      // Save its report as a file.
+      await fs.writeFile(`${reportDir}/${id}.json`, message);
+      console.log(`Report ${id}.json saved in ${reportDir}`);
+      reportCount++;
+      successHosts.push(id);
+      // Run the remaining host scripts.
+      await runHosts(timeStamp, specs);
+    });
+    // If the child process ends:
+    subprocess.on('exit', async () => {
+      // If it ended in a crash:
+      if (! (successHosts.includes(id) || timeoutHosts.includes(id))) {
+        clearTimeout(timer);
+        crashHosts.push(id);
+        console.log(`Script for host ${id} crashed`);
+        // Run the remaining host scripts.
+        await runHosts(timeStamp, specs);
+      }
+    });
+  }
+  // Otherwise, i.e. if no more host scripts are to be run:
+  else {
+    // Report the metadata.
+    console.log(`Count of ${timeStamp}- reports saved in ${reportDir}: ${reportCount}`);
+    if (timeoutHosts.length) {
+      console.log(`Hosts timed out:\n${JSON.stringify(timeoutHosts, null, 2)}`);
+    }
+    if (crashHosts.length) {
+      console.log(`Hosts crashed:\n${JSON.stringify(crashHosts, null, 2)}`);
+    }
+  }
+};
 // Runs a file-based job and writes a report file for the script or each host.
 exports.runJob = async (scriptID, batchID) => {
-  let healthy = true;
-  let childAlive = true;
   process.on('SIGINT', () => {
     console.log('ERROR: Terminal interrupted runJob');
     healthy = false;
@@ -47,8 +116,7 @@ exports.runJob = async (scriptID, batchID) => {
       const scriptJSON = await fs.readFile(`${scriptDir}/${scriptID}.json`, 'utf8');
       const script = JSON.parse(scriptJSON);
       // Get the time limit of the script or, if none, set it to 5 minutes.
-      let {timeLimit} = script;
-      timeLimit = timeLimit || 300;
+      timeLimit = script.timeLimit || timeLimit;
       // Identify the start time and a timestamp.
       const timeStamp = Math.floor((Date.now() - Date.UTC(2022, 1)) / 2000).toString(36);
       // If there is a batch:
@@ -58,79 +126,8 @@ exports.runJob = async (scriptID, batchID) => {
         const batchJSON = await fs.readFile(`${batchDir}/${batchID}.json`, 'utf8');
         batch = JSON.parse(batchJSON);
         const specs = batchify(script, batch, timeStamp);
-        const batchSize = specs.length;
-        const sizedRep = `${batchSize} report${batchSize > 1 ? 's' : ''}`;
-        const timeoutHosts = [];
-        const crashHosts = [];
-        // FUNCTION DEFINITION START
-        // Recursively runs host scripts.
-        const runHosts = specs => {
-          // If any scripts remain to be run and the process has not been interrupted:
-          if (specs.length && healthy) {
-            childAlive = true;
-            // Run the first one and save the report with a host-suffixed ID.
-            const spec = specs.shift();
-            const {id, host, script} = spec;
-            const subprocess = fork(
-              'runHost', [id, JSON.stringify(script), JSON.stringify(host)],
-              {
-                detached: true,
-                stdio: [0, 1, 'ipc']
-              }
-            );
-            subprocess.on('exit', () => {
-              childAlive = false;
-            });
-            const startTime = Date.now();
-            // At 5-second intervals:
-            const reCheck = setInterval(async () => {
-              // If the user has not interrupted the process:
-              if (healthy) {
-                // If there is no need to keep checking:
-                const reportNames = await fs.readdir(reportDir);
-                const timedOut = Date.now() - startTime > 1000 * timeLimit;
-                const reportWritten = reportNames.includes(`${id}.json`);
-                if (timedOut || reportWritten || ! childAlive) {
-                  // Stop checking.
-                  clearInterval(reCheck);
-                  // If the cause is a timeout:
-                  if (timedOut) {
-                    // Add the host to the array of timed-out hosts.
-                    timeoutHosts.push(id);
-                  }
-                  // Otherwise, if the cause is a child crash:
-                  else if (! (childAlive || reportWritten)) {
-                    // Add the host to the array of crashed hosts.
-                    crashHosts.push(id);
-                  }
-                  // Run the script of the next host.
-                  runHosts(specs);
-                }
-              }
-              // Otherwise, i.e. if the user has interrupted the process:
-              else {
-                // Tell the script run to quit.
-                subprocess.send('interrupt');
-                // Stop checking.
-                clearInterval(reCheck);
-              }
-            }, 5000);
-          }
-          else {
-            console.log(`${sizedRep} ${timeStamp}-....json in ${process.env.REPORTDIR}`);
-            if (timeoutHosts.length) {
-              console.log(`Reports not created:\n${JSON.stringify(timeoutHosts), null, 2}`);
-            }
-            if (crashHosts.length) {
-              console.log(
-                `Hosts crashed with or without report:\n${JSON.stringify(crashHosts, null, 2)}`
-              );
-            }
-          }
-        };
-        // FUNCTION DEFINITION END
         // Recursively run each host script and save the reports.
-        runHosts(specs);
+        runHosts(timeStamp, specs);
       }
       // Otherwise, i.e. if there is no batch:
       else {
@@ -140,7 +137,7 @@ exports.runJob = async (scriptID, batchID) => {
       }
     }
     catch(error) {
-      console.log(`ERROR: ${error.message}\n${error.stack}`);
+      console.log(`ERROR running job (${error.message})\n${error.stack}`);
     }
   }
   else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "5.5.3",
+  "version": "5.5.6",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -98,8 +98,6 @@ let actCount = 0;
 let browserContext;
 let browserTypeName;
 let requestedURL = '';
-// All browsers launched.
-let browsers = [];
 
 // ########## VALIDATORS
 
@@ -248,24 +246,11 @@ const isValidReport = async report => {
 
 // ########## OTHER FUNCTIONS
 
-// Closes all existing browsers.
-const closeBrowsers = async () => {
-  for (const browserObj of browsers) {
-    const {browser, typeName} = browserObj;
-    if (browser) {
-      console.log(`Closing ${typeName} browser, version ${browser.version()}`);
-      await browser.close();
-    }
-  }
-  browsers = [];
-};
 // Launches a browser.
 const launch = async typeName => {
   const browserType = require('playwright')[typeName];
   // If the specified browser type exists:
   if (browserType) {
-    // Close any existing browser.
-    await closeBrowsers();
     // Launch a browser of the specified type.
     const browserOptions = {};
     if (debug) {
@@ -274,55 +259,70 @@ const launch = async typeName => {
     if (waits) {
       browserOptions.slowMo = waits;
     }
-    const browser = await browserType.launch(browserOptions);
-    // Register it.
-    browsers.push({
-      browser,
-      typeName
+    let healthy = true;
+    const browser = await browserType.launch(browserOptions)
+    .catch(error => {
+      healthy = false;
+      console.log(`ERROR launching browser: ${error.message.replace(/\n.+/s, '')}`);
     });
-    // Create a new context (window) in it, taller if debugging is on.
-    const viewport = debug ? {
-      viewPort: {
-        width: 1280,
-        height: 1120
-      }
-    } : {};
-    browserContext = await browser.newContext(viewport);
-    // When a page is added to the browser context:
-    browserContext.on('page', page => {
-      // Make abbreviations of its console messages get reported in the Playwright console.
-      page.on('console', msg => {
-        let msgText = msg.text();
-        if (msgText.length > 300) {
-          msgText = `${msgText.slice(0, 150)} ... ${msgText.slice(-150)}`;
+    // If the launch succeeded:
+    if (healthy) {
+      // Create a new context (window) in it, taller if debugging is on.
+      const viewport = debug ? {
+        viewPort: {
+          width: 1280,
+          height: 1120
         }
-        console.log(`[${msgText}]`);
-        const msgTextLC = msgText.toLowerCase();
-        const msgLength = msgText.length;
-        logCount++;
-        logSize += msgLength;
-        if (errorWords.some(word => msgTextLC.includes(word))) {
-          errorLogCount++;
-          errorLogSize += msgLength;
-        }
-        const msgLC = msgText.toLowerCase();
-        if (msgText.includes('403') && (msgLC.includes('status') || msgLC.includes('prohibited'))) {
-          prohibitedCount++;
-        }
-      });
-    });
-    // Open the first page of the context.
-    const page = await browserContext.newPage();
-    if (debug) {
-      page.setViewportSize({
-        width: 1280,
-        height: 1120
+      } : {};
+      browserContext = await browser.newContext(viewport);
+      // When a page is added to the browser context:
+      browserContext.on('page', page => {
+        // Make abbreviations of its console messages get reported in the Playwright console.
+        page.on('console', msg => {
+          const msgText = msg.text();
+          const parts = [msgText.slice(0, 75)];
+          if (msgText.length > 75) {
+            parts.push(msgText.slice(75, 150));
+            if (msgText.length > 150) {
+              const tail = msgText.slice(150).slice(-150);
+              if (msgText.length > 300) {
+                parts.push('...');
+              }
+              parts.push(tail.slice(0, 75));
+              if (tail.length > 75) {
+                parts.push(tail.slice(75));
+              }
+            }
+          }
+          const indentedMsg = parts.map(part => `    | ${part}`).join('\n');
+          console.log(`\n${indentedMsg}`);
+          const msgTextLC = msgText.toLowerCase();
+          const msgLength = msgText.length;
+          logCount++;
+          logSize += msgLength;
+          if (errorWords.some(word => msgTextLC.includes(word))) {
+            errorLogCount++;
+            errorLogSize += msgLength;
+          }
+          const msgLC = msgText.toLowerCase();
+          if (msgText.includes('403') && (msgLC.includes('status') || msgLC.includes('prohibited'))) {
+            prohibitedCount++;
+          }
+        });
       });
+      // Open the first page of the context.
+      const page = await browserContext.newPage();
+      if (debug) {
+        page.setViewportSize({
+          width: 1280,
+          height: 1120
+        });
+      }
+      // Wait until it is stable.
+      await page.waitForLoadState('domcontentloaded');
+      // Update the name of the current browser type and store it in the page.
+      page.browserTypeName = browserTypeName = typeName;
     }
-    // Wait until it is stable.
-    await page.waitForLoadState('domcontentloaded');
-    // Update the name of the current browser type and store it in the page.
-    page.browserTypeName = browserTypeName = typeName;
   }
 };
 // Normalizes spacing characters and cases in a string.
@@ -1295,8 +1295,6 @@ const doScript = async (report) => {
   report.testTimes = [];
   // Perform the specified acts.
   await doActs(report, 0, null);
-  // Close all browsers.
-  await closeBrowsers();
   // Add the log statistics to the report.
   report.logCount = logCount;
   report.logSize = logSize;

package/runHost.js

@@ -5,18 +5,11 @@
 
 // ########## IMPORTS
 
-// Module to keep secrets.
-require('dotenv').config();
-// Module to read and write files.
-const fs = require('fs/promises');
 const {handleRequest} = require('./run');
 
-// ########## CONSTANTS
-const reportDir = process.env.REPORTDIR;
-
 // ########## FUNCTIONS
 
-// Runs one script and writes a report file.
+// Runs one script and sends the report to the parent.
 const runHost = async (id, scriptJSON, hostJSON) => {
   const report = {
     id,
@@ -27,7 +20,7 @@ const runHost = async (id, scriptJSON, hostJSON) => {
   };
   await handleRequest(report);
   const reportJSON = JSON.stringify(report, null, 2);
-  await fs.writeFile(`${reportDir}/${id}.json`, reportJSON);
+  process.send(reportJSON);
   process.disconnect();
   process.exit();
 };

package/samples/scripts/tp12.json

@@ -2,6 +2,7 @@
   "id": "tp12",
   "what": "Alfa, Axe, HTML CodeSniffer, IBM, Tenon, WAVE, and 16 custom tests",
   "strict": true,
+  "timeLimit": 300,
   "commands": [
     {
       "type": "launch",
@@ -146,17 +147,17 @@
       "withItems": true,
       "what": "IBM Accessibility Checker, with page content and again with URL"
     },
-    {
-      "type": "test",
-      "which": "tenon",
-      "id": "a",
-      "what": "Tenon API version 2 result retrieval"
-    },
     {
       "type": "test",
       "which": "wave",
       "reportType": 4,
       "what": "WAVE, report-type 4"
+    },
+    {
+      "type": "test",
+      "which": "tenon",
+      "id": "a",
+      "what": "Tenon API version 2 result retrieval"
     }
   ]
 }

package/tests/ibm.js

@@ -19,23 +19,37 @@
 */
 // Import required modules.
 const fs = require('fs').promises;
-const {getCompliance, close} = require('accessibility-checker');
+// Scanner. Importing and executing 'close' crashed the Node process.
+const {getCompliance} = require('accessibility-checker');
 // Runs the IBM test.
-const run = async content => {
+const run = async (content, timeLimit) => {
   const nowLabel = (new Date()).toISOString().slice(0, 19);
-  // Return the result of a test.
-  const ibmReport = await getCompliance(content, nowLabel);
-  return ibmReport;
+  // Start the timeout clock.
+  let timeoutID;
+  const timeout = new Promise(resolve => {
+    timeoutID = setTimeout(() => {
+      resolve(null);
+    }, 1000 * timeLimit);
+  });
+  // Return the result of the test, or null if it timed out.
+  const ibmReport = getCompliance(content, nowLabel)
+  .catch(error => {
+    console.log(`ERROR: getCompliance failed (${error.message.replace(/\n+/s, '')}).`);
+    return null;
+  });
+  const result = await Promise.race([ibmReport, timeout]);
+  clearTimeout(timeoutID);
+  return result;
 };
 // Trims an IBM report.
-const report = (ibmReport, withItems) => {
+const trimReport = (report, withItems) => {
   const data = {};
-  if (ibmReport && ibmReport.report && ibmReport.report.summary) {
-    const totals = ibmReport.report.summary.counts;
+  if (report && report.report && report.report.summary) {
+    const totals = report.report.summary.counts;
     if (totals) {
       data.totals = totals;
       if (withItems) {
-        data.items = ibmReport.report.results;
+        data.items = report.report.results;
         data.items.forEach(item => {
           delete item.apiArgs;
           delete item.category;
@@ -58,34 +72,25 @@ const report = (ibmReport, withItems) => {
   }
   return data;
 };
-// Performs an IBM test.
+// Performs an IBM test and return the result.
 const doTest = async (content, withItems, timeLimit) => {
-  // Start a timeout clock.
-  let timeoutID;
-  const wait = new Promise(resolve => {
-    timeoutID = setTimeout(() => {
-      resolve({});
-    }, 1000 * timeLimit);
-  });
-  // Conduct the test and get a Promise of the report.
-  const ibmReport = run(content);
-  // Wait for completion or until the time limit expires.
-  const ibmReportIfFast = await Promise.race([ibmReport, wait]);
-  // Delete existing report files.
-  try {
-    const reportNames = await fs.readdir('results');
-    for (const reportName of reportNames) {
-      await fs.rm(`results/${reportName}`);
+  // Conduct the test and get the result.
+  const report = await run(content, timeLimit);
+  // If the test did not time out:
+  if (report) {
+    // Delete any report files.
+    try {
+      const reportNames = await fs.readdir('results');
+      for (const reportName of reportNames) {
+        await fs.rm(`results/${reportName}`);
+      }
     }
-  }
-  catch(error) {
-    console.log('ibm test created no result files.');
-  }
-  // Return the result.
-  if (ibmReportIfFast.report) {
-    clearTimeout(timeoutID);
-    const ibmTypeReport = report(ibmReportIfFast, withItems);
-    return ibmTypeReport;
+    catch(error) {
+      console.log('ibm test created no result files.');
+    }
+    // Return the result.
+    const typeReport = trimReport(report, withItems);
+    return typeReport;
   }
   else {
     return {
@@ -117,6 +122,6 @@ exports.reporter = async (page, withItems, withNewContent) => {
       console.log(`ERROR: Getting ibm test report from URL timed out at ${timeLimit} seconds`);
     }
   }
-  await close();
+  // Return the result. Execution of close() crashed the Node process.
   return {result};
 };