testaro 60.2.1 → 60.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "60.2.1",
+  "version": "60.4.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/doTestAct.js

@@ -50,9 +50,10 @@ const actIndex = Number.parseInt(process.argv[2]);
 
 // FUNCTIONS
 
+// Performs the tests of the act specified by the caller.
 const doTestAct = async () => {
   const reportPath = `${tmpDir}/report.json`;
-  // Get the saved report.
+  // Get the report from the temporary directory.
   const reportJSON = await fs.readFile(reportPath, 'utf8');
   const report = JSON.parse(reportJSON);
   // Get a reference to the act in the report.

package/procs/screenShot.js

@@ -0,0 +1,32 @@
+/*
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+  Licensed under the MIT License. See LICENSE file for details.
+*/
+
+/*
+  screenShot
+  This procedure creates and returns a full-page screenshot, optionally with an exclused element.
+  This procedure uses the Playwright page.screenshot method, which is not implemented for the
+  firefox browser type.
+*/
+
+// FUNCTIONS
+
+// Creates and returns a screenshot.
+exports.screenShot = async (page, exclusion = null) => {
+  const options = {
+    fullPage: true,
+    omitBackground: true,
+    timeout: 2000
+  };
+  if (exclusion) {
+    options.mask = [exclusion];
+  }
+  // Make and return a screenshot as a buffer.
+  return await page.screenshot(options)
+  .catch(error => {
+    console.log(`ERROR: Screenshot failed (${error.message})`);
+    return '';
+  });
+};

package/procs/standardize.js

@@ -1,5 +1,6 @@
 /*
   © 2023–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -705,7 +706,7 @@ const convert = (toolName, data, result, standardResult) => {
   // Round the totals of the standard result.
   standardResult.totals = standardResult.totals.map(total => Math.round(total));
 };
-// Converts the results.
+// Converts the results of a test act.
 exports.standardize = act => {
   const {which, data, result, standardResult} = act;
   if (which && result && standardResult) {

package/procs/testaro.js

@@ -68,7 +68,9 @@ const init = exports.init = async (sampleMax, page, locAllSelector, options = {}
 };
 
 // Populates and returns a result.
-const report = exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName = '') => {
+const getRuleResult = exports.getRuleResult = async (
+  withItems, all, ruleID, whats, ordinalSeverity, tagName = ''
+) => {
   const {locs, result} = all;
   const {data, totals, standardInstances} = result;
   // For each violation locator:
@@ -149,7 +151,7 @@ exports.simplify = async (page, withItems, ruleData) => {
     complaints.instance,
     complaints.summary
   ];
-  const result = await report(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
+  const result = await getRuleResult(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
   // Return the result.
   return result;
 };

package/procs/visChange.js

@@ -1,5 +1,6 @@
 /*
-  © 2023–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -25,37 +26,18 @@
 /*
   visChange
   This procedure reports a change in the visible content of a page between two times, optionally
-  hovering over a locator-defined element immediately after the first time.
-
-  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
-  when the browser type is chromium and is not implemented for the firefox browser type. The only
-  browser type usable with this test is webkit.
+  hovering over a locator-defined element immediately after the first time. This test uses the
+  Playwright page.screenshot method, which is not implemented for the firefox browser type.
 */
 
 // IMPORTS
 
 const pixelmatch = require('pixelmatch').default;
 const {PNG} = require('pngjs');
+const {screenShot} = require('./screenShot');
 
 // FUNCTIONS
 
-// Creates and returns a screenshot.
-const shoot = async (page, exclusion = null) => {
-  // Make a screenshot as a buffer.
-  const options = {
-    fullPage: true,
-    omitBackground: true,
-    timeout: 2000
-  };
-  if (exclusion) {
-    options.mask = [exclusion];
-  }
-  return await page.screenshot(options)
-  .catch(error => {
-    console.log(`ERROR: Screenshot failed (${error.message})`);
-    return '';
-  });
-};
 exports.visChange = async (page, options = {}) => {
   const {delayBefore, delayBetween, exclusion} = options;
   // Wait, if required.
@@ -74,7 +56,7 @@ exports.visChange = async (page, options = {}) => {
     });
   }
   // Make and get a screenshot, excluding an element if specified.
-  const shot0 = await shoot(page, exclusion);
+  const shot0 = await screenShot(page, exclusion);
   // If it succeeded:
   if (shot0.length) {
     // If an exclusion was specified:
@@ -96,7 +78,7 @@ exports.visChange = async (page, options = {}) => {
     // Wait as specified, or 3 seconds.
     await page.waitForTimeout(delayBetween || 3000);
     // Make and get another screenshot.
-    const shot1 = await shoot(page, exclusion);
+    const shot1 = await screenShot(page, exclusion);
     // If it succeeded:
     if (shot1.length) {
       // Get the shots as PNG images.

package/run.js

@@ -34,6 +34,8 @@
 const fs = require('fs/promises');
 // Module to keep secrets.
 require('dotenv').config({quiet: true});
+// Module to execute shell commands.
+const {execSync} = require('child_process');
 // Module to validate jobs.
 const {isBrowserID, isDeviceID, isURL, isValidJob, tools} = require('./procs/job');
 // Module to evade automation detection.
@@ -109,12 +111,12 @@ const tmpDir = os.tmpdir();
 
 // Facts about the current session.
 let actCount = 0;
-let browserCloseIntentional = false;
 // Facts about the current act.
 let actIndex = 0;
 let browser;
 let browserContext;
 let page;
+let report;
 let requestedURL = '';
 
 // FUNCTIONS
@@ -241,25 +243,6 @@ const goTo = async (report, page, url, timeout, waitUntil) => {
     };
   }
 };
-// Closes the current browser.
-const browserClose = async () => {
-  if (browser) {
-    browserCloseIntentional = true;
-    for (const context of browser.contexts()) {
-      try {
-        await context.close();
-      }
-      catch(error) {
-        console.log(
-          `ERROR trying to close context: ${error.message.slice(0, 200).replace(/\n.+/s, '')}`
-        );
-      }
-    }
-    await browser.close();
-    browserCloseIntentional = false;
-    browser = null;
-  }
-};
 // Adds an error result to an act.
 const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
   // If the error is to be logged:
@@ -286,6 +269,25 @@ const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
     abortActs(report, actIndex);
   }
 };
+// Closes the current browser.
+const browserClose = async () => {
+  if (browser) {
+    browserCloseIntentional = true;
+    for (const context of browser.contexts()) {
+      try {
+        await context.close();
+      }
+      catch(error) {
+        console.log(
+          `ERROR trying to close context: ${error.message.slice(0, 200).replace(/\n.+/s, '')}`
+        );
+      }
+    }
+    await browser.close();
+    browserCloseIntentional = false;
+    browser = null;
+  }
+};
 // Launches a browser and navigates to a URL.
 const launch = exports.launch = async (
   report, debug, waits, tempBrowserID, tempURL, retries = 2
@@ -668,6 +670,11 @@ const abortActs = (report, actIndex) => {
   // Report that the job is aborted.
   console.log(`ERROR: Job aborted on act ${actIndex}`);
 };
+// Returns the combination of browser ID and target URL of an act.
+const launchSpecs = (act, report) => [
+  act.browserID || report.browserID || '',
+  act.target && act.target.url || report.target && report.target.url || ''
+];
 // Performs the acts in a report and adds the results to the report.
 const doActs = async (report, opts = {}) => {
   const {acts} = report;
@@ -752,13 +759,14 @@ const doActs = async (report, opts = {}) => {
       }
       // Otherwise, if the act is a launch:
       else if (type === 'launch') {
+        const actLaunchSpecs = launchSpecs(act, report);
         // Launch a browser, navigate to a page, and add the result to the act.
         await launch(
           report,
           debug,
           waits,
-          act.browserID || report.browserID || '',
-          act.target && act.target.url || report.target && report.target.url || ''
+          actLaunchSpecs[0],
+          actLaunchSpecs[1]
         );
         // If this failed:
         if (page.prevented) {
@@ -819,50 +827,6 @@ const doActs = async (report, opts = {}) => {
           toolTimes[act.which] += time;
           // If the act was not prevented:
           if (act.data && ! act.data.prevented) {
-            // If standardization is required:
-            if (['also', 'only'].includes(standard)) {
-              console.log('>>>>>> Standardizing');
-              // Initialize the standard result.
-              act.standardResult = {
-                totals: [0, 0, 0, 0],
-                instances: []
-              };
-              // Populate it.
-              standardize(act);
-              // Launch a browser and navigate to the page.
-              await launch(
-                report,
-                debug,
-                waits,
-                act.browserID || report.browserID || '',
-                act.target && act.target.url || report.target && report.target.url || ''
-              );
-              // If this failed:
-              if (page.prevented) {
-                // Add this to the act.
-                act.data ??= {};
-                act.data.prevented = true;
-                act.data.error = page.error || '';
-              }
-              // Otherwise, i.e. if it succeeded:
-              else {
-                // Add a box ID and a path ID to each of its standard instances if missing.
-                for (const instance of act.standardResult.instances) {
-                  const elementID = await identify(instance, page);
-                  if (! instance.boxID) {
-                    instance.boxID = elementID ? elementID.boxID : '';
-                  }
-                  if (! instance.pathID) {
-                    instance.pathID = elementID ? elementID.pathID : '';
-                  }
-                };
-              }
-              // If the original-format result is not to be included in the report:
-              if (standard === 'only') {
-                // Remove it.
-                delete act.result;
-              }
-            }
             // If the act has expectations:
             const expectations = act.expect;
             if (expectations) {
@@ -892,7 +856,7 @@ const doActs = async (report, opts = {}) => {
       // Otherwise, if a current page exists:
       else if (page) {
         // If the act is navigation to a url:
-        if (act.type === 'url') {
+        if (type === 'url') {
           // Identify the URL.
           const resolved = act.which.replace('__dirname', __dirname);
           requestedURL = resolved;
@@ -926,7 +890,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for text:
-        else if (act.type === 'wait') {
+        else if (type === 'wait') {
           const {what, which} = act;
           console.log(`>> ${what}`);
           const result = act.result = {};
@@ -994,7 +958,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for a state:
-        else if (act.type === 'state') {
+        else if (type === 'state') {
           // Wait for it.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
           await page.waitForLoadState(
@@ -1016,7 +980,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a page switch:
-        else if (act.type === 'page') {
+        else if (type === 'page') {
           // Wait for a page to be created and identify it as current.
           page = await browserContext.waitForEvent('page');
           // Wait until it is idle.
@@ -1033,7 +997,7 @@ const doActs = async (report, opts = {}) => {
           // Add the URL to the act.
           act.actualURL = url;
           // If the act is a revelation:
-          if (act.type === 'reveal') {
+          if (type === 'reveal') {
             act.result = {
               success: true
             };
@@ -1055,8 +1019,8 @@ const doActs = async (report, opts = {}) => {
             });
           }
           // Otherwise, if the act is a move:
-          else if (moves[act.type]) {
-            const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
+          else if (moves[type]) {
+            const selector = typeof moves[type] === 'string' ? moves[type] : act.what;
             // Try up to 5 times to:
             act.result = {found: false};
             let selection = {};
@@ -1165,18 +1129,18 @@ const doActs = async (report, opts = {}) => {
               };
               // FUNCTION DEFINITION END
               // If the move is a button click, perform it.
-              if (act.type === 'button') {
+              if (type === 'button') {
                 await selection.click({timeout: 3000});
                 act.result.success = true;
                 act.result.move = 'clicked';
               }
               // Otherwise, if it is checking a radio button or checkbox, perform it.
-              else if (['checkbox', 'radio'].includes(act.type)) {
+              else if (['checkbox', 'radio'].includes(type)) {
                 await selection.waitForElementState('stable', {timeout: 2000})
                 .catch(error => {
-                  console.log(`ERROR waiting for stable ${act.type} (${error.message})`);
+                  console.log(`ERROR waiting for stable ${type} (${error.message})`);
                   act.result.success = false;
-                  act.result.error = `ERROR waiting for stable ${act.type}`;
+                  act.result.error = `ERROR waiting for stable ${type}`;
                 });
                 if (! act.result.error) {
                   const isEnabled = await selection.isEnabled();
@@ -1186,9 +1150,9 @@ const doActs = async (report, opts = {}) => {
                       timeout: 2000
                     })
                     .catch(error => {
-                      console.log(`ERROR checking ${act.type} (${error.message})`);
+                      console.log(`ERROR checking ${type} (${error.message})`);
                       act.result.success = false;
-                      act.result.error = `ERROR checking ${act.type}`;
+                      act.result.error = `ERROR checking ${type}`;
                     });
                     if (! act.result.error) {
                       act.result.success = true;
@@ -1196,20 +1160,20 @@ const doActs = async (report, opts = {}) => {
                     }
                   }
                   else {
-                    const report = `ERROR: could not check ${act.type} because disabled`;
+                    const report = `ERROR: could not check ${type} because disabled`;
                     act.result.success = false;
                     act.result.error = report;
                   }
                 }
               }
               // Otherwise, if it is focusing the element, perform it.
-              else if (act.type === 'focus') {
+              else if (type === 'focus') {
                 await selection.focus({timeout: 2000});
                 act.result.success = true;
                 act.result.move = 'focused';
               }
               // Otherwise, if it is clicking a link:
-              else if (act.type === 'link') {
+              else if (type === 'link') {
                 const href = await selection.getAttribute('href');
                 const target = await selection.getAttribute('target');
                 act.result.href = href || 'NONE';
@@ -1248,7 +1212,7 @@ const doActs = async (report, opts = {}) => {
                 }
               }
               // Otherwise, if it is selecting an option in a select list, perform it.
-              else if (act.type === 'select') {
+              else if (type === 'select') {
                 const options = await selection.$$('option');
                 let optionText = '';
                 if (options && Array.isArray(options) && options.length) {
@@ -1271,7 +1235,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.option = optionText;
               }
               // Otherwise, if it is entering text in an input element:
-              else if (['text', 'search'].includes(act.type)) {
+              else if (['text', 'search'].includes(type)) {
                 act.result.attributes = {};
                 const {attributes} = act.result;
                 const type = await selection.getAttribute('type');
@@ -1294,7 +1258,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.success = true;
                 act.result.move = 'entered';
                 // If the input is a search input:
-                if (act.type === 'search') {
+                if (type === 'search') {
                   // Press the Enter key and wait for a network to be idle.
                   doAndWait(false);
                 }
@@ -1318,7 +1282,7 @@ const doActs = async (report, opts = {}) => {
             }
           }
           // Otherwise, if the act is a keypress:
-          else if (act.type === 'press') {
+          else if (type === 'press') {
             // Identify the number of times to press the key.
             let times = 1 + (act.again || 0);
             report.jobData.presses += times;
@@ -1334,7 +1298,7 @@ const doActs = async (report, opts = {}) => {
             };
           }
           // Otherwise, if it is a repetitive keyboard navigation:
-          else if (act.type === 'presses') {
+          else if (type === 'presses') {
             const {navKey, what, which, withItems} = act;
             const matchTexts = which ? which.map(text => debloat(text)) : [];
             // Initialize the loop variables.
@@ -1501,14 +1465,102 @@ const doActs = async (report, opts = {}) => {
     }
   }
   console.log('Acts completed');
+  // If standardization is required:
+  if (['also', 'only'].includes(standard)) {
+    // If granular reporting has been specified:
+    if (report.observe) {
+      // If a progress callback has been provided:
+      if (onProgress) {
+        // Notify the observer of the start of standardization.
+        try {
+          onProgress({
+            type: 'standardization',
+            which: 'start'
+          });
+          console.log(`${'Standardization started'} (observer notified)`);
+        }
+        catch (error) {
+          console.log(`${message} (observer notification failed: ${errorStart(error)})`);
+        }
+      }
+      // Otherwise, i.e. if no progress callback has been provided:
+      else {
+        // Notify the observer of the act and log it.
+        tellServer(report, messageParams, message);
+      }
+
+    // Notify the observer and log the start of standardization.
+    tellServer(report, '', 'Starting result standardization');
+    }
+    const launchSpecActs = {};
+    // For each act:
+    report.acts.forEach((act, index) => {
+      // If it is a test act:
+      if (act.type === 'test') {
+        // Classify it by its browser ID and target URL.
+        const specs = launchSpecs(act, report);
+        const specString = `${specs[0]}>${specs[1]}`;
+        if (launchSpecActs[specString]) {
+          launchSpecActs[specString].push(index);
+        }
+        else {
+          launchSpecActs[specString] = [index];
+        }
+      }
+    });
+    // For each browser ID/target URL class:
+    for (const specString of Object.keys(launchSpecActs)) {
+      const specs = specString.split('>');
+      // Launch a browser and navigate to the page.
+      await launch(
+        report,
+        debug,
+        waits,
+        specs[0],
+        specs[1]
+      );
+      // For each test act with the class:
+      for (const specActIndex of launchSpecActs[specString]) {
+        const act = report.acts[specActIndex];
+        // Initialize the standard result.
+        act.standardResult = {
+          totals: [0, 0, 0, 0],
+          instances: []
+        };
+        // Populate it.
+        standardize(act);
+        // If the launch and navigation succeeded:
+        if (! page.prevented) {
+          // Add a box ID and a path ID to each of its standard instances if missing.
+          for (const instance of act.standardResult.instances) {
+            const elementID = await identify(instance, page);
+            if (! instance.boxID) {
+              instance.boxID = elementID ? elementID.boxID : '';
+            }
+            if (! instance.pathID) {
+              instance.pathID = elementID ? elementID.pathID : '';
+            }
+          };
+        }
+        // If the original-format result is not to be included in the report:
+        if (standard === 'only') {
+          // Remove it.
+          delete act.result;
+        }
+      };
+    };
+    console.log('>>>> Standardization completed');
+  }
+  // Close the browser.
   await browserClose();
+  // Delete the temporary report file.
   await fs.rm(reportPath, {force: true});
   return report;
 };
 // Runs a job and returns a report.
 exports.doJob = async (job, opts = {}) => {
   // Make a report as a copy of the job.
-  let report = JSON.parse(JSON.stringify(job));
+  report = JSON.parse(JSON.stringify(job));
   const jobData = report.jobData = {};
   // Get whether the job is valid and, if not, why.
   const jobInvalidity = isValidJob(job);
@@ -1568,3 +1620,52 @@ exports.doJob = async (job, opts = {}) => {
   // Return the report.
   return report;
 };
+
+// CLEANUP HANDLERS
+
+// Track whether cleanup is in progress.
+let cleanupInProgress = false;
+let browserCloseIntentional = false;
+
+// Force-kills any Playwright browser processes synchronously.
+const forceKillBrowsers = () => {
+  if (cleanupInProgress) {
+    return;
+  }
+  cleanupInProgress = true;
+  try {
+    // Kill any Chromium headless shell processes.
+    execSync('pkill -9 -f "chromium_headless_shell.*headless_shell"', {stdio: 'ignore'});
+  }
+  catch(error) {}
+};
+// Force-kills any headless shell processes synchronously on process exit.
+process.on('exit', () => {
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on beforeExit.
+process.on('beforeExit', async () => {
+  if (!browserCloseIntentional) {
+    await browserClose();
+  }
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on uncaught exceptions.
+process.on('uncaughtException', async error => {
+  console.error('Uncaught exception:', error);
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(1);
+});
+// Force-kills any headless shell processes synchronously on SIGINT.
+process.on('SIGINT', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});
+// Force-kills any headless shell processes synchronously on SIGTERM.
+process.on('SIGTERM', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});

package/testaro/adbID.js

@@ -28,7 +28,7 @@
   Clean-room rule: report elements that reference aria-describedby targets that are missing or ambiguous (duplicate ids).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // elements that reference aria-describedby
@@ -63,5 +63,5 @@ exports.reporter = async (page, withItems) => {
     'Referenced description of the element is ambiguous or missing',
     'Referenced descriptions of elements are ambiguous or missing'
   ];
-  return await report(withItems, all, 'adbID', whats, 3);
+  return await getRuleResult(withItems, all, 'adbID', whats, 3);
 };

package/testaro/altScheme.js

@@ -28,7 +28,7 @@
   Identify img elements whose alt attribute is an entire URL or clearly a file name (favicon).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // Candidate images: any img with an alt attribute (including empty)
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems) => {
     'Element has an alt attribute with a URL as its entire value',
     'img elements have alt attributes with URLs as their entire values'
   ];
-  return await report(withItems, all, 'altScheme', whats, 2);
+  return await getRuleResult(withItems, all, 'altScheme', whats, 2);
 };

package/testaro/attVal.js

@@ -30,7 +30,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
     `Element has attribute ${attributeName} with illicit value “__param__”`,
     `Elements have attribute ${attributeName} with illicit values`
   ];
-  return await report(withItems, all, 'attVal', whats, 2);
+  return await getRuleResult(withItems, all, 'attVal', whats, 2);
 };