testaro 60.3.0 → 60.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "60.3.0",
+  "version": "60.4.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/screenShot.js

@@ -0,0 +1,32 @@
+/*
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+  Licensed under the MIT License. See LICENSE file for details.
+*/
+
+/*
+  screenShot
+  This procedure creates and returns a full-page screenshot, optionally with an exclused element.
+  This procedure uses the Playwright page.screenshot method, which is not implemented for the
+  firefox browser type.
+*/
+
+// FUNCTIONS
+
+// Creates and returns a screenshot.
+exports.screenShot = async (page, exclusion = null) => {
+  const options = {
+    fullPage: true,
+    omitBackground: true,
+    timeout: 2000
+  };
+  if (exclusion) {
+    options.mask = [exclusion];
+  }
+  // Make and return a screenshot as a buffer.
+  return await page.screenshot(options)
+  .catch(error => {
+    console.log(`ERROR: Screenshot failed (${error.message})`);
+    return '';
+  });
+};

package/procs/testaro.js

@@ -68,7 +68,9 @@ const init = exports.init = async (sampleMax, page, locAllSelector, options = {}
 };
 
 // Populates and returns a result.
-const report = exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName = '') => {
+const getRuleResult = exports.getRuleResult = async (
+  withItems, all, ruleID, whats, ordinalSeverity, tagName = ''
+) => {
   const {locs, result} = all;
   const {data, totals, standardInstances} = result;
   // For each violation locator:
@@ -149,7 +151,7 @@ exports.simplify = async (page, withItems, ruleData) => {
     complaints.instance,
     complaints.summary
   ];
-  const result = await report(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
+  const result = await getRuleResult(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
   // Return the result.
   return result;
 };

package/procs/visChange.js

@@ -1,5 +1,6 @@
 /*
-  © 2023–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -25,37 +26,18 @@
 /*
   visChange
   This procedure reports a change in the visible content of a page between two times, optionally
-  hovering over a locator-defined element immediately after the first time.
-
-  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
-  when the browser type is chromium and is not implemented for the firefox browser type. The only
-  browser type usable with this test is webkit.
+  hovering over a locator-defined element immediately after the first time. This test uses the
+  Playwright page.screenshot method, which is not implemented for the firefox browser type.
 */
 
 // IMPORTS
 
 const pixelmatch = require('pixelmatch').default;
 const {PNG} = require('pngjs');
+const {screenShot} = require('./screenShot');
 
 // FUNCTIONS
 
-// Creates and returns a screenshot.
-const shoot = async (page, exclusion = null) => {
-  // Make a screenshot as a buffer.
-  const options = {
-    fullPage: true,
-    omitBackground: true,
-    timeout: 2000
-  };
-  if (exclusion) {
-    options.mask = [exclusion];
-  }
-  return await page.screenshot(options)
-  .catch(error => {
-    console.log(`ERROR: Screenshot failed (${error.message})`);
-    return '';
-  });
-};
 exports.visChange = async (page, options = {}) => {
   const {delayBefore, delayBetween, exclusion} = options;
   // Wait, if required.
@@ -74,7 +56,7 @@ exports.visChange = async (page, options = {}) => {
     });
   }
   // Make and get a screenshot, excluding an element if specified.
-  const shot0 = await shoot(page, exclusion);
+  const shot0 = await screenShot(page, exclusion);
   // If it succeeded:
   if (shot0.length) {
     // If an exclusion was specified:
@@ -96,7 +78,7 @@ exports.visChange = async (page, options = {}) => {
     // Wait as specified, or 3 seconds.
     await page.waitForTimeout(delayBetween || 3000);
     // Make and get another screenshot.
-    const shot1 = await shoot(page, exclusion);
+    const shot1 = await screenShot(page, exclusion);
     // If it succeeded:
     if (shot1.length) {
       // Get the shots as PNG images.

package/run.js

@@ -34,6 +34,8 @@
 const fs = require('fs/promises');
 // Module to keep secrets.
 require('dotenv').config({quiet: true});
+// Module to execute shell commands.
+const {execSync} = require('child_process');
 // Module to validate jobs.
 const {isBrowserID, isDeviceID, isURL, isValidJob, tools} = require('./procs/job');
 // Module to evade automation detection.
@@ -109,12 +111,12 @@ const tmpDir = os.tmpdir();
 
 // Facts about the current session.
 let actCount = 0;
-let browserCloseIntentional = false;
 // Facts about the current act.
 let actIndex = 0;
 let browser;
 let browserContext;
 let page;
+let report;
 let requestedURL = '';
 
 // FUNCTIONS
@@ -241,25 +243,6 @@ const goTo = async (report, page, url, timeout, waitUntil) => {
     };
   }
 };
-// Closes the current browser.
-const browserClose = async () => {
-  if (browser) {
-    browserCloseIntentional = true;
-    for (const context of browser.contexts()) {
-      try {
-        await context.close();
-      }
-      catch(error) {
-        console.log(
-          `ERROR trying to close context: ${error.message.slice(0, 200).replace(/\n.+/s, '')}`
-        );
-      }
-    }
-    await browser.close();
-    browserCloseIntentional = false;
-    browser = null;
-  }
-};
 // Adds an error result to an act.
 const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
   // If the error is to be logged:
@@ -286,6 +269,25 @@ const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
     abortActs(report, actIndex);
   }
 };
+// Closes the current browser.
+const browserClose = async () => {
+  if (browser) {
+    browserCloseIntentional = true;
+    for (const context of browser.contexts()) {
+      try {
+        await context.close();
+      }
+      catch(error) {
+        console.log(
+          `ERROR trying to close context: ${error.message.slice(0, 200).replace(/\n.+/s, '')}`
+        );
+      }
+    }
+    await browser.close();
+    browserCloseIntentional = false;
+    browser = null;
+  }
+};
 // Launches a browser and navigates to a URL.
 const launch = exports.launch = async (
   report, debug, waits, tempBrowserID, tempURL, retries = 2
@@ -854,7 +856,7 @@ const doActs = async (report, opts = {}) => {
       // Otherwise, if a current page exists:
       else if (page) {
         // If the act is navigation to a url:
-        if (act.type === 'url') {
+        if (type === 'url') {
           // Identify the URL.
           const resolved = act.which.replace('__dirname', __dirname);
           requestedURL = resolved;
@@ -888,7 +890,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for text:
-        else if (act.type === 'wait') {
+        else if (type === 'wait') {
           const {what, which} = act;
           console.log(`>> ${what}`);
           const result = act.result = {};
@@ -956,7 +958,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for a state:
-        else if (act.type === 'state') {
+        else if (type === 'state') {
           // Wait for it.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
           await page.waitForLoadState(
@@ -978,7 +980,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a page switch:
-        else if (act.type === 'page') {
+        else if (type === 'page') {
           // Wait for a page to be created and identify it as current.
           page = await browserContext.waitForEvent('page');
           // Wait until it is idle.
@@ -995,7 +997,7 @@ const doActs = async (report, opts = {}) => {
           // Add the URL to the act.
           act.actualURL = url;
           // If the act is a revelation:
-          if (act.type === 'reveal') {
+          if (type === 'reveal') {
             act.result = {
               success: true
             };
@@ -1017,8 +1019,8 @@ const doActs = async (report, opts = {}) => {
             });
           }
           // Otherwise, if the act is a move:
-          else if (moves[act.type]) {
-            const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
+          else if (moves[type]) {
+            const selector = typeof moves[type] === 'string' ? moves[type] : act.what;
             // Try up to 5 times to:
             act.result = {found: false};
             let selection = {};
@@ -1127,18 +1129,18 @@ const doActs = async (report, opts = {}) => {
               };
               // FUNCTION DEFINITION END
               // If the move is a button click, perform it.
-              if (act.type === 'button') {
+              if (type === 'button') {
                 await selection.click({timeout: 3000});
                 act.result.success = true;
                 act.result.move = 'clicked';
               }
               // Otherwise, if it is checking a radio button or checkbox, perform it.
-              else if (['checkbox', 'radio'].includes(act.type)) {
+              else if (['checkbox', 'radio'].includes(type)) {
                 await selection.waitForElementState('stable', {timeout: 2000})
                 .catch(error => {
-                  console.log(`ERROR waiting for stable ${act.type} (${error.message})`);
+                  console.log(`ERROR waiting for stable ${type} (${error.message})`);
                   act.result.success = false;
-                  act.result.error = `ERROR waiting for stable ${act.type}`;
+                  act.result.error = `ERROR waiting for stable ${type}`;
                 });
                 if (! act.result.error) {
                   const isEnabled = await selection.isEnabled();
@@ -1148,9 +1150,9 @@ const doActs = async (report, opts = {}) => {
                       timeout: 2000
                     })
                     .catch(error => {
-                      console.log(`ERROR checking ${act.type} (${error.message})`);
+                      console.log(`ERROR checking ${type} (${error.message})`);
                       act.result.success = false;
-                      act.result.error = `ERROR checking ${act.type}`;
+                      act.result.error = `ERROR checking ${type}`;
                     });
                     if (! act.result.error) {
                       act.result.success = true;
@@ -1158,20 +1160,20 @@ const doActs = async (report, opts = {}) => {
                     }
                   }
                   else {
-                    const report = `ERROR: could not check ${act.type} because disabled`;
+                    const report = `ERROR: could not check ${type} because disabled`;
                     act.result.success = false;
                     act.result.error = report;
                   }
                 }
               }
               // Otherwise, if it is focusing the element, perform it.
-              else if (act.type === 'focus') {
+              else if (type === 'focus') {
                 await selection.focus({timeout: 2000});
                 act.result.success = true;
                 act.result.move = 'focused';
               }
               // Otherwise, if it is clicking a link:
-              else if (act.type === 'link') {
+              else if (type === 'link') {
                 const href = await selection.getAttribute('href');
                 const target = await selection.getAttribute('target');
                 act.result.href = href || 'NONE';
@@ -1210,7 +1212,7 @@ const doActs = async (report, opts = {}) => {
                 }
               }
               // Otherwise, if it is selecting an option in a select list, perform it.
-              else if (act.type === 'select') {
+              else if (type === 'select') {
                 const options = await selection.$$('option');
                 let optionText = '';
                 if (options && Array.isArray(options) && options.length) {
@@ -1233,7 +1235,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.option = optionText;
               }
               // Otherwise, if it is entering text in an input element:
-              else if (['text', 'search'].includes(act.type)) {
+              else if (['text', 'search'].includes(type)) {
                 act.result.attributes = {};
                 const {attributes} = act.result;
                 const type = await selection.getAttribute('type');
@@ -1256,7 +1258,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.success = true;
                 act.result.move = 'entered';
                 // If the input is a search input:
-                if (act.type === 'search') {
+                if (type === 'search') {
                   // Press the Enter key and wait for a network to be idle.
                   doAndWait(false);
                 }
@@ -1280,7 +1282,7 @@ const doActs = async (report, opts = {}) => {
             }
           }
           // Otherwise, if the act is a keypress:
-          else if (act.type === 'press') {
+          else if (type === 'press') {
             // Identify the number of times to press the key.
             let times = 1 + (act.again || 0);
             report.jobData.presses += times;
@@ -1296,7 +1298,7 @@ const doActs = async (report, opts = {}) => {
             };
           }
           // Otherwise, if it is a repetitive keyboard navigation:
-          else if (act.type === 'presses') {
+          else if (type === 'presses') {
             const {navKey, what, which, withItems} = act;
             const matchTexts = which ? which.map(text => debloat(text)) : [];
             // Initialize the loop variables.
@@ -1465,7 +1467,31 @@ const doActs = async (report, opts = {}) => {
   console.log('Acts completed');
   // If standardization is required:
   if (['also', 'only'].includes(standard)) {
-    console.log('>>>> Standardizing results of test acts');
+    // If granular reporting has been specified:
+    if (report.observe) {
+      // If a progress callback has been provided:
+      if (onProgress) {
+        // Notify the observer of the start of standardization.
+        try {
+          onProgress({
+            type: 'standardization',
+            which: 'start'
+          });
+          console.log(`${'Standardization started'} (observer notified)`);
+        }
+        catch (error) {
+          console.log(`${message} (observer notification failed: ${errorStart(error)})`);
+        }
+      }
+      // Otherwise, i.e. if no progress callback has been provided:
+      else {
+        // Notify the observer of the act and log it.
+        tellServer(report, messageParams, message);
+      }
+
+    // Notify the observer and log the start of standardization.
+    tellServer(report, '', 'Starting result standardization');
+    }
     const launchSpecActs = {};
     // For each act:
     report.acts.forEach((act, index) => {
@@ -1534,7 +1560,7 @@ const doActs = async (report, opts = {}) => {
 // Runs a job and returns a report.
 exports.doJob = async (job, opts = {}) => {
   // Make a report as a copy of the job.
-  let report = JSON.parse(JSON.stringify(job));
+  report = JSON.parse(JSON.stringify(job));
   const jobData = report.jobData = {};
   // Get whether the job is valid and, if not, why.
   const jobInvalidity = isValidJob(job);
@@ -1594,3 +1620,52 @@ exports.doJob = async (job, opts = {}) => {
   // Return the report.
   return report;
 };
+
+// CLEANUP HANDLERS
+
+// Track whether cleanup is in progress.
+let cleanupInProgress = false;
+let browserCloseIntentional = false;
+
+// Force-kills any Playwright browser processes synchronously.
+const forceKillBrowsers = () => {
+  if (cleanupInProgress) {
+    return;
+  }
+  cleanupInProgress = true;
+  try {
+    // Kill any Chromium headless shell processes.
+    execSync('pkill -9 -f "chromium_headless_shell.*headless_shell"', {stdio: 'ignore'});
+  }
+  catch(error) {}
+};
+// Force-kills any headless shell processes synchronously on process exit.
+process.on('exit', () => {
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on beforeExit.
+process.on('beforeExit', async () => {
+  if (!browserCloseIntentional) {
+    await browserClose();
+  }
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on uncaught exceptions.
+process.on('uncaughtException', async error => {
+  console.error('Uncaught exception:', error);
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(1);
+});
+// Force-kills any headless shell processes synchronously on SIGINT.
+process.on('SIGINT', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});
+// Force-kills any headless shell processes synchronously on SIGTERM.
+process.on('SIGTERM', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});

package/testaro/adbID.js

@@ -28,7 +28,7 @@
   Clean-room rule: report elements that reference aria-describedby targets that are missing or ambiguous (duplicate ids).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // elements that reference aria-describedby
@@ -63,5 +63,5 @@ exports.reporter = async (page, withItems) => {
     'Referenced description of the element is ambiguous or missing',
     'Referenced descriptions of elements are ambiguous or missing'
   ];
-  return await report(withItems, all, 'adbID', whats, 3);
+  return await getRuleResult(withItems, all, 'adbID', whats, 3);
 };

package/testaro/altScheme.js

@@ -28,7 +28,7 @@
   Identify img elements whose alt attribute is an entire URL or clearly a file name (favicon).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // Candidate images: any img with an alt attribute (including empty)
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems) => {
     'Element has an alt attribute with a URL as its entire value',
     'img elements have alt attributes with URLs as their entire values'
   ];
-  return await report(withItems, all, 'altScheme', whats, 2);
+  return await getRuleResult(withItems, all, 'altScheme', whats, 2);
 };

package/testaro/attVal.js

@@ -30,7 +30,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
     `Element has attribute ${attributeName} with illicit value “__param__”`,
     `Elements have attribute ${attributeName} with illicit values`
   ];
-  return await report(withItems, all, 'attVal', whats, 2);
+  return await getRuleResult(withItems, all, 'attVal', whats, 2);
 };

package/testaro/autocomplete.js

@@ -30,7 +30,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 // Module to get locator data.
 const {getLocatorData} = require('../procs/getLocatorData');
 
@@ -81,5 +81,5 @@ exports.reporter = async (
     'Input is missing an autocomplete attribute with value __param__',
     'Inputs are missing applicable autocomplete attributes'
   ];
-  return await report(withItems, all, 'autocomplete', whats, 2);
+  return await getRuleResult(withItems, all, 'autocomplete', whats, 2);
 };

package/testaro/captionLoc.js

@@ -28,7 +28,7 @@
   Report caption elements that are not the first child of their table element.
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   const all = await init(100, page, 'caption');
@@ -44,5 +44,5 @@ exports.reporter = async (page, withItems) => {
     'Element is not the first child of a table element',
     'caption elements are not the first children of table elements'
   ];
-  return await report(withItems, all, 'captionLoc', whats, 3, 'CAPTION');
+  return await getRuleResult(withItems, all, 'captionLoc', whats, 3, 'CAPTION');
 };

package/testaro/datalistRef.js

@@ -28,7 +28,7 @@
   Report inputs whose list attribute references a missing or ambiguous datalist
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   const all = await init(100, page, 'input[list]');
@@ -45,5 +45,5 @@ exports.reporter = async (page, withItems) => {
     'list attribute of the element references an ambiguous or missing datalist element',
     'list attributes of elements reference ambiguous or missing datalist elements'
   ];
-  return await report(withItems, all, 'datalistRef', whats, 3, 'INPUT');
+  return await getRuleResult(withItems, all, 'datalistRef', whats, 3, 'INPUT');
 };

package/testaro/embAc.js

@@ -33,7 +33,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -66,5 +66,5 @@ exports.reporter = async (page, withItems) => {
     'Interactive element is embedded in __param__',
     'Interactive elements are contained by links or buttons'
   ];
-  return await report(withItems, all, 'embAc', whats, 2);
+  return await getRuleResult(withItems, all, 'embAc', whats, 2);
 };

package/testaro/focInd.js

@@ -44,7 +44,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -126,5 +126,5 @@ exports.reporter = async (page, withItems) => {
   }
   // Populate and return the result.
   const whats = ['Element has __param__', 'Elements fail to have standard focus indicators'];
-  return await report(withItems, all, 'focInd', whats, 1);
+  return await getRuleResult(withItems, all, 'focInd', whats, 1);
 };

package/testaro/focOp.js

@@ -38,7 +38,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 // Module to get operabilities.
 const {isOperable} = require('../procs/operable');
 
@@ -70,6 +70,6 @@ exports.reporter = async (page, withItems) => {
   const whats = [
     'Element is Tab-focusable but not operable', 'Elements are Tab-focusable but not operable'
   ];
-  const result = await report(withItems, all, 'focOp', whats, 2);
+  const result = await getRuleResult(withItems, all, 'focOp', whats, 2);
   return result;
 };

package/testaro/focVis.js

@@ -32,7 +32,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -69,5 +69,5 @@ exports.reporter = async (page, withItems) => {
     'Visible link is __param__ the display',
     'Visible links are above or to the left of the display'
   ];
-  return await report(withItems, all, 'focVis', whats, 2);
+  return await getRuleResult(withItems, all, 'focVis', whats, 2);
 };

package/testaro/hover.js

@@ -36,7 +36,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -105,5 +105,5 @@ exports.reporter = async (page, withItems) => {
     'Hovering over the element __param__',
     'Hovering over elements adds elements to or subtracts elements from the page'
   ];
-  return await report(withItems, all, 'hover', whats, 0);
+  return await getRuleResult(withItems, all, 'hover', whats, 0);
 };

package/testaro/labClash.js

@@ -30,7 +30,7 @@
 */
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -66,5 +66,5 @@ exports.reporter = async (page, withItems) => {
   const whats = [
     'Element has inconsistent label types (__param__)', 'Elements have inconsistent label types'
   ];
-  return await report(withItems, all, 'labClash', whats, 2);
+  return await getRuleResult(withItems, all, 'labClash', whats, 2);
 };

package/testaro/lineHeight.js

@@ -35,7 +35,7 @@
 // IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // FUNCTIONS
 
@@ -66,5 +66,5 @@ exports.reporter = async (page, withItems) => {
     'Element line height is less than 1.5 times its font size (__param__)',
     'Elements have line heights less than 1.5 times their font sizes'
   ];
-  return await report(withItems, all, 'lineHeight', whats, 1);
+  return await getRuleResult(withItems, all, 'lineHeight', whats, 1);
 };