testaro 60.3.0 → 60.4.1

package/UPGRADES.md

@@ -2802,3 +2802,113 @@ This gives you:
 ✅ No git conflicts
 
 Claude Sonnet 4.5 • 1x
+
+## Cleanup synchronization
+
+Claude Sonnet 4.5 recommendation to handle already-closed error messages on server (but not the local host) after end of a job:
+
+The issue is that forceKillBrowsers() is killing the browser processes immediately, but the async browserClose() calls in the signal handlers are still trying to close already-dead browsers, causing repeated errors.
+
+Solution: Coordinate Async and Sync Cleanup
+Replace the cleanup handlers in run.js:
+
+```
+// CLEANUP HANDLERS
+
+// Track cleanup state.
+let cleanupInProgress = false;
+let forceKillScheduled = false;
+
+// Force kills all Playwright browser processes synchronously.
+const forceKillBrowsers = () => {
+  if (cleanupInProgress) {
+    return;
+  }
+  cleanupInProgress = true;
+
+  const {execSync} = require('child_process');
+  try {
+    // Kill Chromium headless shell processes.
+    execSync('pkill -9 -f "chromium_headless_shell.*headless_shell"', {stdio: 'ignore'});
+  }
+  catch(error) {
+    // Ignore errors - processes may not exist.
+  }
+};
+
+// Graceful cleanup with timeout fallback.
+const gracefulCleanup = async (timeoutMs = 2000) => {
+  if (forceKillScheduled) {
+    return;
+  }
+  forceKillScheduled = true;
+
+  // Try graceful close with timeout.
+  const closePromise = browserClose();
+  const timeoutPromise = new Promise(resolve => setTimeout(resolve, timeoutMs));
+
+  await Promise.race([closePromise, timeoutPromise]);
+
+  // Force kill any remaining processes.
+  forceKillBrowsers();
+};
+
+// Synchronous cleanup on process exit (last resort).
+process.on('exit', () => {
+  if (!cleanupInProgress) {
+    forceKillBrowsers();
+  }
+});
+
+// Graceful cleanup on SIGINT (Ctrl+C).
+process.on('SIGINT', async () => {
+  console.log('\nReceived SIGINT, cleaning up...');
+  await gracefulCleanup();
+  process.exit(130);
+});
+
+// Graceful cleanup on SIGTERM.
+process.on('SIGTERM', async () => {
+  console.log('Received SIGTERM, cleaning up...');
+  await gracefulCleanup();
+  process.exit(143);
+});
+
+// Cleanup on uncaught exceptions.
+process.on('uncaughtException', async error => {
+  console.error('Uncaught exception:', error);
+  await gracefulCleanup(1000);
+  process.exit(1);
+});
+```
+
+And in `doActs`, replace the cleanup:
+
+```
+  console.log('Acts completed');
+  // If standardization is required:
+  if (['also', 'only'].includes(standard)) {
+    // ... existing standardization code ...
+  }
+  // Close the browser gracefully.
+  await gracefulCleanup();
+  // Delete the temporary report file.
+  await fs.rm(reportPath, {force: true});
+  return report;
+  ```
+
+Key changes:
+
+gracefulCleanup(): Attempts async browserClose() but races it against a timeout, then force-kills
+Guard flags: Prevent multiple simultaneous cleanup attempts that cause the error messages
+Timeout: Gives browserClose 2 seconds to complete before forcing kill
+No redundant calls: Only gracefulCleanup() calls both browserClose() and forceKillBrowsers()
+process.on('exit') as fallback: Only runs if other cleanup didn't complete
+This approach:
+
+Tries graceful closure first (avoiding the errors)
+Falls back to force-kill after timeout
+Prevents the async cleanup from running after force-kill
+Works consistently on both macOS and Ubuntu
+
+The above recommendation seems complex. Meanwhile the error message has been suppressed on the basis that context closure is not necessarily an error.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "60.3.0",
+  "version": "60.4.1",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/procs/screenShot.js

@@ -0,0 +1,32 @@
+/*
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
+  Licensed under the MIT License. See LICENSE file for details.
+*/
+
+/*
+  screenShot
+  This procedure creates and returns a full-page screenshot, optionally with an exclused element.
+  This procedure uses the Playwright page.screenshot method, which is not implemented for the
+  firefox browser type.
+*/
+
+// FUNCTIONS
+
+// Creates and returns a screenshot.
+exports.screenShot = async (page, exclusion = null) => {
+  const options = {
+    fullPage: true,
+    omitBackground: true,
+    timeout: 2000
+  };
+  if (exclusion) {
+    options.mask = [exclusion];
+  }
+  // Make and return a screenshot as a buffer.
+  return await page.screenshot(options)
+  .catch(error => {
+    console.log(`ERROR: Screenshot failed (${error.message})`);
+    return '';
+  });
+};

package/procs/tellServer.js

@@ -35,7 +35,7 @@ const agent = process.env.AGENT;
 
 // FUNCTIONS
 
-// Sends a notification to an observer.
+// Sends a notice to an observer.
 exports.tellServer = (report, messageParams, logMessage) => {
   const {serverID} = report.sources;
   const observerURL = typeof serverID === 'number' ? process.env[`NETWATCH_URL_${serverID}_OBSERVE`] : '';

package/procs/testaro.js

@@ -68,7 +68,9 @@ const init = exports.init = async (sampleMax, page, locAllSelector, options = {}
 };
 
 // Populates and returns a result.
-const report = exports.report = async (withItems, all, ruleID, whats, ordinalSeverity, tagName = '') => {
+const getRuleResult = exports.getRuleResult = async (
+  withItems, all, ruleID, whats, ordinalSeverity, tagName = ''
+) => {
   const {locs, result} = all;
   const {data, totals, standardInstances} = result;
   // For each violation locator:
@@ -149,7 +151,7 @@ exports.simplify = async (page, withItems, ruleData) => {
     complaints.instance,
     complaints.summary
   ];
-  const result = await report(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
+  const result = await getRuleResult(withItems, all, ruleID, whats, ordinalSeverity, summaryTagName);
   // Return the result.
   return result;
 };

package/procs/visChange.js

@@ -1,5 +1,6 @@
 /*
-  © 2023–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2023–2025 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2025 Jonathan Robert Pool. All rights reserved.
 
   MIT License
 
@@ -25,37 +26,18 @@
 /*
   visChange
   This procedure reports a change in the visible content of a page between two times, optionally
-  hovering over a locator-defined element immediately after the first time.
-
-  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
-  when the browser type is chromium and is not implemented for the firefox browser type. The only
-  browser type usable with this test is webkit.
+  hovering over a locator-defined element immediately after the first time. This test uses the
+  Playwright page.screenshot method, which is not implemented for the firefox browser type.
 */
 
 // IMPORTS
 
 const pixelmatch = require('pixelmatch').default;
 const {PNG} = require('pngjs');
+const {screenShot} = require('./screenShot');
 
 // FUNCTIONS
 
-// Creates and returns a screenshot.
-const shoot = async (page, exclusion = null) => {
-  // Make a screenshot as a buffer.
-  const options = {
-    fullPage: true,
-    omitBackground: true,
-    timeout: 2000
-  };
-  if (exclusion) {
-    options.mask = [exclusion];
-  }
-  return await page.screenshot(options)
-  .catch(error => {
-    console.log(`ERROR: Screenshot failed (${error.message})`);
-    return '';
-  });
-};
 exports.visChange = async (page, options = {}) => {
   const {delayBefore, delayBetween, exclusion} = options;
   // Wait, if required.
@@ -74,7 +56,7 @@ exports.visChange = async (page, options = {}) => {
     });
   }
   // Make and get a screenshot, excluding an element if specified.
-  const shot0 = await shoot(page, exclusion);
+  const shot0 = await screenShot(page, exclusion);
   // If it succeeded:
   if (shot0.length) {
     // If an exclusion was specified:
@@ -96,7 +78,7 @@ exports.visChange = async (page, options = {}) => {
     // Wait as specified, or 3 seconds.
     await page.waitForTimeout(delayBetween || 3000);
     // Make and get another screenshot.
-    const shot1 = await shoot(page, exclusion);
+    const shot1 = await screenShot(page, exclusion);
     // If it succeeded:
     if (shot1.length) {
       // Get the shots as PNG images.

package/run.js

@@ -34,6 +34,8 @@
 const fs = require('fs/promises');
 // Module to keep secrets.
 require('dotenv').config({quiet: true});
+// Module to execute shell commands.
+const {execSync} = require('child_process');
 // Module to validate jobs.
 const {isBrowserID, isDeviceID, isURL, isValidJob, tools} = require('./procs/job');
 // Module to evade automation detection.
@@ -109,12 +111,14 @@ const tmpDir = os.tmpdir();
 
 // Facts about the current session.
 let actCount = 0;
-let browserCloseIntentional = false;
 // Facts about the current act.
 let actIndex = 0;
 let browser;
+let cleanupInProgress = false;
+let browserCloseIntentional = false;
 let browserContext;
 let page;
+let report;
 let requestedURL = '';
 
 // FUNCTIONS
@@ -241,25 +245,6 @@ const goTo = async (report, page, url, timeout, waitUntil) => {
     };
   }
 };
-// Closes the current browser.
-const browserClose = async () => {
-  if (browser) {
-    browserCloseIntentional = true;
-    for (const context of browser.contexts()) {
-      try {
-        await context.close();
-      }
-      catch(error) {
-        console.log(
-          `ERROR trying to close context: ${error.message.slice(0, 200).replace(/\n.+/s, '')}`
-        );
-      }
-    }
-    await browser.close();
-    browserCloseIntentional = false;
-    browser = null;
-  }
-};
 // Adds an error result to an act.
 const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
   // If the error is to be logged:
@@ -286,6 +271,24 @@ const addError = (alsoLog, alsoAbort, report, actIndex, message) => {
     abortActs(report, actIndex);
   }
 };
+// Closes any current browser.
+const browserClose = async () => {
+  // If a browser exists:
+  if (browser) {
+    browserCloseIntentional = true;
+    // Try to close all its contexts and ignore any messages that they are already closed.
+    for (const context of browser.contexts()) {
+      try {
+        await context.close();
+      }
+      catch(error) {}
+    }
+    // Close the browser.
+    await browser.close();
+    browserCloseIntentional = false;
+    browser = null;
+  }
+};
 // Launches a browser and navigates to a URL.
 const launch = exports.launch = async (
   report, debug, waits, tempBrowserID, tempURL, retries = 2
@@ -301,7 +304,7 @@ const launch = exports.launch = async (
     report.target.url = url;
     // Create a browser of the specified or default type.
     const browserType = playwrightBrowsers[browserID];
-    // Close the current browser, if any.
+    // Close any current browser.
     await browserClose();
     // Define browser options.
     const browserOptions = {
@@ -676,6 +679,7 @@ const launchSpecs = (act, report) => [
 // Performs the acts in a report and adds the results to the report.
 const doActs = async (report, opts = {}) => {
   const {acts} = report;
+  // Get the granular observation options, if any.
   const {onProgress = null, signal = null} = opts;
   // Get the standardization specification.
   const standard = report.standard || 'only';
@@ -694,7 +698,7 @@ const doActs = async (report, opts = {}) => {
       if (report.observe) {
         const whichParam = which ? `&which=${which}` : '';
         const messageParams = `act=${type}${whichParam}`;
-        // If a progress callback has been provided:
+        // If a progress callback has been provided by a caller on this host:
         if (onProgress) {
           // Notify the observer of the act.
           try {
@@ -710,7 +714,7 @@ const doActs = async (report, opts = {}) => {
         }
         // Otherwise, i.e. if no progress callback has been provided:
         else {
-          // Notify the observer of the act and log it.
+          // Notify the remote observer of the act and log it.
           tellServer(report, messageParams, message);
         }
       }
@@ -854,7 +858,7 @@ const doActs = async (report, opts = {}) => {
       // Otherwise, if a current page exists:
       else if (page) {
         // If the act is navigation to a url:
-        if (act.type === 'url') {
+        if (type === 'url') {
           // Identify the URL.
           const resolved = act.which.replace('__dirname', __dirname);
           requestedURL = resolved;
@@ -888,7 +892,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for text:
-        else if (act.type === 'wait') {
+        else if (type === 'wait') {
           const {what, which} = act;
           console.log(`>> ${what}`);
           const result = act.result = {};
@@ -956,7 +960,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a wait for a state:
-        else if (act.type === 'state') {
+        else if (type === 'state') {
           // Wait for it.
           const stateIndex = ['loaded', 'idle'].indexOf(act.which);
           await page.waitForLoadState(
@@ -978,7 +982,7 @@ const doActs = async (report, opts = {}) => {
           }
         }
         // Otherwise, if the act is a page switch:
-        else if (act.type === 'page') {
+        else if (type === 'page') {
           // Wait for a page to be created and identify it as current.
           page = await browserContext.waitForEvent('page');
           // Wait until it is idle.
@@ -995,7 +999,7 @@ const doActs = async (report, opts = {}) => {
           // Add the URL to the act.
           act.actualURL = url;
           // If the act is a revelation:
-          if (act.type === 'reveal') {
+          if (type === 'reveal') {
             act.result = {
               success: true
             };
@@ -1017,8 +1021,8 @@ const doActs = async (report, opts = {}) => {
             });
           }
           // Otherwise, if the act is a move:
-          else if (moves[act.type]) {
-            const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
+          else if (moves[type]) {
+            const selector = typeof moves[type] === 'string' ? moves[type] : act.what;
             // Try up to 5 times to:
             act.result = {found: false};
             let selection = {};
@@ -1127,18 +1131,18 @@ const doActs = async (report, opts = {}) => {
               };
               // FUNCTION DEFINITION END
               // If the move is a button click, perform it.
-              if (act.type === 'button') {
+              if (type === 'button') {
                 await selection.click({timeout: 3000});
                 act.result.success = true;
                 act.result.move = 'clicked';
               }
               // Otherwise, if it is checking a radio button or checkbox, perform it.
-              else if (['checkbox', 'radio'].includes(act.type)) {
+              else if (['checkbox', 'radio'].includes(type)) {
                 await selection.waitForElementState('stable', {timeout: 2000})
                 .catch(error => {
-                  console.log(`ERROR waiting for stable ${act.type} (${error.message})`);
+                  console.log(`ERROR waiting for stable ${type} (${error.message})`);
                   act.result.success = false;
-                  act.result.error = `ERROR waiting for stable ${act.type}`;
+                  act.result.error = `ERROR waiting for stable ${type}`;
                 });
                 if (! act.result.error) {
                   const isEnabled = await selection.isEnabled();
@@ -1148,9 +1152,9 @@ const doActs = async (report, opts = {}) => {
                       timeout: 2000
                     })
                     .catch(error => {
-                      console.log(`ERROR checking ${act.type} (${error.message})`);
+                      console.log(`ERROR checking ${type} (${error.message})`);
                       act.result.success = false;
-                      act.result.error = `ERROR checking ${act.type}`;
+                      act.result.error = `ERROR checking ${type}`;
                     });
                     if (! act.result.error) {
                       act.result.success = true;
@@ -1158,20 +1162,20 @@ const doActs = async (report, opts = {}) => {
                     }
                   }
                   else {
-                    const report = `ERROR: could not check ${act.type} because disabled`;
+                    const report = `ERROR: could not check ${type} because disabled`;
                     act.result.success = false;
                     act.result.error = report;
                   }
                 }
               }
               // Otherwise, if it is focusing the element, perform it.
-              else if (act.type === 'focus') {
+              else if (type === 'focus') {
                 await selection.focus({timeout: 2000});
                 act.result.success = true;
                 act.result.move = 'focused';
               }
               // Otherwise, if it is clicking a link:
-              else if (act.type === 'link') {
+              else if (type === 'link') {
                 const href = await selection.getAttribute('href');
                 const target = await selection.getAttribute('target');
                 act.result.href = href || 'NONE';
@@ -1210,7 +1214,7 @@ const doActs = async (report, opts = {}) => {
                 }
               }
               // Otherwise, if it is selecting an option in a select list, perform it.
-              else if (act.type === 'select') {
+              else if (type === 'select') {
                 const options = await selection.$$('option');
                 let optionText = '';
                 if (options && Array.isArray(options) && options.length) {
@@ -1233,7 +1237,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.option = optionText;
               }
               // Otherwise, if it is entering text in an input element:
-              else if (['text', 'search'].includes(act.type)) {
+              else if (['text', 'search'].includes(type)) {
                 act.result.attributes = {};
                 const {attributes} = act.result;
                 const type = await selection.getAttribute('type');
@@ -1256,7 +1260,7 @@ const doActs = async (report, opts = {}) => {
                 act.result.success = true;
                 act.result.move = 'entered';
                 // If the input is a search input:
-                if (act.type === 'search') {
+                if (type === 'search') {
                   // Press the Enter key and wait for a network to be idle.
                   doAndWait(false);
                 }
@@ -1280,7 +1284,7 @@ const doActs = async (report, opts = {}) => {
             }
           }
           // Otherwise, if the act is a keypress:
-          else if (act.type === 'press') {
+          else if (type === 'press') {
             // Identify the number of times to press the key.
             let times = 1 + (act.again || 0);
             report.jobData.presses += times;
@@ -1296,7 +1300,7 @@ const doActs = async (report, opts = {}) => {
             };
           }
           // Otherwise, if it is a repetitive keyboard navigation:
-          else if (act.type === 'presses') {
+          else if (type === 'presses') {
             const {navKey, what, which, withItems} = act;
             const matchTexts = which ? which.map(text => debloat(text)) : [];
             // Initialize the loop variables.
@@ -1465,7 +1469,31 @@ const doActs = async (report, opts = {}) => {
   console.log('Acts completed');
   // If standardization is required:
   if (['also', 'only'].includes(standard)) {
-    console.log('>>>> Standardizing results of test acts');
+    // If granular reporting has been specified:
+    if (report.observe) {
+      // If a progress callback has been provided:
+      if (onProgress) {
+        // Notify the observer of the start of standardization.
+        try {
+          onProgress({
+            type: 'standardization',
+            which: 'start'
+          });
+          console.log(`${'Standardization started'} (observer notified)`);
+        }
+        catch (error) {
+          console.log(`${message} (observer notification failed: ${errorStart(error)})`);
+        }
+      }
+      // Otherwise, i.e. if no progress callback has been provided:
+      else {
+        // Notify the observer of the act and log it.
+        tellServer(report, messageParams, message);
+      }
+
+    // Notify the observer and log the start of standardization.
+    tellServer(report, '', 'Starting result standardization');
+    }
     const launchSpecActs = {};
     // For each act:
     report.acts.forEach((act, index) => {
@@ -1534,7 +1562,7 @@ const doActs = async (report, opts = {}) => {
 // Runs a job and returns a report.
 exports.doJob = async (job, opts = {}) => {
   // Make a report as a copy of the job.
-  let report = JSON.parse(JSON.stringify(job));
+  report = JSON.parse(JSON.stringify(job));
   const jobData = report.jobData = {};
   // Get whether the job is valid and, if not, why.
   const jobInvalidity = isValidJob(job);
@@ -1572,7 +1600,7 @@ exports.doJob = async (job, opts = {}) => {
         process.exit();
       }
     });
-    // Perform the acts and get a report.
+    // Perform the acts with any specified same-host observation options and get a report.
     report = await doActs(report, opts);
     // Add the end time and duration to the report.
     const endTime = new Date();
@@ -1594,3 +1622,48 @@ exports.doJob = async (job, opts = {}) => {
   // Return the report.
   return report;
 };
+
+// CLEANUP HANDLERS
+
+// Force-kills any Playwright browser processes synchronously.
+const forceKillBrowsers = () => {
+  if (cleanupInProgress) {
+    return;
+  }
+  cleanupInProgress = true;
+  try {
+    // Kill any Chromium headless shell processes.
+    execSync('pkill -9 -f "chromium_headless_shell.*headless_shell"', {stdio: 'ignore'});
+  }
+  catch(error) {}
+};
+// Force-kills any headless shell processes synchronously on process exit.
+process.on('exit', () => {
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on beforeExit.
+process.on('beforeExit', async () => {
+  if (!browserCloseIntentional) {
+    await browserClose();
+  }
+  forceKillBrowsers();
+});
+// Force-kills any headless shell processes synchronously on uncaught exceptions.
+process.on('uncaughtException', async error => {
+  console.error('Uncaught exception:', error);
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(1);
+});
+// Force-kills any headless shell processes synchronously on SIGINT.
+process.on('SIGINT', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});
+// Force-kills any headless shell processes synchronously on SIGTERM.
+process.on('SIGTERM', async () => {
+  await browserClose();
+  forceKillBrowsers();
+  process.exit(0);
+});

package/testaro/adbID.js

@@ -28,7 +28,7 @@
   Clean-room rule: report elements that reference aria-describedby targets that are missing or ambiguous (duplicate ids).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // elements that reference aria-describedby
@@ -63,5 +63,5 @@ exports.reporter = async (page, withItems) => {
     'Referenced description of the element is ambiguous or missing',
     'Referenced descriptions of elements are ambiguous or missing'
   ];
-  return await report(withItems, all, 'adbID', whats, 3);
+  return await getRuleResult(withItems, all, 'adbID', whats, 3);
 };

package/testaro/altScheme.js

@@ -28,7 +28,7 @@
   Identify img elements whose alt attribute is an entire URL or clearly a file name (favicon).
 */
 
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 exports.reporter = async (page, withItems) => {
   // Candidate images: any img with an alt attribute (including empty)
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems) => {
     'Element has an alt attribute with a URL as its entire value',
     'img elements have alt attributes with URLs as their entire values'
   ];
-  return await report(withItems, all, 'altScheme', whats, 2);
+  return await getRuleResult(withItems, all, 'altScheme', whats, 2);
 };

package/testaro/attVal.js

@@ -30,7 +30,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 
 // ########## FUNCTIONS
 
@@ -54,5 +54,5 @@ exports.reporter = async (page, withItems, attributeName, areLicit, values) => {
     `Element has attribute ${attributeName} with illicit value “__param__”`,
     `Elements have attribute ${attributeName} with illicit values`
   ];
-  return await report(withItems, all, 'attVal', whats, 2);
+  return await getRuleResult(withItems, all, 'attVal', whats, 2);
 };

package/testaro/autocomplete.js

@@ -30,7 +30,7 @@
 // ########## IMPORTS
 
 // Module to perform common operations.
-const {init, report} = require('../procs/testaro');
+const {init, getRuleResult} = require('../procs/testaro');
 // Module to get locator data.
 const {getLocatorData} = require('../procs/getLocatorData');
 
@@ -81,5 +81,5 @@ exports.reporter = async (
     'Input is missing an autocomplete attribute with value __param__',
     'Inputs are missing applicable autocomplete attributes'
   ];
-  return await report(withItems, all, 'autocomplete', whats, 2);
+  return await getRuleResult(withItems, all, 'autocomplete', whats, 2);
 };