testaro 76.2.1 → 77.0.0

package/CONTAINERS.md

@@ -14,7 +14,9 @@ Containerization is practical. The reference image runs every Testaro tool:
 
 ## Version coupling
 
-The tag of the base image in the `Dockerfile` (`mcr.microsoft.com/playwright:v1.60.0-noble`) must match the version of the `playwright` package in `package-lock.json`, so that the browsers baked into the image are the ones the installed Playwright expects. When upgrading Playwright, change both together.
+The tag of the base image in the `Dockerfile` (such as `mcr.microsoft.com/playwright:v1.62.0-noble`) must match the version of the `playwright` package in `package-lock.json`, so that the browsers baked into the image are the ones the installed Playwright expects. When you update dependencies, check the version of the `playwright` package (with `npm list playwright --depth=0`) and incorporate that version into the `Dockerfile`.
+
+However, to be sure that this works, check `https://mcr.microsoft.com/v2/playwright/tags/list` to determine whether an image for that version exists yet. If not and you want to run Testaro containerized, pin the version in `package.json` to the last version for which a base image exists and use that version in `Dockerfile`, too.
 
 ## The Chromium sandbox
 

package/Dockerfile

@@ -10,7 +10,7 @@
 # The base-image tag must match the version of the playwright package in
 # package-lock.json, so that the browsers baked into the image are the ones
 # the installed Playwright expects.
-FROM mcr.microsoft.com/playwright:v1.61.1-noble
+FROM mcr.microsoft.com/playwright:v1.62.0-noble
 
 # Install a headless Java runtime for the nuVnu test (vnu-jar); baking it in
 # prevents vnu-jar from downloading a Temurin runtime over the network on

package/README.md

@@ -209,18 +209,50 @@ The act types and their options are documented in the `etc` property of the [act
 
 ### Job as an object
 
+#### Simple execution
+
 An application can execute a job with:
 
 ```javascript
 const {doJob} = require('testaro/run');
+// Perform the job, which adds content to the job, making it a report.
 doJob(job)
 .then(report => {
-  // Perform the job and create a report.
+  // Optionally, modify the report.
   …
   return report;
 });
 ```
 
+#### Execution with observability
+
+If the application is capable of listening for events while a job is being executed, it is possible to do that by passing a second argument to `doJob`:
+
+```javascript
+const {doJob} = require('testaro/run');
+// Perform the job, which adds content to the job, making it a report.
+doJob(job, {
+  onProgress: event => {
+    // Handle the event. For example:
+    console.log(event);
+  }
+})
+.then(report => {
+  // Optionally, modify the report.
+  …
+  return report;
+});
+```
+
+Testaro emits events during job execution that allow an application to monitor progress. The events are:
+
+- `jobStart` - Emitted when a job starts.
+- `catalogStart` - Emitted when catalog compilation starts.
+- `catalogEnd` - Emitted when catalog compilation ends.
+- `actStart` - Emitted when an act starts.
+- `actEnd` - Emitted when an act ends.
+- `jobEnd` - Emitted when a job ends.
+
 ### Job as a file
 
 Jobs can be stored as JSON files in the `todo` subdirectory of a directory identified by the `JOBDIR` environment variable. After Testaro performs such a job, Testaro moves the job file to the `done` subdirectory of the same directory and saves the report in the `raw` subdirectory of the directory identified by the `REPORTDIR` environment variable.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "76.2.1",
+  "version": "77.0.0",
   "description": "Run 1300 web accessibility tests from 10 tools and get a standardized report",
   "main": "index.js",
   "scripts": {
@@ -29,7 +29,6 @@
     "url": "https://github.com/jrpool/testaro/issues"
   },
   "homepage": "https://github.com/jrpool/testaro#readme",
-  "//": "playwright is temporarily pinned to version 1.61.1 because mcr.microsoft.com/playwright:v1.62.0-noble has not yet been released, so containerized deployment requires version 1.61.1.",
   "dependencies": {
     "@qualweb/core": "*",
     "@qualweb/act-rules": "*",
@@ -45,7 +44,7 @@
     "axe-playwright": "*",
     "dotenv": "*",
     "pixelmatch": "*",
-    "playwright": "1.61.1",
+    "playwright": "*",
     "playwright-dompath": "*",
     "playwright-extra": "*",
     "playwright-extra-plugin-stealth": "*",

package/procs/doActs.js

@@ -227,7 +227,20 @@ const waitError = (page, act, error, what) => {
   return false;
 };
 // Performs the acts in a report and adds the results to the report.
-exports.doActs = async report => {
+exports.doActs = async (report, opts = {}) => {
+  // Fire-and-forget progress emitter (see #44), mirroring run.js: a handler
+  // exception is logged, never propagated, so a faulty consumer can never abort
+  // the job.
+  const emitProgress = event => {
+    if (opts && typeof opts.onProgress === 'function') {
+      try {
+        opts.onProgress(event);
+      }
+      catch (error) {
+        console.log(`ERROR in onProgress callback: ${error.message}`);
+      }
+    }
+  };
   // Make a temporary copy of the report. Precondition: report is valid.
   let tempReport = JSON.parse(JSON.stringify(report));
   let page = null;
@@ -247,6 +260,9 @@ exports.doActs = async report => {
     if (tempReport?.jobData && ! tempReport.jobData.aborted) {
       let act = acts[actIndex];
       const {type, which} = act;
+      // #44: signal act start so a consumer can heartbeat and advance a live
+      // per-act (per-tool, for test acts) view while the act runs.
+      emitProgress({event: 'actStart', index: Number(actIndex), type, which});
       const actSuffix = type === 'test' ? ` ${which}` : '';
       const message = `>>>> ${type}${actSuffix}`;
       // Log the act.
@@ -477,6 +493,19 @@ exports.doActs = async report => {
             act.expectationFailures = failureCount;
           }
         }
+        // #44: signal test-act completion with its outcome — timedOut is the
+        // most useful, since doActs already SIGKILLs a timed-out child — and its
+        // elapsed time, so consumers need not re-derive them from the report.
+        emitProgress({
+          event: 'actEnd',
+          index: Number(actIndex),
+          type,
+          which,
+          outcome: timedOut
+            ? 'timedOut'
+            : act.data && act.data.prevented ? 'prevented' : 'success',
+          elapsedMs: Date.now() - startTime
+        });
       }
       // Otherwise, if a current page exists:
       else if (page) {

package/run.js

@@ -76,6 +76,20 @@ const getTmpDirPath = async jobName => {
 };
 // Runs a job and returns a report.
 exports.doJob = async (job, opts = {}) => {
+  // Fire-and-forget progress emitter (see #44). A handler exception is logged,
+  // never propagated, so a faulty consumer can never abort the job. Payloads
+  // carry identifiers and timing only — no report content — and are delivered
+  // via the existing opts param, so doJob's signature is unchanged.
+  const emitProgress = event => {
+    if (opts && typeof opts.onProgress === 'function') {
+      try {
+        opts.onProgress(event);
+      }
+      catch (error) {
+        console.log(`ERROR in onProgress callback: ${error.message}`);
+      }
+    }
+  };
   // Initialize a report as a copy of the job.
   let report = JSON.parse(JSON.stringify(job));
   report.jobData ??= {};
@@ -94,6 +108,7 @@ exports.doJob = async (job, opts = {}) => {
   else {
     // Report this.
     console.log(`Starting job ${job.id} (${job.target.what})`);
+    emitProgress({event: 'jobStart', id: job.id});
     const tmpDir = await getTmpDirPath(job.id);
     // Add initialized job data to the report.
     const startTime = new Date();
@@ -128,8 +143,15 @@ exports.doJob = async (job, opts = {}) => {
     if (job.browserID && job.target && job.standard !== 'no') {
       // Initialize a catalog so it precedes any page images in the report.
       report.catalog = {};
+      // getCatalog() navigates to the target (procs/catalog.js) — which, on a
+      // slow, blocked, or dead target, can retry across browser types — before
+      // any act runs. Bracket it with events so a consumer can heartbeat through
+      // this phase instead of seeing the job fall silent until the first act.
+      emitProgress({event: 'catalogStart'});
+      const catalogStartMs = Date.now();
       // Add a catalog of the target, and a page image if required, to the report.
       report.catalog = await getCatalog(report);
+      emitProgress({event: 'catalogEnd', elapsedMs: Date.now() - catalogStartMs});
     }
     // Perform the acts and revise the report.
     report = await doActs(report, opts);
@@ -152,6 +174,9 @@ exports.doJob = async (job, opts = {}) => {
       report.jobData.toolTimes[item[0]] = item[1];
     });
   }
+  // Signal job completion (also fires for an invalid job, after its abort data
+  // is set) so a consumer can finalize regardless of outcome.
+  emitProgress({event: 'jobEnd', id: job.id, aborted: !! report.jobData.aborted});
   // Return the report.
   return report;
 };