testaro 74.2.3 → 75.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -0
- package/README.md +68 -61
- package/actSpecs-doc.md +25 -1
- package/actSpecs.js +6 -4
- package/package.json +2 -2
- package/procs/catalog.js +12 -1
- package/procs/dateTime.js +4 -2
- package/procs/doActs.js +15 -8
- package/procs/launch.js +0 -1
- package/procs/shoot.js +48 -43
- package/run.js +3 -1
- package/testaro/adbID.js +2 -2
- package/testaro/allCapStyle.js +2 -2
- package/testaro/allCaps.js +3 -3
- package/testaro/allHidden.js +2 -2
- package/testaro/allSlanted.js +2 -2
- package/testaro/altScheme.js +2 -2
- package/testaro/attVal.js +2 -2
- package/testaro/autocomplete.js +3 -2
- package/testaro/bulk.js +2 -2
- package/testaro/buttonMenu.js +3 -3
- package/testaro/captionLoc.js +2 -2
- package/testaro/datalistRef.js +2 -2
- package/testaro/distortion.js +2 -2
- package/testaro/docType.js +1 -1
- package/testaro/dupAtt.js +2 -2
- package/testaro/embAc.js +2 -2
- package/testaro/focAll.js +2 -2
- package/testaro/focAndOp.js +2 -2
- package/testaro/focInd.js +2 -2
- package/testaro/focVis.js +2 -2
- package/testaro/headEl.js +2 -2
- package/testaro/headingAmb.js +2 -2
- package/testaro/hovInd.js +2 -2
- package/testaro/hover.js +2 -2
- package/testaro/hr.js +2 -2
- package/testaro/imageLink.js +2 -2
- package/testaro/labClash.js +2 -2
- package/testaro/legendLoc.js +2 -2
- package/testaro/lineHeight.js +2 -2
- package/testaro/linkAmb.js +2 -2
- package/testaro/linkExt.js +2 -2
- package/testaro/linkOldAtt.js +2 -2
- package/testaro/linkTo.js +2 -2
- package/testaro/linkUl.js +2 -2
- package/testaro/miniText.js +2 -2
- package/testaro/motion.js +55 -40
- package/testaro/nonTable.js +2 -2
- package/testaro/optRoleSel.js +2 -2
- package/testaro/phOnly.js +2 -2
- package/testaro/pseudoP.js +2 -2
- package/testaro/radioSet.js +2 -2
- package/testaro/role.js +2 -2
- package/testaro/secHeading.js +2 -2
- package/testaro/styleDiff.js +2 -2
- package/testaro/tabNav.js +2 -2
- package/testaro/targetsNear.js +2 -2
- package/testaro/textNodes.js +1 -1
- package/testaro/textSem.js +2 -2
- package/testaro/titledEl.js +2 -2
- package/testaro/zIndex.js +2 -2
- package/tests/testaro.js +3 -30
- package/testaro/shoot0.js +0 -27
- package/testaro/shoot1.js +0 -33
package/LICENSE
CHANGED
|
@@ -2,6 +2,8 @@ MIT License
|
|
|
2
2
|
|
|
3
3
|
© 2021–2025 CVS Health and/or one of its affiliates. All rights reserved.
|
|
4
4
|
© 2025–2026 Jonathan Robert Pool.
|
|
5
|
+
© 2025 Juan S. Casado.
|
|
6
|
+
© 2026 Jeff Witt.
|
|
5
7
|
|
|
6
8
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
7
9
|
|
package/README.md
CHANGED
|
@@ -2,17 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
Ensemble testing for web accessibility
|
|
4
4
|
|
|
5
|
-
## Breaking change
|
|
5
|
+
## Breaking change notices
|
|
6
6
|
|
|
7
|
-
Version
|
|
7
|
+
Version 75.0.0 introduced a breaking change in the methods for making screenshots of web pages.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
Revision of this `README` document to reflect the latest version is in progress but is incomplete.
|
|
9
|
+
Version 68.0.0 introduced a breaking change in the format of reports.
|
|
12
10
|
|
|
13
11
|
## Purposes
|
|
14
12
|
|
|
15
|
-
Testaro is an application that performs ensemble testing of web pages,
|
|
13
|
+
Testaro is an application that performs ensemble testing of web pages for accessibility, usability, and conformity to HTML and CSS specifications.
|
|
16
14
|
|
|
17
15
|
The purposes of Testaro are to:
|
|
18
16
|
|
|
@@ -28,22 +26,22 @@ Testaro is described in two papers:
|
|
|
28
26
|
|
|
29
27
|
## Functionality
|
|
30
28
|
|
|
31
|
-
Testaro performs tasks defined by a _job_. Typically, a job identifies the URL of a web page and asks Testaro to call an ensemble of tools to test the page. Testaro adds the results of the testing to the job, thereby converting the job
|
|
29
|
+
Testaro performs tasks defined by a _job_. Typically, a job identifies the URL of a web page and asks Testaro to call an ensemble of tools to test the page. Testaro adds the results of the testing to the job, thereby converting the job to a _report_.
|
|
32
30
|
|
|
33
31
|
Testaro can be given a job to perform, in which case it performs the job, delivers the report, and quits.
|
|
34
32
|
|
|
35
33
|
Alternatively, testaro can run as a daemon, polling a server or a directory for new jobs and performing them when they are provided or when they appear in the directory.
|
|
36
34
|
|
|
37
|
-
A practical application that leverages Testaro will use other software to prepare jobs, schedule them, post-process the reports as needed, and manage the report files. Some utilities for such purposes can be found in the [Testilo project](https://www.npmjs.com/package/testilo). One application that leverages Testaro for a web service is [Kilotest](https://
|
|
35
|
+
A practical application that leverages Testaro will use other software to prepare jobs, schedule them, post-process the reports as needed, and manage the report files. Some utilities for such purposes can be found in the [Testilo project](https://www.npmjs.com/package/testilo). One application that leverages Testaro for a web service is [Kilotest](https://www.npmjs.com/package/@jrpool/kilotest).
|
|
38
36
|
|
|
39
37
|
## Dependencies
|
|
40
38
|
|
|
41
39
|
Testaro uses:
|
|
42
40
|
|
|
43
41
|
- [Playwright](https://playwright.dev/) to launch browsers, perform user actions in them, and perform tests
|
|
44
|
-
- [playwright-extra](https://www.npmjs.com/package/playwright-extra) and [puppeteer-extra-plugin-stealth](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) to make a Playwright-controlled browser more indistinguishable from a human-operated browser and thus make
|
|
42
|
+
- [playwright-extra](https://www.npmjs.com/package/playwright-extra) and [puppeteer-extra-plugin-stealth](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) to make a Playwright-controlled browser more indistinguishable from a human-operated browser and thus make its requests more likely to succeed
|
|
45
43
|
- [playwright-dompath](https://www.npmjs.com/package/playwright-dompath) to retrieve XPaths of elements
|
|
46
|
-
- [
|
|
44
|
+
- [pixelmatch](https://www.npmjs.com/package/pixelmatch) to measure motion
|
|
47
45
|
- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables
|
|
48
46
|
|
|
49
47
|
Testaro can perform tests of these _tools_:
|
|
@@ -68,16 +66,16 @@ As shown, Testaro is not only an integrator but also one of the integrated tools
|
|
|
68
66
|
The main concepts of Testaro are:
|
|
69
67
|
|
|
70
68
|
- `job`: a document that tells Testaro what to do.
|
|
71
|
-
- `act`: one step in a job
|
|
69
|
+
- `act`: one step in a job.
|
|
72
70
|
- `report`: a job that Testaro has added results to.
|
|
73
|
-
- `tool`: one of the testing applications in the ensemble
|
|
71
|
+
- `tool`: one of the testing applications in the ensemble assembled by Testaro.
|
|
74
72
|
- `rule`: a success or failure criterion defined by a tool (currently about 1300 across all tools).
|
|
75
73
|
- `test`: the software that a tool uses to apply a rule.
|
|
76
74
|
- `target`: a web page that a job tells Testaro to test.
|
|
77
|
-
- `result`: the information that Testaro adds to a job to describe the
|
|
78
|
-
- `native result`: the
|
|
79
|
-
- `standard result`: the
|
|
80
|
-
- `catalog`: a collection of data on the HTML elements relevant to one or more tests.
|
|
75
|
+
- `result`: the information that Testaro adds to a job to describe the outcomes of the tests of a tool.
|
|
76
|
+
- `native result`: the outcomes of the tests of a tool in exactly or approximately the original form.
|
|
77
|
+
- `standard result`: the outcomes of the tests of a tool in a uniform Testaro-defined form.
|
|
78
|
+
- `catalog`: a collection of data on the HTML elements of a target relevant to one or more tests.
|
|
81
79
|
|
|
82
80
|
## System requirements
|
|
83
81
|
|
|
@@ -99,7 +97,7 @@ One way to cope with this prohibition is to configure Playwright and Puppeteer t
|
|
|
99
97
|
- For the `qualWeb` tool, this is done in the Testaro `tests/qualweb.js` file, where the `qualWeb.start` method is called with an options argument. Its `args` array property is modified to include `'--no-sandbox'` and `'--disable-setuid-sandbox'`.
|
|
100
98
|
- The `ibm` tool, too, can launch a Puppeteer `chromium` browser, if page content instead of a Playwright page is passed to the `accessibilityChecker.getCompliance` method, or if the implementation of the tool is changed in the future. For anticipation of such a case, the Testaro `aceconfig.js` file is modified. That file defines a `module.exports` object with a `puppeteerArgs` property, and, `--no-sandbox` and `--disable-setuid-sandbox` are added to its array value.
|
|
101
99
|
|
|
102
|
-
Non-sandboxed browsers are less secure than sandboxed ones, particularly when there is no restriction on who can use Testaro and what web pages they can test with it.
|
|
100
|
+
Non-sandboxed browsers are less secure than sandboxed ones, particularly when there is no restriction on who can use Testaro and what targets (web pages) they can test with it.
|
|
103
101
|
|
|
104
102
|
### Option B
|
|
105
103
|
|
|
@@ -115,13 +113,11 @@ EOF
|
|
|
115
113
|
sudo sysctl --system
|
|
116
114
|
```
|
|
117
115
|
|
|
118
|
-
This
|
|
119
|
-
|
|
120
|
-
## Installation
|
|
116
|
+
This application implements option B.
|
|
121
117
|
|
|
122
|
-
|
|
118
|
+
## Installation an independent application
|
|
123
119
|
|
|
124
|
-
To install Testaro as an independent application, clone the [Testaro repository](https://github.com/jrpool/testaro). To ensure that the binary browsers of its Playwright dependency get installed, execute `(p)npx playwright install` after executing `(p)npm install`.
|
|
120
|
+
To install Testaro as an independent application, rather than a dependency, clone the [Testaro repository](https://github.com/jrpool/testaro). To ensure that the binary browsers of its Playwright dependency get installed, execute `(p)npx playwright install` after executing `(p)npm install`.
|
|
125
121
|
|
|
126
122
|
To update Testaro when it is an independent application, execute:
|
|
127
123
|
|
|
@@ -131,19 +127,13 @@ git pull
|
|
|
131
127
|
(p)npm run deps
|
|
132
128
|
```
|
|
133
129
|
|
|
134
|
-
### As a dependency
|
|
135
|
-
|
|
136
|
-
You can make `testaro` a dependency in another application. As noted at the beginning of this file, the entry in `package.json` should be `"testaro": "67.1.0"` if your application has not been designed to work with version 68.0.0 or later.
|
|
137
|
-
|
|
138
130
|
## Environment configuration
|
|
139
131
|
|
|
140
132
|
The `.env` file stores your decisions about the environment in which Testaro runs. The variables that can be defined there are documented in the `env.example` file.
|
|
141
133
|
|
|
142
134
|
## Jobs
|
|
143
135
|
|
|
144
|
-
Jobs tell Testaro what
|
|
145
|
-
|
|
146
|
-
### Job example
|
|
136
|
+
Jobs tell Testaro what to do.
|
|
147
137
|
|
|
148
138
|
Here is a sample job, showing properties that you can set:
|
|
149
139
|
|
|
@@ -152,7 +142,8 @@ Here is a sample job, showing properties that you can set:
|
|
|
152
142
|
id: 'healthcheck2611', // Job identifier
|
|
153
143
|
what: 'monthly health check', // Job description
|
|
154
144
|
strict: true, // Whether to reject redirections from the target URL
|
|
155
|
-
standard: '
|
|
145
|
+
standard: 'only', // Report native (no), standard (only), or both (also) results
|
|
146
|
+
imageColor: 0, // Color type (0, 2, 4, 6) of the page image, if one is to be created along with a catalog
|
|
156
147
|
device: { // Device to emulate
|
|
157
148
|
id: 'iPhone 8',
|
|
158
149
|
windowOptions: {
|
|
@@ -204,8 +195,8 @@ Here is a sample job, showing properties that you can set:
|
|
|
204
195
|
}
|
|
205
196
|
},
|
|
206
197
|
which: 'qualWeb',
|
|
207
|
-
withNewContent: false,
|
|
208
|
-
rules: ['QW-BP25', 'QW-BP26']
|
|
198
|
+
withNewContent: false, // An argument required by this tool
|
|
199
|
+
rules: ['QW-BP25', 'QW-BP26'] // Which rules of the tool to test for
|
|
209
200
|
}
|
|
210
201
|
]
|
|
211
202
|
}
|
|
@@ -213,11 +204,11 @@ Here is a sample job, showing properties that you can set:
|
|
|
213
204
|
|
|
214
205
|
The `device` property lets you choose among [about 125 devices recognized by Playwright](https://github.com/microsoft/playwright/blob/main/packages/playwright-core/src/server/deviceDescriptorsSource.json).
|
|
215
206
|
|
|
216
|
-
|
|
207
|
+
The act types and their options are documented in the `etc` property of the [actSpecs.js](actSpecs.js) object and in the [actSpecs-doc.md](actSpecs-doc.md) file.
|
|
217
208
|
|
|
218
|
-
|
|
209
|
+
## Running jobs
|
|
219
210
|
|
|
220
|
-
|
|
211
|
+
### Job as an object
|
|
221
212
|
|
|
222
213
|
An application can execute a job with:
|
|
223
214
|
|
|
@@ -231,35 +222,39 @@ doJob(job)
|
|
|
231
222
|
});
|
|
232
223
|
```
|
|
233
224
|
|
|
234
|
-
|
|
225
|
+
### Job as a file
|
|
226
|
+
|
|
227
|
+
Jobs can be stored as JSON files in the `todo` subdirectory of a directory identified by the `JOBDIR` environment variable. After Testaro performs such a job, Testaro moves the job file to the `done` subdirectory of the same directory and saves the report in the `raw` subdirectory of the directory identified by the `REPORTDIR` environment variable.
|
|
235
228
|
|
|
236
|
-
|
|
229
|
+
### Immediate performance
|
|
237
230
|
|
|
238
|
-
A user can make Testaro
|
|
231
|
+
A user can make Testaro perform a job from a file with a command like either of:
|
|
239
232
|
|
|
240
233
|
```bash
|
|
241
234
|
node call run
|
|
242
235
|
node call run 250725T
|
|
243
236
|
```
|
|
244
237
|
|
|
245
|
-
|
|
238
|
+
Testaro will find the first file in the `todo` subdirectory, or, if the second form of the command is used, the first file there whose name begins with the specified string.
|
|
239
|
+
|
|
240
|
+
### Directory polling
|
|
241
|
+
|
|
242
|
+
An application can poll the `todo` subdirectory for jobs with:
|
|
246
243
|
|
|
247
244
|
```javaScript
|
|
248
245
|
const {dirWatch} = require('testaro/dirWatch');
|
|
249
246
|
dirWatch(true, 300);
|
|
250
247
|
```
|
|
251
248
|
|
|
252
|
-
A user can make Testaro
|
|
249
|
+
A user can make Testaro start to poll that subdirectory with:
|
|
253
250
|
|
|
254
251
|
```javaScript
|
|
255
252
|
node call dirWatch true 300
|
|
256
253
|
```
|
|
257
254
|
|
|
258
|
-
In both cases, the first argument of `dirWatch` tells Testaro whether to continue
|
|
259
|
-
|
|
260
|
-
Testaro finds a file containing a job in the `todo` subdirectory of the `process.env.JOBDIR` directory and saves the report of that job in the `done/raw` subdirectory. In the `node call run` case, the job selected will be the first one whose file name begins with the argument of `run`, or the first one if that argument is absent.
|
|
255
|
+
In both cases, the first argument of `dirWatch` tells Testaro whether to continue polling after performing one job, and the second argument tells Testaro how many seconds to wait after not finding a job to perform, before polling again.
|
|
261
256
|
|
|
262
|
-
|
|
257
|
+
### Server polling
|
|
263
258
|
|
|
264
259
|
Testaro can poll a server for jobs to be performed. The server can act as the “controller” described in [How to run a thousand accessibility tests](https://medium.com/cvs-health-tech-blog/how-to-run-a-thousand-accessibility-tests-63692ad120c3). The server is responsible for preparing Testaro jobs, assigning them to Testaro agents, receiving reports back from those agents, and performing any further processing of the reports, including enhancement, storage, and disclosure to audiences. It can be any server reachable with a URL. That includes a server running on the same host as Testaro, with a URL such as `localhost:3000`.
|
|
265
260
|
|
|
@@ -273,7 +268,7 @@ The job request sent to the server can be a `POST` request, in which the `agentP
|
|
|
273
268
|
|
|
274
269
|
Testaro will send the report as a `POST` request whose payload is a JSON object with two properties: `agentPW` (the password) and `report` (the report). However, if the environment does not contain a password, the payload is a JSON object containing only the report.
|
|
275
270
|
|
|
276
|
-
An application can poll a server for jobs with:
|
|
271
|
+
An application can make Testaro poll a server for jobs with:
|
|
277
272
|
|
|
278
273
|
```javaScript
|
|
279
274
|
const {netWatch} = require('testaro/netWatch');
|
|
@@ -286,24 +281,27 @@ A user can make Testaro poll a server for jobs with:
|
|
|
286
281
|
node call netWatch true 300 true
|
|
287
282
|
```
|
|
288
283
|
|
|
289
|
-
The first argument of `netWatch` tells Testaro whether to continue polling after performing the first job. The second argument tells Testaro how many seconds to wait after receiving a no-jobs response. The third argument tells Testaro whether to be certificate-tolerant, i.e. to accept SSL certificates that fail verification against a list of certificate authorities (the default is `true`).
|
|
284
|
+
The first argument of `netWatch` tells Testaro whether to continue polling after performing the first job. The second argument tells Testaro how many seconds to wait after receiving a no-jobs response before polling again. The third argument tells Testaro whether to be certificate-tolerant, i.e. to accept SSL certificates that fail verification against a list of certificate authorities (the default is `true`).
|
|
290
285
|
|
|
291
286
|
## Reports
|
|
292
287
|
|
|
293
288
|
A report is a job with information about the results of the performance of the job inserted by Testaro into the job.
|
|
294
289
|
|
|
295
|
-
### Whole-job
|
|
290
|
+
### Whole-job data
|
|
296
291
|
|
|
297
|
-
As Testaro performs a job, information about the job as a whole is inserted into the job. That information is organized into one or
|
|
292
|
+
As Testaro performs a job, information about the job as a whole is inserted into the job. That information is organized into one, two, or three properties:
|
|
298
293
|
|
|
299
294
|
- `jobData`: Facts about the performance of the job
|
|
300
|
-
- `catalog`: A collection of data about the HTML elements
|
|
295
|
+
- `catalog`: A collection of data about the HTML elements of the target that are relevant to any test failures
|
|
296
|
+
- `images`: A collection of page images captured during the job
|
|
301
297
|
|
|
302
|
-
|
|
298
|
+
#### `jobData`
|
|
303
299
|
|
|
304
|
-
|
|
300
|
+
Testaro inserts the `jobData` property into every job.
|
|
305
301
|
|
|
306
|
-
|
|
302
|
+
#### `catalog`
|
|
303
|
+
|
|
304
|
+
Testaro inserts the `catalog` property only into jobs that instruct Testaro to produce standard results. The catalog is an inventory of HTML elements in the DOM of the target.
|
|
307
305
|
|
|
308
306
|
The `catalog` property has an object value. Here is an example:
|
|
309
307
|
|
|
@@ -330,9 +328,10 @@ The catalog is a mechanism for the integration of the tools. Most rule violation
|
|
|
330
328
|
Testaro uses the following techniques to make the tools calculate XPaths:
|
|
331
329
|
|
|
332
330
|
- `alfa` and `aslint`: They report XPaths, so Testaro needs only to normalize them.
|
|
333
|
-
- `ed11y`: Testaro adds it and a `window.getXPath` method to the page
|
|
331
|
+
- `ed11y`: Testaro adds it and a `window.getXPath` method to the page. When the tool reports an element, Testaro computes its XPath.
|
|
334
332
|
- `wave`: It reports a selector for each element; Testaro finds each element in the page via its selector and executes `window.getXPath` on the element.
|
|
335
|
-
- `
|
|
333
|
+
- `htmlcs`, `ibm`, `nuVal`, `nuVnu`, `qualWeb`: Testaro adds `data-xpath` attributes to all elements. The tools include code excerpts, with the `data-xpath` attributes, in the reported violations.
|
|
334
|
+
- `axe`: It reports a selector for each element, and Testaro adds `data-xpath` attributes to all elements. Testaro finds each element in the page via its selector and uses the `data-xpath` attribute. When this fails, Testaro uses the `data-xpath` attribute if its complete value is included in the reported `node.html` value.
|
|
336
335
|
- `testaro`: Testaro designs each of its own tests to report element XPaths.
|
|
337
336
|
|
|
338
337
|
By attaching a catalog entry to each reported element, Testaro allows an application that uses Testaro to tell users, for any particular HTML element, which tools ascribed violations of which rules to that element. An application could, for example, use a screenshot or a text-fragment link or could ask the user to paste the XPath into a browser developer tool.
|
|
@@ -343,14 +342,20 @@ In some cases no catalog entry can be found. The reasons may include:
|
|
|
343
342
|
- The element is inside a `noscript` element and therefore not considered an element in the DOM.
|
|
344
343
|
- The violation is not ascribed to a single element.
|
|
345
344
|
|
|
346
|
-
|
|
345
|
+
#### `images`
|
|
346
|
+
|
|
347
|
+
Testaro inserts an `images` array property if necessary to store page images in the report. If the job has an `imageColor` property with `0`, `2`, `4`, or `6` as its value and Testaro will insert a `catalog` property, then Testaro also creates a page image with that color type and makes its base64-encoded PNG the first item in the `images` array.
|
|
347
348
|
|
|
348
|
-
|
|
349
|
+
There is a `shoot` act type that can be used to make additional page images during a job.
|
|
350
|
+
|
|
351
|
+
### Act data
|
|
352
|
+
|
|
353
|
+
As Testaro performs the acts of a job, information about the result of each act is inserted into that act. For acts of type `test`, the added properties are:
|
|
349
354
|
|
|
350
355
|
- `startTime`: When Testaro began to perform the act
|
|
351
356
|
- `actualURL`: The tested URL (different from the target URL if the request was redirected)
|
|
352
357
|
- `data`: Data generated by the tool
|
|
353
|
-
- `result`:
|
|
358
|
+
- `result`: Result of the testing by the tool
|
|
354
359
|
|
|
355
360
|
The `result` property is an object with one or two (depending on the value of `standard`, as described above) subproperties:
|
|
356
361
|
|
|
@@ -453,7 +458,7 @@ In a previous version of the package, the tool operated on the page content when
|
|
|
453
458
|
|
|
454
459
|
### Nu Html Checker
|
|
455
460
|
|
|
456
|
-
The `nuVal` and `nuVnu` tools perform the tests of the Nu Html Checker.
|
|
461
|
+
The `nuVal` and `nuVnu` tools perform the tests of the Nu Html Checker. The `nuVal` tool is a remote service with an API. The `nuVnu` tool is installed as a dependency. A job can choose either one, or can try `nuVal` and if it fails then invoke `nuVnu`.
|
|
457
462
|
|
|
458
463
|
Its `rules` argument is **not** an array of rule IDs, but instead is an array of rule _specifications_. A rule specification for `nuVal` or `nuVnu` is a string with the format `=ruleID` or `~ruleID`. The `=` prefix indicates that the rule ID is invariable. The `~` prefix indicates that the rule ID is variable, in which case the `ruleID` part of the specification is a matching regular expression, rather than the exact text of a message. This `rules` format arises from the fact that `nuVal` and `nuVnu` generate customized messages and do not accompany them with rule identifiers.
|
|
459
464
|
|
|
@@ -482,7 +487,7 @@ Thus, when the `rules` argument is omitted, QualWeb will test for all of the rul
|
|
|
482
487
|
|
|
483
488
|
The target can be provided to QualWeb either as HTML or as a URL. Experience indicates that the results can differ between these methods, with each method reporting some rule violations or some instances that the other method does not report. For at least some cases, more rules are reported violated when HTML is provided (`withNewItems: false`).
|
|
484
489
|
|
|
485
|
-
QualWeb creates sandboxed Puppeteer pages to perform its tests on. Therefore, the host must permit sandboxed browsers to be launched. See the discussion above about browser security.
|
|
490
|
+
QualWeb creates sandboxed Puppeteer pages to perform its tests on. Therefore, the host must permit sandboxed browsers to be launched. See the discussion above about browser security.
|
|
486
491
|
|
|
487
492
|
### Testaro
|
|
488
493
|
|
|
@@ -545,10 +550,12 @@ The Playwright “Receives Events” actionability check does **not** check whet
|
|
|
545
550
|
|
|
546
551
|
### Test prevention
|
|
547
552
|
|
|
548
|
-
Test targets employ mechanisms to prevent scraping, multiple requests within a short time, automated form submission, and other automated actions. These mechanisms may interfere with testing. When a test act is prevented
|
|
553
|
+
Test targets employ mechanisms to prevent scraping, multiple requests within a short time, automated form submission, and other automated actions. These mechanisms may interfere with testing. When a test act is prevented, Testaro reports this prevention.
|
|
549
554
|
|
|
550
555
|
Some targets prohibit the execution of alien scripts unless the client can demonstrate that it is the requester of the page. Failure to provide that evidence results in the script being blocked and an error message being logged, saying “Refused to execute a script because its hash, its nonce, or unsafe-inline does not appear in the script-src directive of the Content Security Policy”. This mechanism affects tools that insert scripts into a target in order to test it. To comply with this requirement, Testaro obtains a _nonce_ from the response that serves the target. Then the file that runs the tool adds that nonce to the script as the value of a `nonce` attribute when it inserts its script into the target.
|
|
551
556
|
|
|
557
|
+
Some targets have been found erratically to prevent the creation of page images. When page images have been created, during the `motion` test in `testaro` some targets have been found to prevent their comparison by BlazeDiff, but comparison by `pixelmatch` has succeeded. For this reason, although reportedly slower, `pixelmatch` is the library used for image comparison.
|
|
558
|
+
|
|
552
559
|
### Tool duplicativity
|
|
553
560
|
|
|
554
561
|
Tools sometimes do redundant testing, in that two or more tools test for the same defects, although such duplications are not necessarily perfect. This fact creates problems:
|
|
@@ -615,7 +622,7 @@ From 12 February 2024 through 30 September 2025, contributors of code to Testaro
|
|
|
615
622
|
|
|
616
623
|
## Future work
|
|
617
624
|
|
|
618
|
-
Future work
|
|
625
|
+
Future work contemplated for this project is described in its [issues](https://github.com/jrpool/testaro/issues) and also discussed in the [UPGRADES.md](UPGRADES.md) file.
|
|
619
626
|
|
|
620
627
|
## Etymology
|
|
621
628
|
|
package/actSpecs-doc.md
CHANGED
|
@@ -30,7 +30,10 @@ link: [
|
|
|
30
30
|
]
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
-
The rule is an array with two elements:
|
|
33
|
+
The rule is an array with two elements:
|
|
34
|
+
|
|
35
|
+
- a string ('Click a link' in this case) describing the act
|
|
36
|
+
- an object containing requirements for any act of the type identified by the key (`link` in this case).
|
|
34
37
|
|
|
35
38
|
The requirement `which: [true, 'string', 'hasLength', 'substring of the link text']` specifies what is required for the `which` property of a `link`-type act. The requirement is an array.
|
|
36
39
|
|
|
@@ -52,6 +55,27 @@ The validity criterion named in item 2 may be any of these:
|
|
|
52
55
|
- `'isWaitable'`: is `'url'`, `'title'`, or `'body'`
|
|
53
56
|
- `'areStrings'`: is an array of strings
|
|
54
57
|
|
|
58
|
+
## testaro tool
|
|
59
|
+
|
|
60
|
+
The `tools.testaro` object has an `args` property specifying that a `testaro` test act may include an `args` property with an object value.
|
|
61
|
+
|
|
62
|
+
If it does, the property names of the object value must be `testaro` rule IDs. Any property value must be an array of the positional arguments to be concatenated to the four default arguments (`page`, `report`, `actIndex`, and `withItems`) in the signature of the `reporter` function of each `testaro` rule.
|
|
63
|
+
|
|
64
|
+
The rules of `testaro` that accept additional arguments are `autocomplete`, `buttonMenu`, `focInd`, and `hover`.
|
|
65
|
+
|
|
66
|
+
## Screenshots
|
|
67
|
+
|
|
68
|
+
An act of type `shoot` creates a full-page screenshot and converts it to a base64 encoding of a PNG image. With its arguments, you can decide:
|
|
69
|
+
|
|
70
|
+
- Whether to mask certain elements (using a CSS selector)
|
|
71
|
+
- The color type (grayscale, RGB, grayscale alpha, or RGBA)
|
|
72
|
+
- What action to take to dispose of the base64 string:
|
|
73
|
+
- return it to the `shoot` act and save it in the result of the act (`return`)
|
|
74
|
+
- concatenate it to an `images` array in the report and return its index in the array (`report`)
|
|
75
|
+
- save it as a file in the temporary directory and return the file path (`file`)
|
|
76
|
+
|
|
77
|
+
Both the `return` and the `report` actions save the base64 string in the report, but in different places. The `file` action would permit you to create a custom act that retrieves and uses the image without the image being added to the report.
|
|
78
|
+
|
|
55
79
|
## License
|
|
56
80
|
|
|
57
81
|
© 2026 Jonathan Robert Pool.
|
package/actSpecs.js
CHANGED
|
@@ -45,6 +45,7 @@ exports.actSpecs = {
|
|
|
45
45
|
{
|
|
46
46
|
target: [false, 'object', '', 'target different from target of the job'],
|
|
47
47
|
browserID: [false, 'string', 'isBrowserID', 'browser type different from browserID of the job'],
|
|
48
|
+
shoot: [false, 'object', '', 'if screenshot to be made, exclusionSelector, colorType, and action'],
|
|
48
49
|
what: [false, 'string', 'hasLength', 'comment']
|
|
49
50
|
}
|
|
50
51
|
],
|
|
@@ -123,10 +124,11 @@ exports.actSpecs = {
|
|
|
123
124
|
}
|
|
124
125
|
],
|
|
125
126
|
shoot: [
|
|
126
|
-
'
|
|
127
|
+
'Create and dispose of a full-page screenshot as a base-64-encoded PNG',
|
|
127
128
|
{
|
|
128
|
-
|
|
129
|
-
|
|
129
|
+
exclusionSelector: [false, 'string', 'hasLength', 'CSS selector for an element to mask'],
|
|
130
|
+
colorType: [false, 'number', '', '0=grayscale, 2=RGB, 4=grayscale alpha, 6=RGBA'],
|
|
131
|
+
action: [true, 'string', 'hasLength', 'disposition: return, report, file'],
|
|
130
132
|
what: [false, 'string', 'hasLength', 'comment']
|
|
131
133
|
}
|
|
132
134
|
],
|
|
@@ -208,7 +210,7 @@ exports.actSpecs = {
|
|
|
208
210
|
{
|
|
209
211
|
withItems: [true, 'boolean', '', 'itemize'],
|
|
210
212
|
stopOnFail: [true, 'boolean', '', 'whether testing is to stop after first failure'],
|
|
211
|
-
args: [false, 'object', 'areArrays', 'extra
|
|
213
|
+
args: [false, 'object', 'areArrays', 'extra arguments of rules taking any']
|
|
212
214
|
}
|
|
213
215
|
],
|
|
214
216
|
wave: [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testaro",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "75.1.0",
|
|
4
4
|
"description": "Run 1300 web accessibility tests from 10 tools and get a standardized report",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -30,7 +30,6 @@
|
|
|
30
30
|
},
|
|
31
31
|
"homepage": "https://github.com/jrpool/testaro#readme",
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@blazediff/core": "*",
|
|
34
33
|
"@qualweb/core": "*",
|
|
35
34
|
"@qualweb/act-rules": "*",
|
|
36
35
|
"@qualweb/wcag-techniques": "*",
|
|
@@ -43,6 +42,7 @@
|
|
|
43
42
|
"aslint-testaro": "*",
|
|
44
43
|
"axe-playwright": "*",
|
|
45
44
|
"dotenv": "*",
|
|
45
|
+
"pixelmatch": "*",
|
|
46
46
|
"playwright": "*",
|
|
47
47
|
"playwright-dompath": "*",
|
|
48
48
|
"playwright-extra": "*",
|
package/procs/catalog.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
|
|
26
26
|
// Module to close and launch browsers.
|
|
27
27
|
const {browserClose, launch} = require('./launch');
|
|
28
|
-
const {
|
|
28
|
+
const {shoot} = require('./shoot');
|
|
29
29
|
|
|
30
30
|
// FUNCTIONS
|
|
31
31
|
|
|
@@ -44,7 +44,18 @@ exports.getCatalog = async report => {
|
|
|
44
44
|
});
|
|
45
45
|
// If the launch and navigation succeeded:
|
|
46
46
|
if (page) {
|
|
47
|
+
// If a page image is required:
|
|
48
|
+
if ([0, 2, 4, 6].includes(report.imageColor)) {
|
|
49
|
+
// Create one and add it to the report.
|
|
50
|
+
console.log('Creating page image');
|
|
51
|
+
await shoot(page, report, {
|
|
52
|
+
exclusionSelector: '',
|
|
53
|
+
colorType: report.imageColor,
|
|
54
|
+
action: 'report'
|
|
55
|
+
});
|
|
56
|
+
}
|
|
47
57
|
// Get a catalog of the elements in the page.
|
|
58
|
+
console.log('Creating catalog');
|
|
48
59
|
const catalog = await page.evaluate(() => {
|
|
49
60
|
const elements = Array.from(document.querySelectorAll('*'));
|
|
50
61
|
// Initialize a catalog.
|
package/procs/dateTime.js
CHANGED
|
@@ -32,5 +32,7 @@ exports.dateOf = timeStamp => {
|
|
|
32
32
|
return null;
|
|
33
33
|
}
|
|
34
34
|
};
|
|
35
|
-
// Returns a
|
|
36
|
-
exports.nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
35
|
+
// Returns a human-friendly representation of the current date and time.
|
|
36
|
+
const nowString = exports.nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
37
|
+
// Returns a compact representation of the current date and time.
|
|
38
|
+
exports.nowStamp = () => nowString().replace(/[:-]/g, '');
|
package/procs/doActs.js
CHANGED
|
@@ -289,13 +289,14 @@ exports.doActs = async report => {
|
|
|
289
289
|
}
|
|
290
290
|
// Otherwise, if the act is a launch:
|
|
291
291
|
else if (type === 'launch') {
|
|
292
|
-
// Launch a browser, navigate
|
|
292
|
+
// Launch a browser, navigate, optionally make a screenshot, and add the result to the act.
|
|
293
293
|
page = await launch({
|
|
294
|
-
tempReport,
|
|
294
|
+
report: tempReport,
|
|
295
295
|
actIndex,
|
|
296
296
|
tempBrowserID: getActBrowserID(tempReport, actIndex),
|
|
297
297
|
tempURL: getActTargetURL(tempReport, actIndex),
|
|
298
|
-
xPathNeed: 'none'
|
|
298
|
+
xPathNeed: 'none',
|
|
299
|
+
shoot: act.shoot
|
|
299
300
|
});
|
|
300
301
|
// If this failed:
|
|
301
302
|
if (! page) {
|
|
@@ -641,11 +642,17 @@ exports.doActs = async report => {
|
|
|
641
642
|
}
|
|
642
643
|
// Otherwise, if the act is a screenshot:
|
|
643
644
|
else if (type === 'shoot') {
|
|
644
|
-
const
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
?
|
|
648
|
-
:
|
|
645
|
+
const {exclusionSelector, colorType, action} = act;
|
|
646
|
+
// Make and dispose of a full-page screenshot.
|
|
647
|
+
const shotInfo = await shoot(page, tempReport, {
|
|
648
|
+
exclusion: exclusionSelector ? page.locator(exclusionSelector) : null,
|
|
649
|
+
colorType: [0, 2, 4, 6].includes(colorType) ? colorType : null,
|
|
650
|
+
action: ['return', 'report', 'file'].includes(action) ? action : 'report'
|
|
651
|
+
});
|
|
652
|
+
// Add the PNG base-64 encoding, image index, or file path to the act result.
|
|
653
|
+
act.result = shotInfo !== ''
|
|
654
|
+
? {success: true, shotInfo}
|
|
655
|
+
: {success: false, prevented: true};
|
|
649
656
|
}
|
|
650
657
|
// Otherwise, if the act is a move:
|
|
651
658
|
else if (moves[type]) {
|