testilo 24.3.1 → 25.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +213 -183
- package/call.js +43 -6
- package/difgest.js +17 -0
- package/package.json +1 -1
- package/procs/compare/cpA11yMessage/index.js +2 -2
- package/procs/compare/cpProductPrice/index.js +2 -2
- package/procs/compare/{tcp39 → tcp40}/index.html +2 -2
- package/procs/compare/{tcp39 → tcp40}/index.js +2 -5
- package/procs/difgest/tfp40/index.html +56 -0
- package/procs/difgest/tfp40/index.js +109 -0
- package/procs/digest/dpProductPrice/index.js +2 -2
- package/procs/digest/dpa11yMessage/index.js +2 -3
- package/procs/digest/tdp40/index.html +2 -2
- package/procs/digest/tdp40/index.js +7 -5
- package/procs/score/tic40.js +5 -0
- package/procs/score/tsp40.js +1 -0
- package/procs/util.js +9 -0
package/README.md
CHANGED
|
@@ -33,21 +33,16 @@ Environment variables for Testilo can be specified in a `.env` file. An example:
|
|
|
33
33
|
|
|
34
34
|
```bash
|
|
35
35
|
FUNCTIONDIR=./procs
|
|
36
|
+
SPECDIR=../testdir/specs
|
|
36
37
|
JOBDIR=../testdir/jobs
|
|
37
38
|
REPORTDIR=../testdir/reports
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
SCORED_REPORT_URL=../scored/__id__.json
|
|
40
|
+
DIGEST_URL=../digested/__id__.html
|
|
40
41
|
```
|
|
41
42
|
|
|
42
|
-
The
|
|
43
|
-
|
|
44
|
-
`JOBDIR` references a directory in the filesystem where jobs created by the `merge` proc are to be saved.
|
|
45
|
-
|
|
46
|
-
`REPORTDIR` references a directory in the filesystem where reports are saved.
|
|
47
|
-
|
|
48
|
-
`REQUESTER` is an email address that will be used as a job property if no other email address is specified for the `sources.requester` property of the job.
|
|
43
|
+
The first four variables above tell Testilo where to find or save files when a user invokes a function. This decreases the count of arguments that the user would otherwise need to specify.
|
|
49
44
|
|
|
50
|
-
|
|
45
|
+
The other two variables specify the destinations of links to scored reports and digests. Digests produced by Testilo digesters link to scored reports, and difgests produced by Testilo difgesters link to digests. The URLs can be absolute, or, if digests and difgests will be opened from a local filesystem, they can be relative to the linking document, as shown. They include the substring `__id__`. A function that needs the URL of a scored report or digest is expected to substitute the ID of that document for `__id__` to produce the URL. Since the `.env` file is excluded from the repository, importing modules that will use `digest()` need a `SCORED_REPORT_URL` environment variable, and importing modules that will use `difgest()` need a `DIGEST_URL` environment variable.
|
|
51
46
|
|
|
52
47
|
## Job preparation
|
|
53
48
|
|
|
@@ -195,7 +190,7 @@ As shown in this example, when a browser is launched by placeholder substitution
|
|
|
195
190
|
|
|
196
191
|
### Target list to batch
|
|
197
192
|
|
|
198
|
-
If you have a target list, the `batch` module of Testilo can convert it to a batch. The batch will contain, for each target, one array of acts named `main`, containing a `launch` act (depending on the script to specify the browser type and the target to specify the URL).
|
|
193
|
+
If you have a target list, the `batch` module of Testilo can convert it to a simple batch. The batch will contain, for each target, only one array of acts, named `main`, containing only a `launch` act (depending on the script to specify the browser type and the target to specify the URL).
|
|
199
194
|
|
|
200
195
|
#### Invocation
|
|
201
196
|
|
|
@@ -203,41 +198,45 @@ There are two ways to use the `batch` module.
|
|
|
203
198
|
|
|
204
199
|
##### By a module
|
|
205
200
|
|
|
206
|
-
A module can invoke `batch` in this way:
|
|
201
|
+
A module can invoke `batch()` in this way:
|
|
207
202
|
|
|
208
203
|
```javaScript
|
|
209
204
|
const {batch} = require('testilo/batch');
|
|
205
|
+
const id = 'divns';
|
|
206
|
+
const what = 'divisions';
|
|
207
|
+
const targets = [
|
|
208
|
+
['Energy', 'https://abc.com/energy'],
|
|
209
|
+
['Water', 'https://abc.com/water']
|
|
210
|
+
];
|
|
210
211
|
const batchObj = batch(id, what, targets);
|
|
211
212
|
```
|
|
212
213
|
|
|
213
|
-
|
|
214
|
+
The `id` argument to `batch()` is a unique identifier for the target list. The `what` variable describes the target list. The `targets` variable is an array of arrays, with each array containing the 2 items (description and URL) defining one target.
|
|
214
215
|
|
|
215
|
-
The `batch()` function of the `batch` module generates a batch and returns it as an object.
|
|
216
|
+
The `batch()` function of the `batch` module generates a batch and returns it as an object. Within the batch, each target is given a sequential (base-62 alphanumeric) string as an ID.
|
|
216
217
|
|
|
217
|
-
The
|
|
218
|
+
The invoking module can further dispose of the batch as needed.
|
|
218
219
|
|
|
219
220
|
##### By a user
|
|
220
221
|
|
|
221
|
-
A user can invoke `batch` in this way:
|
|
222
|
+
A user can invoke `batch()` in this way:
|
|
222
223
|
|
|
223
224
|
- Create a target list and save it as a text file (with tab-delimited items in newline-delimited lines) in the `targetLists` subdirectory of the `SPECDIR` directory. Name the file `x.tsv`, where `x` is the list ID.
|
|
224
225
|
- In the Testilo project directory, execute the statement `node call batch id what`.
|
|
225
226
|
|
|
226
|
-
In this statement, replace `id` with the list ID and `what` with a string describing the batch.
|
|
227
|
+
In this statement, replace `id` with the list ID and `what` with a string describing the batch. Example: `node call batch divns 'ABC company divisions'`.
|
|
227
228
|
|
|
228
229
|
The `call` module will retrieve the named target list.
|
|
229
230
|
The `batch` module will convert the target list to a batch.
|
|
230
|
-
The `call` module will save the batch as a JSON file in the `batches` subdirectory of the `SPECDIR` directory.
|
|
231
|
+
The `call` module will save the batch as a JSON file named `x.json` (replacing `x` with the list ID) in the `batches` subdirectory of the `SPECDIR` directory.
|
|
231
232
|
|
|
232
233
|
### Issues to script
|
|
233
234
|
|
|
234
|
-
|
|
235
|
+
You can use the `script()` function of the `script` module to simplify the creation of scripts.
|
|
235
236
|
|
|
236
|
-
|
|
237
|
+
In its simplest form, `script()` requires only one argument, a string that will serve as the ID of the script. Called in this way, `script()` produces a script that tells Testaro to perform all of the tests defined by all of the tools integrated by Testaro.
|
|
237
238
|
|
|
238
|
-
If you want
|
|
239
|
-
|
|
240
|
-
If you want Testaro to test targets for **all** the rules of all the available tools, you can use the `script` module to create a script that does not impose any issue restrictions.
|
|
239
|
+
If you want a more focused script, you can add additional arguments to `script()`. First you must add the ID of a Testilo issue classification (such as `tic99`). After that, you must add one or more arguments giving the IDs of issues in that classification. The latest Testilo issue classification classifies about a thousand rules of the 10 Testaro tools into about 300 _issues_. The classification is found in a file whose name begins with `tic` in the `procs/score` directory. For example, one issue in the `tic40.js` file is `mainNot1`. Four rules are classified as belonging to that issue: rule `main_element_only_one` of the `aslint` tool and 3 more rules defined by 3 other tools. You can also create custom classifications and save them in a `score` subdirectory of the `FUNCTIONDIR` directory.
|
|
241
240
|
|
|
242
241
|
#### Invocation
|
|
243
242
|
|
|
@@ -249,55 +248,103 @@ A module can invoke `script` in this way:
|
|
|
249
248
|
|
|
250
249
|
```javaScript
|
|
251
250
|
const {script} = require('testilo/script');
|
|
252
|
-
const
|
|
251
|
+
const {issues} = require('testilo/procs/score/tic99');
|
|
252
|
+
const scriptObj = script('monthly', issues, 'regionNoText', 'mainNot1');
|
|
253
253
|
```
|
|
254
254
|
|
|
255
|
-
|
|
256
|
-
- The `scriptID` variable specifies the ID that the script will have.
|
|
257
|
-
- The `issues` variable (if present) is an object that classifies issues, such as the `issues` object in a `tic` file.
|
|
258
|
-
- The `issueID` variables (if any) are strings, such as `'regionNoText'`, that name issues, i.e. properties of the `issues` object, that you want jobs from the script to test for.
|
|
259
|
-
|
|
260
|
-
The `script()` function of the `script` module generates a script and returns it as an object. The invoking module can further modify and use the script as needed.
|
|
255
|
+
In this example, the script will have `'monthly'` as its ID. It will tell Testaro to test for all, and only, the rules that are classified into either the `regionNoText` or the `mainNot1` issue.
|
|
261
256
|
|
|
262
|
-
|
|
257
|
+
The invoking module can further modify and use the script (`scriptObj`) as needed.
|
|
263
258
|
|
|
264
|
-
|
|
265
|
-
const {script} = require('testilo/script');
|
|
266
|
-
const scriptObj = script(scriptID);
|
|
267
|
-
```
|
|
259
|
+
To create a script **without** issue restrictions, a module can call `script()` with only the first (ID) argument.
|
|
268
260
|
|
|
269
261
|
##### By a user
|
|
270
262
|
|
|
271
|
-
A user can invoke `script`
|
|
263
|
+
A user can invoke `script()` by executing one of these statements in the Testilo project directory:
|
|
272
264
|
|
|
273
|
-
|
|
274
|
-
|
|
265
|
+
```javascript
|
|
266
|
+
node call script id ticnn issuea issueb …
|
|
267
|
+
node call script id
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
In this statement, replace `id` with an ID for the script, such as `headings`.
|
|
271
|
+
|
|
272
|
+
If specifying issues:
|
|
275
273
|
- Replace `ticnn` with the base, such as `tic99`, of the name of an issue classification file in the `score` subdirectory of the `FUNCTIONDIR` directory.
|
|
276
274
|
- Replace the remaining arguments (`issuea` etc.) with issue names from that classification file.
|
|
277
275
|
|
|
278
|
-
The `call` module will retrieve the named classification.
|
|
276
|
+
The `call` module will retrieve the named classification, if any.
|
|
279
277
|
The `script` module will create a script.
|
|
280
278
|
The `call` module will save the script as a JSON file in the `scripts` subdirectory of the `SPECDIR` directory.
|
|
281
279
|
|
|
282
|
-
To create a script without any issue restrictions, a user can execute the statement `node call script id`.
|
|
283
|
-
|
|
284
280
|
#### Options
|
|
285
281
|
|
|
286
282
|
The `script` module will use the value of the `SEND_REPORT_TO` environment variable as the value of the `sendReportTo` property of the script, if that variable exists, and otherwise will leave that property with an empty-string value.
|
|
287
283
|
|
|
288
|
-
When the `script` module creates a script for you, it does not ask you for all of the options that the script may require. Instead, it chooses default options. After you invoke `script`, you can edit the script that it creates to revise options.
|
|
284
|
+
When the `script` module creates a script for you, it does not ask you for all of the options that the script may require. Instead, it chooses default options. For example, it sets the values of `isolate` and `strict` to `true`. After you invoke `script`, you can edit the script that it creates to revise options.
|
|
289
285
|
|
|
290
286
|
### Merge
|
|
291
287
|
|
|
292
|
-
Testilo merges batches with scripts, producing jobs, by means of the `merge` module.
|
|
288
|
+
Testilo merges batches with scripts, producing Testaro jobs, by means of the `merge` module.
|
|
289
|
+
|
|
290
|
+
#### Invocation
|
|
291
|
+
|
|
292
|
+
There are two ways to use the `merge` module.
|
|
293
|
+
|
|
294
|
+
##### By a module
|
|
295
|
+
|
|
296
|
+
A module can invoke `merge` in this way:
|
|
297
|
+
|
|
298
|
+
```javaScript
|
|
299
|
+
const {merge} = require('testilo/merge');
|
|
300
|
+
const script = …;
|
|
301
|
+
const batch = …;
|
|
302
|
+
const standard = 'only';
|
|
303
|
+
const observe = false;
|
|
304
|
+
const requester = 'me@mydomain.tld';
|
|
305
|
+
const timeStamp = '241215T1200';
|
|
306
|
+
const jobs = merge(script, batch, standard, observe, requester, timeStamp);
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
The first two arguments are a script and a batch obtained from files or from prior calls to `script()` and `batch()`.
|
|
310
|
+
|
|
311
|
+
The `standard` argument specifies how to handle standardization. If `also`, jobs will tell Testaro to include in its reports both the original results of the tests of tools and the Testaro-standardized results. If `only`, reports are to include only the standardized test results. If `no`, reports are to include only the original results, without standardization.
|
|
312
|
+
|
|
313
|
+
The `observe` argument tells Testaro whether the jobs should allow granular observation. If `false`, Testaro will not report job progress, but will only send reports to the server when the reports are completed. It is generally user-friendly to allow granular observation, and for user applications to implement it, if they make users wait while jobs are assigned and performed, since that process typically takes a few minutes.
|
|
314
|
+
|
|
315
|
+
The `timeStamp` argument specifies the earliest UTC date and time when the jobs may be assigned, or it may be an empty string if now.
|
|
316
|
+
|
|
317
|
+
The `merge()` function returns the jobs in an array. The invoking module can further dispose of the jobs as needed.
|
|
318
|
+
|
|
319
|
+
##### By a user
|
|
320
|
+
|
|
321
|
+
A user can invoke `merge()` in this way:
|
|
322
|
+
|
|
323
|
+
- Create a script and save it as a JSON file in the `scripts` subdirectory of the `SPECDIR` directory.
|
|
324
|
+
- Create a batch and save it as a JSON file in the `batches` subdirectory of the `SPECDIR` directory.
|
|
325
|
+
- In the Testilo project directory, execute the statement:
|
|
326
|
+
|
|
327
|
+
```javascript
|
|
328
|
+
node call merge scriptID batchID standard observe requester timeStamp todoDir
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
In this statement, replace:
|
|
332
|
+
- `scriptID` with the ID (which is also the base of the file name) of the script.
|
|
333
|
+
- `batchID` with the ID (which is also the base of the file name) of the batch.
|
|
334
|
+
- `standard`, `observe`, `requester`, and `timeStamp` as described above.
|
|
335
|
+
- `todoDir`: `true` if the jobs are to be saved in the `todo` subdirectory, or `false` if they are to be saved in the `pending` subdirectory, of the `JOBDIR` directory.
|
|
336
|
+
|
|
337
|
+
The `call` module will retrieve the named script and batch from their respective directories.
|
|
338
|
+
The `merge` module will create an array of jobs.
|
|
339
|
+
The `call` module will save the jobs as JSON files in the `todo` or `pending` subdirectory of the `JOBDIR` directory.
|
|
293
340
|
|
|
294
341
|
#### Output
|
|
295
342
|
|
|
296
|
-
|
|
343
|
+
A Testaro job produced by `merge` may look like this:
|
|
297
344
|
|
|
298
345
|
```javaScript
|
|
299
346
|
{
|
|
300
|
-
id: '240115T1200-
|
|
347
|
+
id: '240115T1200-4R-0',
|
|
301
348
|
what: 'aside mislocation',
|
|
302
349
|
strict: true,
|
|
303
350
|
timeLimit: 60,
|
|
@@ -308,29 +355,9 @@ Suppose you ask for a merger of the above batch and script. Then the first job p
|
|
|
308
355
|
acts: [
|
|
309
356
|
{
|
|
310
357
|
type: 'launch',
|
|
311
|
-
which: '
|
|
312
|
-
what: 'Acme Clothes
|
|
313
|
-
url: 'https://acmeclothes.com/
|
|
314
|
-
},
|
|
315
|
-
{
|
|
316
|
-
type: 'text',
|
|
317
|
-
which: 'User Name',
|
|
318
|
-
what: 'tester34'
|
|
319
|
-
},
|
|
320
|
-
{
|
|
321
|
-
type: 'text',
|
|
322
|
-
which: 'Password',
|
|
323
|
-
what: '34SecretTester'
|
|
324
|
-
},
|
|
325
|
-
{
|
|
326
|
-
type: 'button',
|
|
327
|
-
which: 'Submit',
|
|
328
|
-
what: 'submit the login form'
|
|
329
|
-
},
|
|
330
|
-
{
|
|
331
|
-
type: 'wait',
|
|
332
|
-
which: 'title',
|
|
333
|
-
what: 'account'
|
|
358
|
+
which: 'webkit',
|
|
359
|
+
what: 'Acme Clothes',
|
|
360
|
+
url: 'https://acmeclothes.com/'
|
|
334
361
|
},
|
|
335
362
|
{
|
|
336
363
|
type: 'test',
|
|
@@ -341,29 +368,9 @@ Suppose you ask for a merger of the above batch and script. Then the first job p
|
|
|
341
368
|
},
|
|
342
369
|
{
|
|
343
370
|
type: 'launch',
|
|
344
|
-
which: '
|
|
345
|
-
what: 'Acme Clothes
|
|
346
|
-
url: 'https://acmeclothes.com/
|
|
347
|
-
},
|
|
348
|
-
{
|
|
349
|
-
type: 'text',
|
|
350
|
-
which: 'User Name',
|
|
351
|
-
what: 'tester34'
|
|
352
|
-
},
|
|
353
|
-
{
|
|
354
|
-
type: 'text',
|
|
355
|
-
which: 'Password',
|
|
356
|
-
what: '34SecretTester'
|
|
357
|
-
},
|
|
358
|
-
{
|
|
359
|
-
type: 'button',
|
|
360
|
-
which: 'Submit',
|
|
361
|
-
what: 'submit the login form'
|
|
362
|
-
},
|
|
363
|
-
{
|
|
364
|
-
type: 'wait',
|
|
365
|
-
which: 'title',
|
|
366
|
-
what: 'account'
|
|
371
|
+
which: 'webkit',
|
|
372
|
+
what: 'Acme Clothes',
|
|
373
|
+
url: 'https://acmeclothes.com/'
|
|
367
374
|
},
|
|
368
375
|
{
|
|
369
376
|
type: 'test',
|
|
@@ -376,77 +383,22 @@ Suppose you ask for a merger of the above batch and script. Then the first job p
|
|
|
376
383
|
sources: {
|
|
377
384
|
script: 'ts99',
|
|
378
385
|
batch: 'clothing-stores',
|
|
386
|
+
lastTarget: false,
|
|
379
387
|
target: {
|
|
380
388
|
id: 'acme',
|
|
381
389
|
what: 'Acme Clothes'
|
|
382
390
|
},
|
|
383
391
|
requester: 'you@yourdomain.tld'
|
|
384
392
|
},
|
|
385
|
-
|
|
393
|
+
creationTimeStamp: '241120T1550'
|
|
386
394
|
}
|
|
387
395
|
```
|
|
388
396
|
|
|
389
|
-
Testilo has substituted the `private` acts from the `acme` target of the batch for the placeholder when creating the job. Testilo also has:
|
|
390
|
-
- inserted a copy of those same acts after the `axe` test act, because `axe` is a target-modifying tool.
|
|
391
|
-
- let the script determine the browser type of the `launch` act.
|
|
392
|
-
- given the job an ID that combines the time stamp with a differentiator and the batch ID.
|
|
393
|
-
- inserted a `sources` property into the job, recording facts about the script, the batch, the target, the requester, and the report URL.
|
|
394
|
-
- added a time stamp describing the creation time to the job.
|
|
395
|
-
|
|
396
|
-
This is a valid Testaro job.
|
|
397
|
-
|
|
398
|
-
#### Invocation
|
|
399
|
-
|
|
400
|
-
There are two ways to use the `merge` module.
|
|
401
|
-
|
|
402
|
-
##### By a module
|
|
403
|
-
|
|
404
|
-
A module can invoke `merge` in this way:
|
|
405
|
-
|
|
406
|
-
```javaScript
|
|
407
|
-
const {merge} = require('testilo/merge');
|
|
408
|
-
const jobs = merge(script, batch, standard, observe, requester, timeStamp);
|
|
409
|
-
```
|
|
410
|
-
|
|
411
|
-
The `merge` module uses these 4 arguments to create jobs from a script and a batch.
|
|
412
|
-
|
|
413
|
-
The arguments are:
|
|
414
|
-
- `script`: a script.
|
|
415
|
-
- `batch`: a batch.
|
|
416
|
-
- `standard`: how to handle standardization. If `also`, jobs will tell Testaro to include in its reports both the original results of the tests of tools and the Testaro-standardized results. If `only`, reports are to include only the standardized test results. If `no`, reports are to include only the original results, without standardization.
|
|
417
|
-
- `observe`: whether to allow granular observation. If `true`, jobs will tell Testaro to permit granular observation of job progress. If `false`, jobs will tell Testaro not to permit granular observation, but only to send the report to the server when the report is completed. It is generally user-friendly to allow granular observation, and for user applications to implement it, if they make users wait while jobs are assigned and performed, since that process typically takes about 3 minutes.
|
|
418
|
-
- `requester`: an email address.
|
|
419
|
-
- `timeStamp`: the earliest UTC date and time when the jobs may be assigned (format `240415T1230`), or an empty string if now.
|
|
420
|
-
|
|
421
|
-
The `merge()` function of the `merge` module generates jobs and returns them in an array. The invoking module can further dispose of the jobs as needed.
|
|
422
|
-
|
|
423
|
-
##### By a user
|
|
424
|
-
|
|
425
|
-
A user can invoke `merge` in this way:
|
|
426
|
-
|
|
427
|
-
- Create a script and save it as a JSON file in the `scripts` subdirectory of the `SPECDIR` directory.
|
|
428
|
-
- Create a batch and save it as a JSON file in the `batches` subdirectory of the `SPECDIR` directory.
|
|
429
|
-
- In the Testilo project directory, execute the statement:
|
|
430
|
-
|
|
431
|
-
```javascript
|
|
432
|
-
node call merge scriptID batchID standard observe requester timeStamp todoDir
|
|
433
|
-
```
|
|
434
|
-
|
|
435
|
-
In this statement, replace:
|
|
436
|
-
- `scriptID` with the ID (which is also the base of the file name) of the script.
|
|
437
|
-
- `batchID` with the ID (which is also the base of the file name) of the batch.
|
|
438
|
-
- `standard`, `observe`, `requester`, and `timeStamp` as described above.
|
|
439
|
-
- `todoDir`: `true` if the jobs are to be saved in the `todo` subdirectory, or `false` if they are to be saved in the `pending` subdirectory, of the `JOBDIR` directory.
|
|
440
|
-
|
|
441
|
-
The `call` module will retrieve the named script and batch from their respective directories.
|
|
442
|
-
The `merge` module will create an array of jobs.
|
|
443
|
-
The `call` module will save the jobs as JSON files in the `todo` or `pending` subdirectory of the `JOBDIR` directory.
|
|
444
|
-
|
|
445
397
|
#### Validation
|
|
446
398
|
|
|
447
399
|
To test the `merge` module, in the project directory you can execute the statement `node validation/merge/validate`. If `merge` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
|
448
400
|
|
|
449
|
-
## Report
|
|
401
|
+
## Report enhancement
|
|
450
402
|
|
|
451
403
|
### Introduction
|
|
452
404
|
|
|
@@ -459,16 +411,22 @@ Thus, a report produced by Testaro contains these properties:
|
|
|
459
411
|
- `what`
|
|
460
412
|
- `strict`
|
|
461
413
|
- `timeLimit`
|
|
414
|
+
- `standard`
|
|
415
|
+
- `observe`
|
|
416
|
+
- `sendReportTo`
|
|
417
|
+
- `timeStamp`
|
|
462
418
|
- `acts`
|
|
463
419
|
- `sources`
|
|
464
|
-
- `
|
|
465
|
-
- `timeStamp`
|
|
420
|
+
- `creationTimeStamp`
|
|
466
421
|
- `jobData`
|
|
467
422
|
|
|
468
|
-
Testilo can enhance such a report
|
|
423
|
+
Testilo can enhance such a report by:
|
|
469
424
|
- adding scores
|
|
470
425
|
- creating digests
|
|
471
|
-
- creating
|
|
426
|
+
- creating difgests
|
|
427
|
+
- creating comparisons
|
|
428
|
+
|
|
429
|
+
## Scoring
|
|
472
430
|
|
|
473
431
|
To add scores to reports, the `score` module of Testilo performs computations on the test results and adds a `score` property to each report.
|
|
474
432
|
|
|
@@ -480,7 +438,7 @@ A scoring function defines scoring rules. The Testilo package contains a `procs/
|
|
|
480
438
|
|
|
481
439
|
### Scorers
|
|
482
440
|
|
|
483
|
-
The built-in scoring functions are named `scorer` and are exported by files whose names begin with `tsp` (for Testilo scoring proc). Those functions make use of `issues` objects defined in files whose names begin with `tic`. An `issues` object defines an issue classification: a body of data about rules of tools and the tool-agnostic issues that those rules are deemed to belong to.
|
|
441
|
+
The built-in scoring functions are named `scorer()` and are exported by files whose names begin with `tsp` (for Testilo scoring proc). Those functions make use of `issues` objects defined in files whose names begin with `tic`. An `issues` object defines an issue classification: a body of data about rules of tools and the tool-agnostic issues that those rules are deemed to belong to.
|
|
484
442
|
|
|
485
443
|
The properties of an `issues` object are issue objects: objects containing data about issues. Here is an example from `tic40.js`:
|
|
486
444
|
|
|
@@ -530,31 +488,36 @@ There are two ways to invoke the `score` module.
|
|
|
530
488
|
|
|
531
489
|
#### By a module
|
|
532
490
|
|
|
533
|
-
A module can invoke `score` in this way:
|
|
491
|
+
A module can invoke `score()` in this way:
|
|
534
492
|
|
|
535
493
|
```javaScript
|
|
536
494
|
const {score} = require('testilo/score');
|
|
537
495
|
const {scorer} = require('testilo/procs/score/tsp99');
|
|
496
|
+
const reports = …;
|
|
538
497
|
score(scorer, reports);
|
|
539
498
|
```
|
|
540
499
|
|
|
541
|
-
The first argument to `score()` is a scoring function. In this example, it has been obtained from a module in the Testilo package, but it could be a custom function.
|
|
500
|
+
The first argument to `score()` is a scoring function. In this example, it has been obtained from a module in the Testilo package, but it could be a custom function.
|
|
501
|
+
|
|
502
|
+
The second argument to `score()` is an array of report objects. They may have been read from JSON files and parsed, or the array may contain a single report object parsed from the body of a `POST` request received from a Testaro agent.
|
|
503
|
+
|
|
504
|
+
The invoking module can further dispose of the scored reports as needed.
|
|
542
505
|
|
|
543
506
|
#### By a user
|
|
544
507
|
|
|
545
|
-
A user can invoke `score` in this way:
|
|
508
|
+
A user can invoke `score()` in this way:
|
|
546
509
|
|
|
547
510
|
```bash
|
|
548
511
|
node call score tsp99
|
|
549
512
|
node call score tsp99 75m
|
|
550
513
|
```
|
|
551
514
|
|
|
552
|
-
When a user invokes `score` in this example, the `call` module:
|
|
553
|
-
- gets the scoring module `tsp99` from its JSON file `tsp99.json` in the `score` subdirectory of the `
|
|
554
|
-
- gets the reports from the `raw` subdirectory of the `
|
|
555
|
-
- writes the scored reports in JSON format to the `scored` subdirectory of the `
|
|
515
|
+
When a user invokes `score()` in this example, the `call` module:
|
|
516
|
+
- gets the scoring module `tsp99` from its JSON file `tsp99.json` in the `score` subdirectory of the `FUNCTIONDIR` directory.
|
|
517
|
+
- gets the reports from the `raw` subdirectory of the `REPORTDIR` directory.
|
|
518
|
+
- writes the scored reports in JSON format to the `scored` subdirectory of the `REPORTDIR` directory.
|
|
556
519
|
|
|
557
|
-
The optional third argument to call (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `raw` subdirectory. With the argument, `call` gets only those reports whose names begin with the argument string.
|
|
520
|
+
The optional third argument to `call()` (`75m` in this example) is a report selector. Without the argument, `call()` gets all the reports in the `raw` subdirectory. With the argument, `call()` gets only those reports whose names begin with the argument string.
|
|
558
521
|
|
|
559
522
|
### Validation
|
|
560
523
|
|
|
@@ -564,16 +527,16 @@ To test the `score` module, in the project directory you can execute the stateme
|
|
|
564
527
|
|
|
565
528
|
### Introduction
|
|
566
529
|
|
|
567
|
-
Reports from Testaro are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
|
|
530
|
+
Reports from Testaro are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. This is equally true for reports that have been scored by Testilo. But Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
|
|
568
531
|
|
|
569
532
|
The `digest` module digests a scored report. Its `digest()` function takes three arguments:
|
|
570
|
-
- a digest template
|
|
571
533
|
- a digesting function
|
|
572
534
|
- an array of scored report objects
|
|
535
|
+
- the URL of a directory containing the scored reports
|
|
573
536
|
|
|
574
|
-
The
|
|
537
|
+
The digesting function populates an HTML digest template. A copy of the template, with its placeholders replaced by computed values, becomes the digest. The digesting function defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/digest` directory, in which there are subdirectories, each containing a template and a module that exports a digesting function. You can use one of those modules, or you can create your own.
|
|
575
538
|
|
|
576
|
-
The included
|
|
539
|
+
The included templates format placeholders with leading and trailing underscore pairs (such as `__issueCount__`).
|
|
577
540
|
|
|
578
541
|
### Invocation
|
|
579
542
|
|
|
@@ -581,35 +544,103 @@ There are two ways to use the `digest` module.
|
|
|
581
544
|
|
|
582
545
|
#### By a module
|
|
583
546
|
|
|
584
|
-
A module can invoke `digest` in this way:
|
|
547
|
+
A module can invoke `digest()` in this way:
|
|
585
548
|
|
|
586
549
|
```javaScript
|
|
587
550
|
const {digest} = require('testilo/digest');
|
|
588
551
|
const digesterDir = `${process.env.FUNCTIONDIR}/digest/tdp99a`;
|
|
589
552
|
const {digester} = require(`${digesterDir}/index`);
|
|
590
|
-
|
|
553
|
+
const scoredReports = …;
|
|
554
|
+
const reportDirURL = 'https://xyz.org/a11yTesting/reports';
|
|
555
|
+
digest(digester, scoredReports, reportDirURL)
|
|
591
556
|
.then(digestedReports => {…});
|
|
592
557
|
```
|
|
593
558
|
|
|
594
|
-
The first argument to `digest()` is a digesting function. In this example, it has been obtained from a
|
|
559
|
+
The first argument to `digest()` is a digesting function. In this example, it has been obtained from a file in the Testilo package, but it could be custom-made.
|
|
560
|
+
|
|
561
|
+
The second argument to `digest()` is an array of scored report objects. They may have been read from JSON files and parsed, or the array may contain a single scored report output by `score()`.
|
|
562
|
+
|
|
563
|
+
The third argument is the absolute or relative URL of a directory where the reports being digested are located. The `digest()` function needs that URL because a digest includes a link to the full report. The link concatenates the directory URL with the report ID and a `.json` suffix.
|
|
564
|
+
|
|
565
|
+
The `digest()` function returns an array of digested reports. The invoking module can further dispose of the digested reports as needed.
|
|
595
566
|
|
|
596
567
|
#### By a user
|
|
597
568
|
|
|
598
|
-
A user can invoke `digest` in this way:
|
|
569
|
+
A user can invoke `digest()` in this way:
|
|
599
570
|
|
|
600
571
|
```bash
|
|
601
572
|
node call digest tdp99
|
|
602
573
|
node call digest tdp99 75m
|
|
603
574
|
```
|
|
604
575
|
|
|
605
|
-
When a user invokes `digest` in this example, the `call` module:
|
|
606
|
-
- gets the template and the digesting module from subdirectory `tdp99` in the `digest` subdirectory of the `
|
|
607
|
-
- gets the reports from the `scored` subdirectory of the `
|
|
608
|
-
- writes the digested reports to the `digested` subdirectory of the `
|
|
576
|
+
When a user invokes `digest()` in this example, the `call` module:
|
|
577
|
+
- gets the template and the digesting module from subdirectory `tdp99` in the `digest` subdirectory of the `FUNCTIONDIR` directory.
|
|
578
|
+
- gets the reports from the `scored` subdirectory of the `REPORTDIR` directory.
|
|
579
|
+
- writes the digested reports to the `digested` subdirectory of the `REPORTDIR` directory.
|
|
580
|
+
- includes in each digest a link to the scored report, with the link destination being based on `DIGEST_URL`.
|
|
581
|
+
|
|
582
|
+
The third argument to `call()` can be an absolute URL, as shown, or a URL that is relative to the URL of the digest. For example, if it is known that the scored reports and the digests will inhabit the same directory and be retrievable with identical URLs except for the file extensions, then the third argument can be `./`.
|
|
583
|
+
|
|
584
|
+
The optional fourth argument to `call()` (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory. With the argument, `call` gets only those reports whose names begin with the argument string.
|
|
585
|
+
|
|
586
|
+
The digests created by `digest()` are HTML files, and they expect a `style.css` file to exist in their directory. The `reports/digested/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where digested reports are written.
|
|
587
|
+
|
|
588
|
+
## Report difgesting
|
|
589
|
+
|
|
590
|
+
### Introduction
|
|
591
|
+
|
|
592
|
+
A _difgest_ is a digest that compares two reports. They can be reports of different targets, or reports of the same target from two different times or under two different conditions.
|
|
593
|
+
|
|
594
|
+
The `difgest` module difgests two scored reports. Its `difgest()` function takes five arguments:
|
|
595
|
+
- a difgest template
|
|
596
|
+
- a difgesting function
|
|
597
|
+
- a scored report object
|
|
598
|
+
- another scored report object
|
|
599
|
+
- the URL of the digest of the first scored report
|
|
600
|
+
- the URL of the digest of the second scored report
|
|
601
|
+
|
|
602
|
+
The difgest template and module operate like the digest ones.
|
|
603
|
+
|
|
604
|
+
### Invocation
|
|
605
|
+
|
|
606
|
+
There are two ways to use the `difgest` module.
|
|
607
|
+
|
|
608
|
+
#### By a module
|
|
609
|
+
|
|
610
|
+
A module can invoke `difgest()` in this way:
|
|
611
|
+
|
|
612
|
+
```javaScript
|
|
613
|
+
const {difgest} = require('testilo/difgest');
|
|
614
|
+
const difgesterDir = `${process.env.FUNCTIONDIR}/difgest/tdp99a`;
|
|
615
|
+
const {difgester} = require(`${difgesterDir}/index`);
|
|
616
|
+
const scoredReportA = …;
|
|
617
|
+
const scoredReportB = …;
|
|
618
|
+
const digestAURL = 'https://abc.com/testing/reports/digested/241022T1458-0.html';
|
|
619
|
+
const digestBURL = 'https://abc.com/testing/reports/digested/241029T1458-0.html';
|
|
620
|
+
difgest(difgester, scoredReportA, scoredReportB, digestAURL, digestBURL)
|
|
621
|
+
.then(difgestedReport => {…});
|
|
622
|
+
```
|
|
623
|
+
|
|
624
|
+
The difgest will include links to the two digests, which, in turn, contain links to the full reports.
|
|
609
625
|
|
|
610
|
-
|
|
626
|
+
`difgest()` returns a difgest. The invoking module can further dispose of the difgest as needed.
|
|
611
627
|
|
|
612
|
-
|
|
628
|
+
#### By a user
|
|
629
|
+
|
|
630
|
+
A user can invoke `difgest` in this way:
|
|
631
|
+
|
|
632
|
+
```bash
|
|
633
|
+
node call difgest tfp99 20141215T1200-x7-3 20141215T1200-x7-4
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
When a user invokes `difgest` in this example, the `call` module:
|
|
637
|
+
- gets the template and the difgesting module from subdirectory `tfp99` in the `difgest` subdirectory of the `FUNCTIONDIR` directory.
|
|
638
|
+
- gets reports `20141215T1200-x7-3` and `20141215T1200-x7-4` from the `scored` subdirectory of the `REPORTDIR` directory.
|
|
639
|
+
- writes the difgested report to the `difgested` subdirectory of the `REPORTDIR` directory.
|
|
640
|
+
|
|
641
|
+
Difgests include links to the digests of the two reports. The destinations of those links are obtained from the `DIFGEST_URL` environment variable.
|
|
642
|
+
|
|
643
|
+
Difgests expect a `style.css` file to exist in their directory, as digests do.
|
|
613
644
|
|
|
614
645
|
### Validation
|
|
615
646
|
|
|
@@ -648,15 +679,14 @@ The first argument to `compare()` is a comparison function. In this example, it
|
|
|
648
679
|
A user can invoke `compare` in this way:
|
|
649
680
|
|
|
650
681
|
```bash
|
|
682
|
+
node call compare tcp99 legislators
|
|
651
683
|
node call compare tcp99 legislators 23pl
|
|
652
684
|
```
|
|
653
685
|
|
|
654
686
|
When a user invokes `compare` in this example, the `call` module:
|
|
655
|
-
- gets the comparison module from subdirectory `tcp99` of the subdirectory `compare` in the `
|
|
656
|
-
- gets all the reports in the `scored` subdirectory of the `
|
|
657
|
-
- writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `
|
|
658
|
-
|
|
659
|
-
The fourth argument to `call` (`23pl` in this example) is optional. If it is omitted, `call` will get and `comparer` will compare all the reports in the `scored` directory.
|
|
687
|
+
- gets the comparison module from subdirectory `tcp99` of the subdirectory `compare` in the `FUNCTIONDIR` directory.
|
|
688
|
+
- gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory, or, if there is a fourth argument, whose file names begin with `23pl`.
|
|
689
|
+
- writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `REPORTDIR` directory.
|
|
660
690
|
|
|
661
691
|
The comparative report created by `compare` is an HTML file, and it expects a `style.css` file to exist in its directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
|
|
662
692
|
|
|
@@ -699,8 +729,8 @@ node call credit legislators 23pl
|
|
|
699
729
|
```
|
|
700
730
|
|
|
701
731
|
When a user invokes `credit` in this example, the `call` module:
|
|
702
|
-
- gets all the reports in the `scored` subdirectory of the `
|
|
703
|
-
- writes the credit report as a JSON file named `legislators.json` to the `credit` subdirectory of the `
|
|
732
|
+
- gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory whose file names begin with `23pl`.
|
|
733
|
+
- writes the credit report as a JSON file named `legislators.json` to the `credit` subdirectory of the `REPORTDIR` directory.
|
|
704
734
|
|
|
705
735
|
The third argument to `call` (`23pl` in this example) is optional. If it is omitted, `call` will get and `credit()` will tabulate all the reports in the `scored` directory.
|
|
706
736
|
|
package/call.js
CHANGED
|
@@ -16,6 +16,8 @@
|
|
|
16
16
|
|
|
17
17
|
// Module to keep secrets.
|
|
18
18
|
require('dotenv').config();
|
|
19
|
+
// Module to perform common operations.
|
|
20
|
+
const {getNowStamp, getRandomString} = require('./procs/util');
|
|
19
21
|
// Function to process files.
|
|
20
22
|
const fs = require('fs/promises');
|
|
21
23
|
// Function to process a list-to-batch conversion.
|
|
@@ -28,6 +30,8 @@ const {merge} = require('./merge');
|
|
|
28
30
|
const {score} = require('./score');
|
|
29
31
|
// Function to digest reports.
|
|
30
32
|
const {digest} = require('./digest');
|
|
33
|
+
// Function to difgest reports.
|
|
34
|
+
const {difgest} = require('./difgest');
|
|
31
35
|
// Function to compare scores.
|
|
32
36
|
const {compare} = require('./compare');
|
|
33
37
|
|
|
@@ -124,6 +128,7 @@ const callScore = async (scorerID, selector = '') => {
|
|
|
124
128
|
// Score the reports.
|
|
125
129
|
score(scorer, reports);
|
|
126
130
|
const scoredReportDir = `${reportDir}/scored`;
|
|
131
|
+
await fs.mkdir(scoredReportDir, {recursive: true});
|
|
127
132
|
// For each scored report:
|
|
128
133
|
for (const report of reports) {
|
|
129
134
|
// Save it.
|
|
@@ -151,6 +156,7 @@ const callDigest = async (digesterID, selector = '') => {
|
|
|
151
156
|
// Digest the reports.
|
|
152
157
|
const digestedReports = await digest(digester, reports);
|
|
153
158
|
const digestedReportDir = `${reportDir}/digested`;
|
|
159
|
+
await fs.mkdir(digestedReportDir, {recursive: true});
|
|
154
160
|
// For each digested report:
|
|
155
161
|
for (const reportID of Object.keys(digestedReports)) {
|
|
156
162
|
// Save it.
|
|
@@ -164,6 +170,34 @@ const callDigest = async (digesterID, selector = '') => {
|
|
|
164
170
|
console.log('ERROR: No scored reports to be digested');
|
|
165
171
|
}
|
|
166
172
|
};
|
|
173
|
+
// Fulfills a digesting request.
|
|
174
|
+
const callDifgest = async (difgesterID, reportAID, reportBID) => {
|
|
175
|
+
// Get the scored reports to be difgested.
|
|
176
|
+
const reportAArray = await getReports('scored', reportAID);
|
|
177
|
+
const reportBArray = await getReports('scored', reportBID);
|
|
178
|
+
const reportA = reportAArray[0];
|
|
179
|
+
const reportB = reportBArray[0];
|
|
180
|
+
// If both exist:
|
|
181
|
+
if (reportAID && reportBID) {
|
|
182
|
+
// Get the difgester.
|
|
183
|
+
const difgesterDir = `${functionDir}/difgest/${difgesterID}`;
|
|
184
|
+
const {difgester} = require(`${difgesterDir}/index`);
|
|
185
|
+
// Difgest the reports.
|
|
186
|
+
const difgestedReport = await difgest(difgester, reportA, reportB);
|
|
187
|
+
const difgestedReportDir = `${reportDir}/difgested`;
|
|
188
|
+
await fs.mkdir(difgestedReportDir, {recursive: true});
|
|
189
|
+
// Save the difgested report.
|
|
190
|
+
const difgestID = `${getNowStamp()}-${getRandomString(2)}-0`;
|
|
191
|
+
const difgestPath = `${difgestedReportDir}/${difgestID}.html`;
|
|
192
|
+
await fs.writeFile(difgestPath, difgestedReport);
|
|
193
|
+
console.log(`Reports ${reportAID} and ${reportBID} difgested and saved as ${difgestPath}`);
|
|
194
|
+
}
|
|
195
|
+
// Otherwise, i.e. if no scored reports are to be digested:
|
|
196
|
+
else {
|
|
197
|
+
// Report this.
|
|
198
|
+
console.log('ERROR: No pair of scored reports to be digested');
|
|
199
|
+
}
|
|
200
|
+
};
|
|
167
201
|
// Fulfills a comparison request.
|
|
168
202
|
// Get the scored reports to be scored.
|
|
169
203
|
const callCompare = async (compareProcID, comparisonNameBase, selector = '') => {
|
|
@@ -177,6 +211,7 @@ const callCompare = async (compareProcID, comparisonNameBase, selector = '') =>
|
|
|
177
211
|
const comparison = await compare(comparer, reports);
|
|
178
212
|
// Save the comparison.
|
|
179
213
|
const comparisonDir = `${reportDir}/comparative`;
|
|
214
|
+
await fs.mkdir(comparisonDir, {recursive: true});
|
|
180
215
|
await fs.writeFile(`${comparisonDir}/${comparisonNameBase}.html`, comparison);
|
|
181
216
|
console.log(`Comparison completed and saved in ${comparisonDir}`);
|
|
182
217
|
}
|
|
@@ -192,8 +227,10 @@ const callCredit = async (tallyID, selector = '') => {
|
|
|
192
227
|
// Tally the reports.
|
|
193
228
|
const tally = credit(reports);
|
|
194
229
|
// Save the tally.
|
|
195
|
-
|
|
196
|
-
|
|
230
|
+
const creditDir = `${reportDir}/credit`;
|
|
231
|
+
await fs.mkdir(creditDir, {recursive: true});
|
|
232
|
+
await fs.writeFile(`${creditDir}/${tallyID}.json`, JSON.stringify(tally, null, 2));
|
|
233
|
+
console.log(`Reports tallied and credit report saved in ${creditDir}`);
|
|
197
234
|
}
|
|
198
235
|
// Otherwise, i.e. if no scored reports are to be tallied:
|
|
199
236
|
else {
|
|
@@ -217,7 +254,7 @@ else if (fn === 'batch' && fnArgs.length === 2) {
|
|
|
217
254
|
console.log('Execution completed');
|
|
218
255
|
});
|
|
219
256
|
}
|
|
220
|
-
else if (fn === 'script' && fnArgs.length) {
|
|
257
|
+
else if (fn === 'script' && (fnArgs.length === 1 || fnArgs.length > 2)) {
|
|
221
258
|
callScript(... fnArgs)
|
|
222
259
|
.then(() => {
|
|
223
260
|
console.log('Execution completed');
|
|
@@ -241,14 +278,14 @@ else if (fn === 'multiScore' && fnArgs.length === 1) {
|
|
|
241
278
|
console.log('Execution completed');
|
|
242
279
|
});
|
|
243
280
|
}
|
|
244
|
-
else if (fn === 'digest' && fnArgs.length
|
|
281
|
+
else if (fn === 'digest' && fnArgs.length && fnArgs.length < 3) {
|
|
245
282
|
callDigest(... fnArgs)
|
|
246
283
|
.then(() => {
|
|
247
284
|
console.log('Execution completed');
|
|
248
285
|
});
|
|
249
286
|
}
|
|
250
|
-
else if (fn === '
|
|
251
|
-
|
|
287
|
+
else if (fn === 'difgest' && fnArgs.length === 3) {
|
|
288
|
+
callDifgest(... fnArgs)
|
|
252
289
|
.then(() => {
|
|
253
290
|
console.log('Execution completed');
|
|
254
291
|
});
|
package/difgest.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/*
|
|
2
|
+
difgest.js
|
|
3
|
+
Creates difgests from scored reports.
|
|
4
|
+
Arguments:
|
|
5
|
+
0. Difgesting function.
|
|
6
|
+
1. Array of 2 scored reports.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
// ########## FUNCTIONS
|
|
10
|
+
|
|
11
|
+
// Difgests the scored reports and returns a difgest.
|
|
12
|
+
exports.difgest = async (difgester, reportA, reportB) => {
|
|
13
|
+
const difgestedReport = await difgester(reportA, reportB);
|
|
14
|
+
console.log(`Reports ${reportA.id} and ${reportB.id} difgested`);
|
|
15
|
+
// Return the difgest.
|
|
16
|
+
return difgestedReport;
|
|
17
|
+
};
|
package/package.json
CHANGED
|
@@ -12,14 +12,14 @@ const fs = require('fs/promises');
|
|
|
12
12
|
|
|
13
13
|
// ########## CONSTANTS
|
|
14
14
|
|
|
15
|
-
const
|
|
15
|
+
const reportDir = process.env.REPORTDIR;
|
|
16
16
|
const query = {};
|
|
17
17
|
|
|
18
18
|
// ########## FUNCTIONS
|
|
19
19
|
|
|
20
20
|
// Returns data on the hosts in the report directory.
|
|
21
21
|
const getData = async () => {
|
|
22
|
-
const reportDirAbs = `${__dirname}/../../../${
|
|
22
|
+
const reportDirAbs = `${__dirname}/../../../${reportDir}/scored`;
|
|
23
23
|
const reportFileNamesAll = await fs.readdir(reportDirAbs);
|
|
24
24
|
const reportFileNamesSource = reportFileNamesAll.filter(fileName => fileName.endsWith('.json'));
|
|
25
25
|
const pageCount = reportFileNamesSource.length;
|
|
@@ -12,14 +12,14 @@ const fs = require('fs/promises');
|
|
|
12
12
|
|
|
13
13
|
// ########## CONSTANTS
|
|
14
14
|
|
|
15
|
-
const
|
|
15
|
+
const reportDir = process.env.REPORTDIR;
|
|
16
16
|
const query = {};
|
|
17
17
|
|
|
18
18
|
// ########## FUNCTIONS
|
|
19
19
|
|
|
20
20
|
// Returns data on the hosts in the report directory.
|
|
21
21
|
const getData = async () => {
|
|
22
|
-
const reportDirAbs = `${__dirname}/../../../${
|
|
22
|
+
const reportDirAbs = `${__dirname}/../../../${reportDir}/scored`;
|
|
23
23
|
const reportFileNamesAll = await fs.readdir(reportDirAbs);
|
|
24
24
|
const reportFileNamesSource = reportFileNamesAll.filter(fileName => fileName.endsWith('.json'));
|
|
25
25
|
const pageCount = reportFileNamesSource.length;
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
<meta name="keywords" content="accessibility a11y web testing">
|
|
11
11
|
<title>Accessibility score comparison</title>
|
|
12
12
|
<link rel="icon" href="favicon.ico">
|
|
13
|
-
<link rel="stylesheet" href="
|
|
13
|
+
<link rel="stylesheet" href="style.css">
|
|
14
14
|
</head>
|
|
15
15
|
<body>
|
|
16
16
|
<main>
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
</header>
|
|
20
20
|
<h2>Introduction</h2>
|
|
21
21
|
<p>The table below compares __pageCount__ web pages on <a href="https://www.w3.org/WAI/fundamentals/accessibility-intro/">accessibility</a>. The page names are links to the pages on the web. The scores are links to digests that explain in detail how the scores were computed.</p>
|
|
22
|
-
<p>The pages were tested by <a href="https://www.npmjs.com/package/testaro">Testaro</a>. Testaro used
|
|
22
|
+
<p>The pages were tested by <a href="https://www.npmjs.com/package/testaro">Testaro</a>. Testaro used ten tools (Alfa, ASLint, Axe, Editoria11y, Equal Access, HTML CodeSniffer, Nu Html Checker, QualWeb, Testaro, and WAVE) to perform about 900 automated accessibility tests.</p>
|
|
23
23
|
<p><a href="https://www.npmjs.com/package/testilo">Testilo</a> classified the problems found by these tests into <dfn>issues</dfn> and assigned a <dfn>score</dfn> to each page. A perfect score would be 0. Higher scores indicate more issues, more instances of them, more serious issues, and more of the tools reporting instances of issues.</p>
|
|
24
24
|
<h2>Comparison</h2>
|
|
25
25
|
<table class="allBorder">
|
|
@@ -7,14 +7,13 @@
|
|
|
7
7
|
|
|
8
8
|
// Module to access files.
|
|
9
9
|
const fs = require('fs/promises');
|
|
10
|
+
const {getBarCell} = require('../../util');
|
|
10
11
|
|
|
11
12
|
// CONSTANTS
|
|
12
13
|
|
|
13
14
|
// Comparer ID.
|
|
14
15
|
const id = 'tcp39';
|
|
15
16
|
// Newlines with indentations.
|
|
16
|
-
const joiner = '\n ';
|
|
17
|
-
const innerJoiner = '\n ';
|
|
18
17
|
const innestJoiner = '\n ';
|
|
19
18
|
|
|
20
19
|
// ########## FUNCTIONS
|
|
@@ -53,9 +52,7 @@ const getTableBody = async bodyData => {
|
|
|
53
52
|
const pageCell = `<th scope="row"><a href="${url}">${org}</a></th>`;
|
|
54
53
|
const numCell = `<td><a href="testu/digest?jobID=${id}">${score}</a></td>`;
|
|
55
54
|
// Make the bar width proportional.
|
|
56
|
-
const
|
|
57
|
-
const bar = `<rect height="100%" width="${barWidth}%" fill="red"></rect>`;
|
|
58
|
-
const barCell = `<td aria-hidden="true"><svg width="100%" height="0.7em">${bar}</svg></td>`;
|
|
55
|
+
const barCell = getBarCell(score, maxScore);
|
|
59
56
|
const row = `<tr>${pageCell}${numCell}${barCell}</tr>`;
|
|
60
57
|
return row;
|
|
61
58
|
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
<!DOCTYPE HTML>
|
|
2
|
+
<html lang="en-US">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<meta name="author" content="Testilo">
|
|
7
|
+
<meta name="creator" content="Testilo">
|
|
8
|
+
<meta name="publisher" name="Testilo">
|
|
9
|
+
<meta name="description" content="report of accessibility testing of a web page">
|
|
10
|
+
<meta name="keywords" content="accessibility a11y web testing">
|
|
11
|
+
<title>Accessibility difgest</title>
|
|
12
|
+
<link rel="icon" href="favicon.ico">
|
|
13
|
+
<link rel="stylesheet" href="style.css">
|
|
14
|
+
</head>
|
|
15
|
+
<body>
|
|
16
|
+
<main>
|
|
17
|
+
<header>
|
|
18
|
+
<h1>Accessibility difgest</h1>
|
|
19
|
+
<h2>Synopsis</h2>
|
|
20
|
+
<p>Difgested by: Testilo, proc __fp__</p>
|
|
21
|
+
<table class="allBorder">
|
|
22
|
+
<caption>Audit A</caption>
|
|
23
|
+
<tr><th>Page</th><td>__orgA__</td></tr>
|
|
24
|
+
<tr><th>URL</th><td>__urlA__</td></tr>
|
|
25
|
+
<tr><th>Test date</th><td>__dateSlashA__</td></tr>
|
|
26
|
+
<tr><th>Score</th><td>__totalA__</td></tr>
|
|
27
|
+
<tr><th>Digest</th><td><a href="__digestAURL__">__digestAURL__</a></td></tr>
|
|
28
|
+
</table>
|
|
29
|
+
<table class="allBorder">
|
|
30
|
+
<caption>Audit B</caption>
|
|
31
|
+
<tr><th>Page</th><td>__orgB__</td></tr>
|
|
32
|
+
<tr><th>URL</th><td>__urlB__</td></tr>
|
|
33
|
+
<tr><th>Test date</th><td>__dateSlashB__</td></tr>
|
|
34
|
+
<tr><th>Score</th><td>__totalB__</td></tr>
|
|
35
|
+
<tr><th>Digest</th><td><a href="__digestBURL__">__digestBURL__</a></td></tr>
|
|
36
|
+
</table>
|
|
37
|
+
</header>
|
|
38
|
+
<h2>Introduction</h2>
|
|
39
|
+
<p>This <q>difgest</q> summarizes the differences between the scores from the two accessibility audits referenced in the synopsis above. A perfect score would be 0.</p>
|
|
40
|
+
<h2>Issue summary</h2>
|
|
41
|
+
<p>Issues are ordered from the one on which B was most superior to the one on which A was most superior.</p>
|
|
42
|
+
<table class="allBorder">
|
|
43
|
+
<caption>Differences in issue scores</caption>
|
|
44
|
+
<thead>
|
|
45
|
+
<tr><th>Issue</th><th>WCAG</th><th>Score A</th><th>Score B</th><th>A − B</th><th>A superiority</th><th>B superiority</th></tr>
|
|
46
|
+
</thead>
|
|
47
|
+
<tbody class="headersLeft">
|
|
48
|
+
__issueRows__
|
|
49
|
+
</tbody>
|
|
50
|
+
</table>
|
|
51
|
+
<footer>
|
|
52
|
+
<p class="date">Produced <time itemprop="datePublished" datetime="__dateISO__">__dateSlash__</time></p>
|
|
53
|
+
</footer>
|
|
54
|
+
</main>
|
|
55
|
+
</body>
|
|
56
|
+
</html>
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// index: difgester for scoring procedure tsp40.
|
|
2
|
+
|
|
3
|
+
// IMPORTS
|
|
4
|
+
|
|
5
|
+
// Module to keep secrets.
|
|
6
|
+
require('dotenv').config();
|
|
7
|
+
// Module to classify tool rules into issues
|
|
8
|
+
const {issues} = require('../../score/tic40');
|
|
9
|
+
// Module to process files.
|
|
10
|
+
const fs = require('fs/promises');
|
|
11
|
+
// Utility module.
|
|
12
|
+
const {getBarCell} = require('../../util');
|
|
13
|
+
|
|
14
|
+
// CONSTANTS
|
|
15
|
+
|
|
16
|
+
// Difgester ID.
|
|
17
|
+
const id = 'tfp40';
|
|
18
|
+
// Newline with indentations.
|
|
19
|
+
const innerJoiner = '\n ';
|
|
20
|
+
|
|
21
|
+
// FUNCTIONS
|
|
22
|
+
|
|
23
|
+
// Gets a row of the issue-score-summary table.
|
|
24
|
+
const getIssueScoreRow = (summary, wcag, scoreA, scoreB, bSuperiorityMax, aSuperiorityMax) => {
|
|
25
|
+
const bSuperiority = scoreA - scoreB;
|
|
26
|
+
const barCell = getBarCell(
|
|
27
|
+
Math.abs(bSuperiority), bSuperiority > 0 ? bSuperiorityMax : aSuperiorityMax, bSuperiority < 0
|
|
28
|
+
);
|
|
29
|
+
const cells = [
|
|
30
|
+
`<th>${summary}</th>`,
|
|
31
|
+
`<td class="center">${wcag}</td>`,
|
|
32
|
+
`<td class="right">${scoreA}</td>`,
|
|
33
|
+
`<td class="right">${scoreB}</td>`,
|
|
34
|
+
`<td class="right">${scoreA - scoreB}</td>`,
|
|
35
|
+
bSuperiority < 0 ? barCell : '<td></td>',
|
|
36
|
+
bSuperiority > 0 ? barCell : '<td></td>'
|
|
37
|
+
];
|
|
38
|
+
return `<tr>${cells.join('')}</tr>`;
|
|
39
|
+
};
|
|
40
|
+
// Adds parameters to a query for a difgest.
|
|
41
|
+
const populateQuery = (reportA, reportB, query) => {
|
|
42
|
+
// General parameters.
|
|
43
|
+
query.fp = id;
|
|
44
|
+
query.dateISO = new Date().toISOString().slice(0, 10);
|
|
45
|
+
query.dateSlash = query.dateISO.replace(/-/g, '/');
|
|
46
|
+
// For each report:
|
|
47
|
+
const issueIDs = new Set();
|
|
48
|
+
[reportA, reportB].forEach((report, index) => {
|
|
49
|
+
// Add report-specific synopsis parameters to the query.
|
|
50
|
+
const suffix = ['A', 'B'][index];
|
|
51
|
+
const {id, sources, jobData, score} = report;
|
|
52
|
+
const {target} = sources;
|
|
53
|
+
const {summary, details} = score;
|
|
54
|
+
query[`org${suffix}`] = target.what;
|
|
55
|
+
query[`url${suffix}`] = target.which;
|
|
56
|
+
const dateISO = jobData.endTime.slice(0, 8);
|
|
57
|
+
query[`dateSlash${suffix}`] = dateISO.replace(/-/g, '/');
|
|
58
|
+
query[`total${suffix}`] = summary.total;
|
|
59
|
+
query[`digest${suffix}URL`] = process.env.DIGEST_URL.replace('__id__', id);
|
|
60
|
+
// Get the union of the issues in the reports.
|
|
61
|
+
Object.keys(details.issue).forEach(issueID => issueIDs.add(issueID));
|
|
62
|
+
});
|
|
63
|
+
// Get data on the issues.
|
|
64
|
+
const issuesData = Array.from(issueIDs).map(issueID => {
|
|
65
|
+
const issueDataA = reportA.score.details.issue[issueID] || null;
|
|
66
|
+
const issueDataB = reportB.score.details.issue[issueID] || null;
|
|
67
|
+
return {
|
|
68
|
+
id: issueID,
|
|
69
|
+
what: issueDataA ? issueDataA.summary : issueDataB.summary,
|
|
70
|
+
wcag: issueDataA ? issueDataA.wcag : issueDataB.wcag,
|
|
71
|
+
scoreA: issueDataA ? issueDataA.score : 0,
|
|
72
|
+
scoreB: issueDataB ? issueDataB.score : 0
|
|
73
|
+
};
|
|
74
|
+
});
|
|
75
|
+
// Sort the issue data in descending order of B less A scores.
|
|
76
|
+
issuesData.sort((i, j) => i.scoreB - i.scoreA - j.scoreB + j.scoreA);
|
|
77
|
+
// Get rows for the issue-score table.
|
|
78
|
+
const bSuperiorityMax = Math.max(0, issuesData[0].scoreA - issuesData[0].scoreB);
|
|
79
|
+
const lastIssue = issuesData[issueIDs.size - 1];
|
|
80
|
+
const aSuperiorityMax = Math.max(0, lastIssue.scoreB - lastIssue.scoreA);
|
|
81
|
+
const issueRows = [];
|
|
82
|
+
issuesData.forEach(issueData => {
|
|
83
|
+
const {id, what, wcag, scoreA, scoreB} = issueData;
|
|
84
|
+
if (issues[id]) {
|
|
85
|
+
issueRows.push(
|
|
86
|
+
getIssueScoreRow(what, wcag, scoreA, scoreB, bSuperiorityMax, aSuperiorityMax)
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
console.log(`ERROR: Issue ${id} not found`);
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
// Add the rows to the query.
|
|
94
|
+
query.issueRows = issueRows.join(innerJoiner);
|
|
95
|
+
};
|
|
96
|
+
// Returns a difgested report.
|
|
97
|
+
exports.difgester = async (reportA, reportB) => {
|
|
98
|
+
// Create a query to replace placeholders.
|
|
99
|
+
const query = {};
|
|
100
|
+
populateQuery(reportA, reportB, query);
|
|
101
|
+
// Get the template.
|
|
102
|
+
let template = await fs.readFile(`${__dirname}/index.html`, 'utf8');
|
|
103
|
+
// Replace its placeholders.
|
|
104
|
+
Object.keys(query).forEach(param => {
|
|
105
|
+
template = template.replace(new RegExp(`__${param}__`, 'g'), query[param]);
|
|
106
|
+
});
|
|
107
|
+
// Return the digest.
|
|
108
|
+
return template;
|
|
109
|
+
};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/*
|
|
2
2
|
index: digester for scoring procedure spA11yMessage.
|
|
3
3
|
Creator of parameters for substitution into index.html.
|
|
4
|
-
Usage example for selected files in
|
|
5
|
-
Usage example for all files in
|
|
4
|
+
Usage example for selected files in REPORTDIR/scored: node digest dpA11yMessage 35k1r
|
|
5
|
+
Usage example for all files in REPORTDIR/scored: node digest dpA11yMessage
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
// CONSTANTS
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
/*
|
|
2
2
|
index: digester for scoring procedure spA11yMessage.
|
|
3
3
|
Creator of parameters for substitution into index.html.
|
|
4
|
-
Usage example for selected files in
|
|
5
|
-
Usage example for all files in
|
|
4
|
+
Usage example for selected files in REPORTDIR/scored: node digest dpA11yMessage 35k1r
|
|
5
|
+
Usage example for all files in REPORTDIR/scored: node digest dpA11yMessage
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
// CONSTANTS
|
|
9
9
|
|
|
10
10
|
// Newlines with indentations.
|
|
11
|
-
const joiner = '\n ';
|
|
12
11
|
const innerJoiner = '\n ';
|
|
13
12
|
|
|
14
13
|
// FUNCTIONS
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
<tr><th>Digested by</th><td>Testilo, procedure <code>__dp__</code></td></tr>
|
|
29
29
|
<tr>
|
|
30
30
|
<th>Full report</th>
|
|
31
|
-
<td><a href="
|
|
31
|
+
<td><a href="__reportURL__"><code>__reportURL__</code></a></td>
|
|
32
32
|
</tr>
|
|
33
33
|
</table>
|
|
34
34
|
</header>
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
</tbody>
|
|
48
48
|
</table>
|
|
49
49
|
<h2>Itemized issues</h2>
|
|
50
|
-
<p>The reported rule violations are itemized below, issue by issue. Additional details can be inspected in the <a href="
|
|
50
|
+
<p>The reported rule violations are itemized below, issue by issue. Additional details can be inspected in the <a href="__reportURL__">full report</a>.</p>
|
|
51
51
|
__issueDetailRows__
|
|
52
52
|
<footer>
|
|
53
53
|
<p class="date">Produced <time itemprop="datePublished" datetime="__dateISO__">__dateSlash__</time></p>
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
// IMPORTS
|
|
4
4
|
|
|
5
|
+
// Module to keep secrets.
|
|
6
|
+
require('dotenv').config();
|
|
5
7
|
// Module to classify tool rules into issues
|
|
6
8
|
const {issues} = require('../../score/tic40');
|
|
7
9
|
// Module to process files.
|
|
@@ -10,7 +12,7 @@ const fs = require('fs/promises');
|
|
|
10
12
|
// CONSTANTS
|
|
11
13
|
|
|
12
14
|
// Digester ID.
|
|
13
|
-
const
|
|
15
|
+
const digesterID = 'tdp40';
|
|
14
16
|
// Newline with indentations.
|
|
15
17
|
const innerJoiner = '\n ';
|
|
16
18
|
|
|
@@ -21,23 +23,23 @@ const getScoreRow = (componentName, score) => `<tr><th>${componentName}</th><td>
|
|
|
21
23
|
// Gets a row of the issue-score-summary table.
|
|
22
24
|
const getIssueScoreRow = (summary, wcag, score, tools) => {
|
|
23
25
|
const toolList = tools.map(tool => `<code>${tool}</code>`).join(', ');
|
|
24
|
-
return `<tr><th>${summary}</th><td>${wcag}<td>${score}</td><td>${toolList}</tr>`;
|
|
26
|
+
return `<tr><th>${summary}</th><td>${wcag}<td>${score}</td><td>${toolList}</td></tr>`;
|
|
25
27
|
};
|
|
26
28
|
// Adds parameters to a query for a digest.
|
|
27
29
|
const populateQuery = (report, query) => {
|
|
28
|
-
const {
|
|
30
|
+
const {id, sources, jobData, score} = report;
|
|
29
31
|
const {script, target, requester} = sources;
|
|
30
32
|
const {scoreProcID, summary, details} = score;
|
|
31
33
|
query.ts = script;
|
|
32
34
|
query.sp = scoreProcID;
|
|
33
|
-
query.dp =
|
|
35
|
+
query.dp = digesterID;
|
|
34
36
|
// Add the job data to the query.
|
|
35
37
|
query.dateISO = jobData.endTime.slice(0, 8);
|
|
36
38
|
query.dateSlash = query.dateISO.replace(/-/g, '/');
|
|
37
39
|
query.org = target.what;
|
|
38
40
|
query.url = target.which;
|
|
39
41
|
query.requester = requester;
|
|
40
|
-
query.
|
|
42
|
+
query.reportURL = process.env.SCORED_REPORT_URL.replace('__id__', id);
|
|
41
43
|
// Add values for the score-summary table to the query.
|
|
42
44
|
const rows = {
|
|
43
45
|
summaryRows: [],
|
package/procs/score/tic40.js
CHANGED
package/procs/score/tsp40.js
CHANGED
package/procs/util.js
CHANGED
|
@@ -63,3 +63,12 @@ exports.getRandomString = length => {
|
|
|
63
63
|
}
|
|
64
64
|
return chars.join('');
|
|
65
65
|
};
|
|
66
|
+
// Returns a graph bar.
|
|
67
|
+
exports.getBarCell = (num, max, isRight = false) => {
|
|
68
|
+
// Make the bar width proportional.
|
|
69
|
+
const barWidth = 100 * num / max;
|
|
70
|
+
const xAtt = isRight ? ` x=${100 - barWidth}%` : '';
|
|
71
|
+
const bar = `<rect height="100%"${xAtt} width="${barWidth}%" fill="red"></rect>`;
|
|
72
|
+
const barCell = `<td aria-hidden="true"><svg width="100%" height="0.7em">${bar}</svg></td>`;
|
|
73
|
+
return barCell;
|
|
74
|
+
};
|