testilo 10.3.0 → 11.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +188 -190
  2. package/batch.js +60 -0
  3. package/call.js +9 -4
  4. package/compare.js +2 -2
  5. package/merge.js +0 -6
  6. package/package.json +1 -1
  7. package/procs/compare/cp20sqrt/index.js +7 -10
  8. package/procs/compare/{cp15b → tcp24}/index.html +5 -5
  9. package/procs/compare/{cp12a → tcp24}/index.js +13 -20
  10. package/procs/score/{tic24.js → tic25.js} +40 -27
  11. package/procs/score/{tic23.js → tic26.js} +264 -229
  12. package/procs/score/tsp24.js +8 -2
  13. package/procs/score/tsp26.js +120 -0
  14. package/score.js +2 -8
  15. package/procs/compare/cp0/index.html +0 -46
  16. package/procs/compare/cp0/index.js +0 -70
  17. package/procs/compare/cp1/index.html +0 -46
  18. package/procs/compare/cp1/index.js +0 -71
  19. package/procs/compare/cp12a/index.html +0 -47
  20. package/procs/compare/cp12b/index.html +0 -47
  21. package/procs/compare/cp12b/index.js +0 -71
  22. package/procs/compare/cp14a/index.html +0 -47
  23. package/procs/compare/cp14a/index.js +0 -71
  24. package/procs/compare/cp15a/index.html +0 -47
  25. package/procs/compare/cp15a/index.js +0 -71
  26. package/procs/compare/cp15b/index.js +0 -74
  27. package/procs/compare/cp16a/index.html +0 -47
  28. package/procs/compare/cp16a/index.js +0 -74
  29. package/procs/compare/cp16b/index.html +0 -47
  30. package/procs/compare/cp16b/index.js +0 -74
  31. package/procs/digest/dp10a/index.html +0 -137
  32. package/procs/digest/dp10a/index.js +0 -402
  33. package/procs/digest/dp10b/index.html +0 -74
  34. package/procs/digest/dp10b/index.js +0 -130
  35. package/procs/digest/dp10c/index.html +0 -55
  36. package/procs/digest/dp10c/index.js +0 -129
  37. package/procs/digest/dp11a/index.html +0 -76
  38. package/procs/digest/dp11a/index.js +0 -127
  39. package/procs/digest/dp12a/index.html +0 -78
  40. package/procs/digest/dp12a/index.js +0 -126
  41. package/procs/digest/dp12b/index.html +0 -78
  42. package/procs/digest/dp12b/index.js +0 -126
  43. package/procs/digest/dp13a/index.html +0 -78
  44. package/procs/digest/dp13a/index.js +0 -126
  45. package/procs/digest/dp14a/index.html +0 -79
  46. package/procs/digest/dp14a/index.js +0 -126
  47. package/procs/digest/dp15a/index.html +0 -80
  48. package/procs/digest/dp15a/index.js +0 -126
  49. package/procs/digest/dp16a/index.html +0 -80
  50. package/procs/digest/dp16a/index.js +0 -126
  51. package/procs/digest/dp16b/index.html +0 -80
  52. package/procs/digest/dp16b/index.js +0 -126
  53. package/procs/digest/dp18a/index.html +0 -80
  54. package/procs/digest/dp18a/index.js +0 -129
  55. package/procs/digest/dp20c/index.html +0 -54
  56. package/procs/digest/dp20c/index.js +0 -117
  57. package/procs/digest/dp20d/index.html +0 -54
  58. package/procs/digest/dp20d/index.js +0 -121
  59. package/procs/digest/tdp09a/index.html +0 -126
  60. package/procs/digest/tdp09a/index.js +0 -381
  61. package/procs/digest/tdp22/index.html +0 -54
  62. package/procs/digest/tdp22/index.js +0 -120
  63. package/procs/score/tic21.js +0 -6868
  64. package/procs/score/tic22.js +0 -6911
  65. package/procs/score/tsp21.js +0 -720
  66. package/procs/score/tsp22.js +0 -720
  67. package/procs/score/tsp23.js +0 -720
  68. package/specs/batches/orgs.json +0 -43
  69. package/specs/scripts/ts21.json +0 -213
  70. package/specs/scripts/ts23.json +0 -219
  71. package/specs/scripts/ts24.json +0 -225
package/README.md CHANGED
@@ -36,214 +36,231 @@ Testaro executes _jobs_. In a job, Testaro performs _acts_ (tests and other oper
36
36
  You can create a job for Testaro directly, without using Testilo.
37
37
 
38
38
  Testilo can, however, make job preparation more efficient in two scenarios:
39
- - A common use case is to define a battery of tests and to have those tests performed on multiple targets. In that case, you need to create multiple jobs, one per page. The jobs are identical, except for the target-specific acts (including navigating to targets). If you tell Testilo about the tests and the targets, Testilo can create such collections of jobs for you.
40
- - Some tests operate on a copy of a target and modify that copy. Usually, one wants test isolation: The results of a test do not depend on any previously performed tests. To ensure test isolation, a job containing such target-modifying tests must follow them with acts that restore the target to its desired pre-test state. Testilo can insert the required target-restoring acts into each job after target-modifying tests.
39
+ - A common use case is to define a battery of tests and to have those same tests performed on multiple targets. In that case, it is efficient to create multiple jobs, one per target. The jobs are identical, except for the target-specific acts (including navigating to targets). If you tell Testilo about the tests and the targets, Testilo can create such collections of jobs for you.
40
+ - Some tests operate on a copy of a target and modify that copy. Usually, one wants test isolation: The results of a test do not depend on any previously performed tests. To ensure test isolation, a job containing such target-modifying test acts must follow them with acts that restore the target to its pre-test state. Testilo can insert the required target-restoring acts into each job after target-modifying test act.
41
41
 
42
- The `merge` module performs these services.
42
+ The `batch` and `merge` modules performs these services.
43
43
 
44
- ### Procedure
44
+ ### Target lists
45
45
 
46
- To use the `merge` module, you need to create a _script_ and a _batch_:
47
- - The script is an object that specifies a job, except for the targets.
48
- - The batch is an object that specifies target-specific acts for targets.
46
+ The simplest version of a list of targets is _target list_. It is stored as a tab-delimited text file, with one line per target. Each line contains 3 items, with tabs between them:
47
+ - An ID for the target
48
+ - A description of the target
49
+ - The URL of the target
49
50
 
50
- The script contains an array of acts, with placeholders. For each target in the batch, the `merge` module creates a job, in which the placeholders are replaced with the acts specific to that target in the batch.
51
+ For example, a target list (with “→” representing the Tab character) might be:
51
52
 
52
- ### Example
53
+ ```text
54
+ w3c→World Wide Web Consortium→https://www.w3.org/
55
+ moz→Mozilla Foundation→https://foundation.mozilla.org/en/
56
+ ```
53
57
 
54
- Here is an example illustrating how merging works.
58
+ ### Batches
55
59
 
56
- #### Script
60
+ Targets can be specified in a more complex way, too. That allows you to create jobs in which particular targets are handled distinctively in particular contexts. The more complex representation of a set of targets is a _batch_. Here is the start of a batch, showing its first target:
57
61
 
58
- Suppose you have created this script:
62
+ ```javaScript
63
+ {
64
+ id: 'clothing-stores',
65
+ what: 'clothing stores',
66
+ targets: [
67
+ {
68
+ id: 'acme',
69
+ what: 'Acme Clothes',
70
+ acts: {
71
+ public: [
72
+ {
73
+ type: 'launch'
74
+ },
75
+ {
76
+ type: 'url',
77
+ which: 'https://acmeclothes.com/',
78
+ what: 'Acme Clothes home page'
79
+ }
80
+ ],
81
+ private: [
82
+ {
83
+ type: 'launch'
84
+ },
85
+ {
86
+ type: 'url',
87
+ which: 'https://acmeclothes.com/login.html',
88
+ what: 'Acme Clothes login page'
89
+ },
90
+ {
91
+ type: 'text',
92
+ which: 'User Name',
93
+ what: 'tester34'
94
+ },
95
+ {
96
+ type: 'text',
97
+ which: 'Password',
98
+ what: '__TESTER34_PASSWORD__'
99
+ },
100
+ {
101
+ type: 'button',
102
+ which: 'Submit',
103
+ what: 'submit the login form'
104
+ },
105
+ {
106
+ type: 'wait',
107
+ which: 'title',
108
+ what: 'account'
109
+ }
110
+ ]
111
+ }
112
+ },
113
+
114
+ ]
115
+ }
116
+ ```
117
+
118
+ As shown, a batch, unlike a target list, defines named sequences of acts. They can be plugged into jobs, so various complex operations can be performed on each target.
119
+
120
+ ### Scripts
121
+
122
+ The generic, target-independent description of a job is _script_. A script can contain _placeholders_ that Testilo replaces with acts from a batch, creating one job per target. Thus, one script plus one batch can generate an unlimited number of jobs.
123
+
124
+ Here is a script:
59
125
 
60
126
  ```javaScript
61
127
  {
62
128
  id: 'ts25',
63
- what: 'Motion, Axe, and bulk',
129
+ what: 'Axe and QualWeb on account page',
64
130
  strict: true,
65
131
  timeLimit: 60,
66
132
  acts: [
67
133
  {
68
134
  type: 'placeholder',
69
- which: 'main',
70
- launch: 'webkit'
71
- },
72
- {
73
- type: 'test',
74
- which: 'motion',
75
- what: 'spontaneous change of content; requires webkit',
76
- delay: 2500,
77
- interval: 2500,
78
- count: 5
79
- },
80
- {
81
- type: 'placeholder',
82
- which: 'main',
135
+ which: 'private',
83
136
  launch: 'chromium'
84
137
  },
85
138
  {
86
139
  type: 'test',
87
140
  which: 'axe',
88
- withItems: true,
141
+ detailLevel: 1,
89
142
  rules: [],
90
- what: 'Axe core, all rules, with details'
143
+ what: 'Axe core, all rules'
91
144
  },
92
145
  {
93
146
  type: 'test',
94
- which: 'bulk',
95
- what: 'count of visible elements'
147
+ which: 'qualWeb',
148
+ withNewContent: false,
149
+ what: 'QualWeb, all rules'
96
150
  }
97
151
  ]
98
152
  }
99
153
  ```
100
154
 
101
- The `acts` array of this script contains three acts of type `test` and two acts of type `placeholder`.
155
+ This script has 2 acts. The first is a placeholder act. The above batch can be merged with this script to create jobs. In that case, the first job would launch a Chromium browser, navigate to the Acme login page, complete and submit the login form, wait for the account page to load, and then run the Axe tests. If the batch contained additional targets, additional jobs would be created, with the login actions for each target specified in the `private` array of the `acts` object of that target.
102
156
 
103
- #### Batch
157
+ As shown in this example, when a browser is launched by placeholder substitution, the script can determine the browser type (`chromium`, `firefox`, or `webkit`) by assigning a value to a `launch` property of the placeholder. That is useful, because sometimes it is the actions specified in a script that dictate which browser type is appropriate.
104
158
 
105
- Suppose you have also created this batch:
159
+ ### Target list to batch
106
160
 
107
- ```javaScript
108
- {
109
- id: 'weborgs',
110
- what: 'Web standards organizations',
111
- targets: [
112
- {
113
- id: 'mozilla',
114
- what: 'Mozilla Foundation',
115
- acts: {
116
- main: [
117
- {
118
- type: 'launch'
119
- },
120
- {
121
- type: 'url',
122
- which: 'https://foundation.mozilla.org/en/',
123
- what: 'Mozilla Foundation'
124
- }
125
- ]
126
- }
127
- },
128
- {
129
- id: 'w3c',
130
- what: 'World Wide Web Consortium',
131
- acts: {
132
- main: [
133
- {
134
- type: 'launch'
135
- },
136
- {
137
- type: 'url',
138
- which: 'https://www.w3.org/',
139
- what: 'World Wide Web Consortium'
140
- }
141
- ]
142
- }
143
- }
144
- ]
145
- }
146
- ```
161
+ If you have a target list, Testilo can convert it to a batch. The batch will contain, for each target, one array of acts named `main`, containing a `launch` act (depending on the script to specify the browser type) and a `url` act.
147
162
 
148
- This batch defines two targets.
163
+ ### Merge
149
164
 
150
- #### Isolation option
165
+ Testilo merges batches with scripts by means of a `merge` module.
151
166
 
152
- The `merge` module offers an isolation option. If you exercise it, the `merge` module will act as if the latest placeholder were again inserted after each target-modifying test, except where that test is the last act or the next act after it is a placeholder.
167
+ The `merge` module needs to be given a batch and a script. In addition, `merge` offers an isolation option. If you exercise it, the `merge` module will act as if the latest placeholder were **again** inserted after each target-modifying test act, except where that test act is the last act or where the next act after it is a placeholder.
153
168
 
154
169
  #### Output
155
170
 
156
- Suppose you ask for a merger of the above batch and script, **without** the isolation option. Then the first job produced by `merge` could be:
171
+ ##### Without isolation
172
+
173
+ Suppose you ask for a merger of the above batch and script, **without** the isolation option. Then the first job produced by `merge` will look like this:
157
174
 
158
175
  ```javaScript
159
176
  {
160
- id: '7izc1-ts25-mozilla',
161
- what: 'Motion, Axe, and bulk',
177
+ id: '7is3i-ts25-acme',
178
+ what: 'Axe on account page',
162
179
  strict: true,
163
180
  timeLimit: 60,
164
181
  acts: [
165
182
  {
166
183
  type: 'launch',
167
- which: 'webkit'
184
+ which: 'chromium'
168
185
  },
169
186
  {
170
187
  type: 'url',
171
- which: 'https://foundation.mozilla.org/en/',
172
- what: 'Mozilla Foundation'
173
- }
188
+ which: 'https://acmeclothes.com/login.html',
189
+ what: 'Acme Clothes login page'
190
+ },
174
191
  {
175
- type: 'test',
176
- which: 'motion',
177
- what: 'spontaneous change of content; requires webkit',
178
- delay: 2500,
179
- interval: 2500,
180
- count: 5
192
+ type: 'text',
193
+ which: 'User Name',
194
+ what: 'tester34'
181
195
  },
182
196
  {
183
- type: 'launch',
184
- which: 'chromium'
197
+ type: 'text',
198
+ which: 'Password',
199
+ what: '34SecretTester'
185
200
  },
186
201
  {
187
- type: 'url',
188
- which: 'https://foundation.mozilla.org/en/',
189
- what: 'Mozilla Foundation'
190
- }
202
+ type: 'button',
203
+ which: 'Submit',
204
+ what: 'submit the login form'
205
+ },
206
+ {
207
+ type: 'wait',
208
+ which: 'title',
209
+ what: 'account'
210
+ },
191
211
  {
192
212
  type: 'test',
193
213
  which: 'axe',
194
- withItems: true,
214
+ detailLevel: 1,
195
215
  rules: [],
196
- what: 'Axe core, all rules, with details'
216
+ what: 'Axe core, all rules'
197
217
  },
198
218
  {
199
219
  type: 'test',
200
- which: 'bulk',
201
- what: 'count of visible elements'
220
+ which: 'qualWeb',
221
+ withNewContent: false,
222
+ what: 'QualWeb, all rules'
202
223
  }
203
224
  ],
204
225
  sources: {
205
226
  script: 'ts25',
206
- batch: 'weborgs',
227
+ batch: 'clothing-stores',
207
228
  target: {
208
- id: 'mozilla',
209
- what: 'Mozilla Foundation'
229
+ id: 'acme',
230
+ what: 'Acme Clothes'
210
231
  },
211
232
  requester: 'you@yourdomain.tld'
212
233
  },
213
234
  creationTime: '2023-11-20T15:50:27',
214
- timeStamp: 'bizc1'
235
+ timeStamp: '7is3i'
215
236
  }
216
237
  ```
217
238
 
218
- Most of the properties of this job object come from your script and your batch. The acts of type `placeholder` in the script have been replaced with the specified (i.e. `main`) array of acts of the first target of the batch. In this case, that target has only one array of acts, and that array contains two acts, but a target could have multiple act arrays with distinct names, and an act array could include any number of acts or be empty.
239
+ Testilo has substituted the `private` acts from the `acme` target of the batch for the placeholder when creating the job. Testilo also has:
240
+ - let the script determine the browser type of the `launch` act.
241
+ - added a unique timestamp to the job.
242
+ - added the creation time to the job.
243
+ - given the job an ID that combines the timestamp with the script ID and the batch ID.
244
+ - inserted a `sources` property into the job, recording facts about the script, the batch, the target, and the email address given by the user who requested the merger.
219
245
 
220
- If the named array of acts includes an act of type `launch`, that act gets a `which` property, identical to the value of the `launch` property of the `placeholder` object. In this way, a placeholder can dictate which browser type is to be launched.
246
+ This is a valid Testaro job.
221
247
 
222
- The `merge` module has added other properties to the job:
223
- - a creation time
224
- - a timestamp, derived from the creation time
225
- - a job ID, composed of the timestamp, the script ID, and the target ID
226
- - a `sources` property, identifying the script, the batch, the target within the batch, and an email address obtained from the environment variable `process.env.REQUESTER`.
248
+ ##### With isolation
227
249
 
228
- This job is ready to be executed by Testaro.
250
+ If, however, you requested a merger **with** isolation, then Testilo would take cognizance of the fact that an `axe` test act is a target-modifying act. Testilo would therefore act as if another instance of the placeholder had been located in the script after the `axe` test act. So, copies of the same 6 acts that precede the `axe` test act would be inserted **after** the `axe` test act, too.
229
251
 
230
- If, however, you requested a merger **with** isolation, then `merge` would act as if another instance of
252
+ Of the 10 tools providing tests for Testaro, 6 are target-modifying:
253
+ - `axe`
254
+ - `continuum`
255
+ - `htmlcs`
256
+ - `ibm`
257
+ - `wave`
231
258
 
232
- ```javaScript
233
- {
234
- type: 'placeholder',
235
- which: 'main',
236
- launch: 'chromium'
237
- },
238
- ```
239
-
240
- were located in the script after the `axe` test, because the `axe` test is target-modifying and could therefore change the result of the `bulk` test that follows it. So, additional acts of type `launch` and `url` would appear after the `axe` test in the job.
241
-
242
- ### Invocation
259
+ #### Invocation
243
260
 
244
261
  There are two ways to use the `merge` module.
245
262
 
246
- #### By a module
263
+ ##### By a module
247
264
 
248
265
  A module can invoke `merge` in this way:
249
266
 
@@ -254,7 +271,7 @@ const jobs = merge(script, batch, requester, true);
254
271
 
255
272
  This invocation references `script`, `batch`, and `requester` variables that the module must have already defined. The `script` and `batch` variables are a script object and a batch object, respectively. The `requester` variable is an email address. The fourth argument is a boolean, specifying whether to perform isolation; a missing fourth argument is equivalent to `false`. The `merge()` function of the `merge` module generates jobs and returns them in an array. The invoking module can further dispose of the jobs as needed.
256
273
 
257
- #### By a user
274
+ ##### By a user
258
275
 
259
276
  A user can invoke `merge` in this way:
260
277
 
@@ -267,26 +284,16 @@ A user can invoke `merge` in this way:
267
284
 
268
285
  In these statements, replace `s` and `b` with the base names of the script and batch files, respectively. For example, if the script file is named `ts25.json`, then replace `s` with `ts25`. Replace `e` with an email address, or with an empty string if the environment variable `process.env.REQUESTER` exists and you want to use it.
269
286
 
270
- The first statement will cause a merger with isolation.
271
- The second and third statements will cause a merger without isolation.
287
+ The first statement will cause a merger **with** isolation.
288
+ The second and third statements will cause a merger **without* isolation.
272
289
 
273
290
  The `call` module will retrieve the named script and batch from their respective directories.
274
291
  The `merge` module will create an array of jobs.
275
- The `call` module will save the jobs in the `todo` subdirectory of the `process.env.JOBDIR` directory.
276
-
277
- ### Validation
278
-
279
- To test the `merge` module, in the project directory you can execute the statement `node validation/merge/validate`. All logging statements should begin with “Success” and none should begin with “ERROR”.
280
-
281
- ### Examples
282
-
283
- There are script and batch examples in the `specs` directory.
292
+ The `call` module will save the jobs as JSON files in the `todo` subdirectory of the `process.env.JOBDIR` directory.
284
293
 
285
- The script example (with ID `ts21`) performs all of the tests available in Testaro, including the tests of 9 dependent tools. It contains 2 `placeholder` acts, one that specifies the Webkit browser for any `launch` act and one that specifies the Chromium browser for any `launch` act.
294
+ #### Validation
286
295
 
287
- The batch example (with ID `orgs`) specifies 2 targets.
288
- - One of them has the URL `https://example.com` and requires each placeholder to be replaced with 2 acts: a `launch` act and a `url` act.
289
- - The other has the URL `https://www.w3.org/WAI/ARIA/apg/example-index/accordion/accordion` and requires each placeholder to be replaced with 3 acts: a `launch` act, a `url` act, and a `button` act.
296
+ To test the `merge` module, in the project directory you can execute the statement `node validation/merge/validate`. If `merge` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
290
297
 
291
298
  ## Report scoring
292
299
 
@@ -307,9 +314,14 @@ Thus, a report produced by Testaro contains these properties:
307
314
  - `timeStamp`
308
315
  - `jobData`
309
316
 
310
- Testilo can add scores to a report. In this way, a report can not only detail successes and failures of individual tests but also assign scores to those results and combine the partial scores into total scores. The scores are contained in a new `score` property that Testilo adds to a report.
317
+ Testilo can enhance such a report in three ways:
318
+ - adding scores
319
+ - creating digests
320
+ - creating comprisons
311
321
 
312
- The `score` module scores a report. Its `score()` function takes two arguments:
322
+ To add scores to reports, the `score` module of Testilo performs computations on the test results and adds a `score` property to each report.
323
+
324
+ The `score()` function of the `score` module takes two arguments:
313
325
  - a scoring function
314
326
  - an array of report objects
315
327
 
@@ -325,58 +337,44 @@ A module can invoke `score` in this way:
325
337
 
326
338
  ```javaScript
327
339
  const {score} = require('testilo/score');
328
- const {scorer} = require('testilo/procs/score/sp25a');
329
- const scoredReports = score(scorer, rawReports);
340
+ const {scorer} = require('testilo/procs/score/tsp25');
341
+ score(scorer, reports);
330
342
  ```
331
343
 
332
- The first argument to `score()` is a scoring function. In this example, it has been obtained from a JSON file in the Testilo package, but it could be a custom function. The second argument to `score()` is an array of report objects.
344
+ The first argument to `score()` is a scoring function. In this example, it has been obtained from a module in the Testilo package, but it could be a custom function. The second argument to `score()` is an array of report objects. The invoking module can further dispose of the scored reports as needed.
333
345
 
334
346
  #### By a user
335
347
 
336
348
  A user can invoke `score` in this way:
337
349
 
338
350
  ```bash
339
- node call score tsp25a
340
- node call score tsp25a 75
351
+ node call score tsp25
352
+ node call score tsp25 75m
341
353
  ```
342
354
 
343
355
  When a user invokes `score` in this example, the `call` module:
344
- - gets the scoring module `tsp25a` from its JSON file `tsp25a.json` in the `score` subdirectory of the `process.env.FUNCTIONDIR` directory
345
- - gets the reports from the `raw` subdirectory of the `process.env.REPORTDIR` directory
346
- - writes the scored reports to the `scored` subdirectory of the `process.env.REPORTDIR` directory
356
+ - gets the scoring module `tsp25` from its JSON file `tsp25.json` in the `score` subdirectory of the `process.env.FUNCTIONDIR` directory.
357
+ - gets the reports from the `raw` subdirectory of the `process.env.REPORTDIR` directory.
358
+ - writes the scored reports in JSON format to the `scored` subdirectory of the `process.env.REPORTDIR` directory.
347
359
 
348
- The optional third argument to call (`75` in this example) is a report selector. Without the argument, `call` gets all the reports in the `raw` subdirectory. With the argument, `call` gets only those reports whose names begin with the argument string.
360
+ The optional third argument to call (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `raw` subdirectory. With the argument, `call` gets only those reports whose names begin with the argument string.
349
361
 
350
362
  ### Validation
351
363
 
352
- To test the `score` module, in the project directory you can execute the statement `node validation/score/validate`. All logging statements should begin with “Success” and none should begin with “ERROR”.
353
-
354
- ### Comprehensive example
355
-
356
- The module `procs/score/tsp21.js` is an example of a comprehensive score proc. It is designed to score reports produced by Testaro from jobs that Testilo creates from the `ts21` script. That script causes Testaro to perform 1351 tests belonging to Testaro and nine other tools integrated into Testaro as dependencies.
364
+ To test the `score` module, in the project directory you can execute the statement `node validation/score/validate`. If `score` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
357
365
 
358
- The `tsp21` score proc combines the results of those tests into a single score for each report. The `score` property that the proc adds to the report shows how the individual test results are aggregated into a score.
359
-
360
- The `tsp21` score proc imports the `procs/score/tic21.js` module, which classifies all the tests into 255 “issues”. Any two or more tests classified into the same issue are deemed to be approximately equivalent in intent. Using this classification, the score proc tabulates the test results by issue, so that all instances of the issue, regardless of which tool discovered the instances, are reported together.
361
-
362
- The issue classification in `tic21` assigns weights to the issues to reflect their estimated severities,and the scoring algorithm of `tsp21` adopts a compromise between issue-based and tool-based summation. Thus, if 3 tools discover 20 instances of an issue, the score increases (i.e. becomes worse) by more than it would if only 1 tool had discovered them, but less than 3 times as much. The motivations are that discovery of issue instances by multiple tools increases confidence that the issues are real, and passing accessibility tests is per se beneficial because it decreases risks associated with alleged inaccessibility.
363
-
364
- The `tsp21` score proc does **not** attempt to identify instances of issues. Thus, if tool A discovers 4 instances and tool B discovers 7 instances of some issue, `tsp21` does not attempt to say which of the 4 discovered by A are a subset of the 7 discovered by B. Instead,`tsp21` naïvely acts as if the 4 are all a subset of the 7, even though in reality the two sets might be entirely disjoint.
365
-
366
- In the `tsp21` scoring algorithm, the failure of a tool or of a Testaro test to run on a page increases the score of the page, and messages logged by the browser while tests are being performed on a page, especially error messages, also increase the score.
367
-
368
- ## Report digesting
366
+ ## Rport digesting
369
367
 
370
368
  ### Introduction
371
369
 
372
- Reports created by Testaro, both originally and after they are scored, are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
370
+ Reports from Testaro are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
373
371
 
374
372
  The `digest` module digests a scored report. Its `digest()` function takes three arguments:
375
373
  - a digest template
376
374
  - a digesting function
377
- - an array of report objects
375
+ - an array of scored report objects
378
376
 
379
- The digest template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by texts, becomes the digest. The digesting function defines the rules for replacing the placeholders with texts. The Testilo package contains a `procs/digest` directory, in which there are subdirectories containing pairs of templates and modules that export digesting functions. You can use one of those pairs, or you can create your own.
377
+ The digest template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by computed values, becomes the digest. The digesting function defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/digest` directory, in which there are subdirectories containing pairs of templates and modules that export digesting functions. You can use one of those pairs, or you can create your own.
380
378
 
381
379
  ### Invocation
382
380
 
@@ -389,36 +387,36 @@ A module can invoke `digest` in this way:
389
387
  ```javaScript
390
388
  const {digest} = require('testilo/digest');
391
389
  const digesterDir = `${process.env.FUNCTIONDIR}/digest/dp25a`;
392
- fs.readFile(`${digesterDir}/index.html`)
390
+ fs.readFile(`${digesterDir}/index.html`, 'utf8')
393
391
  .then(template => {
394
392
  const {digester} = require(`${digesterDir}/index`);
395
393
  const digestedReports = digest(template, digester, scoredReports);
396
394
  });
397
395
  ```
398
396
 
399
- The first two arguments to `digest()` are a digest template and a digesting function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `digest()` is an array of report objects.
397
+ The first two arguments to `digest()` are a digest template and a digesting function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `digest()` is an array of scored report objects. The `digest()` function returns an array of digested reports. The invoking module can further dispose of the digested reports as needed.
400
398
 
401
399
  #### By a user
402
400
 
403
401
  A user can invoke `digest` in this way:
404
402
 
405
403
  ```bash
406
- node call digest dp25a
407
- node call digest dp25a 75
404
+ node call digest tdp25
405
+ node call digest tdp25 75m
408
406
  ```
409
407
 
410
408
  When a user invokes `digest` in this example, the `call` module:
411
- - gets the template and the digesting module from subdirectory `dp25a` in the `digest` subdirectory of the `process.env.FUNCTIONDIR` directory
412
- - gets the reports from the `scored` subdirectory of the `process.env.REPORTDIR` directory
413
- - writes the digested reports to the `digested` subdirectory of the `process.env.REPORTDIR` directory
409
+ - gets the template and the digesting module from subdirectory `tdp25` in the `digest` subdirectory of the `process.env.FUNCTIONDIR` directory.
410
+ - gets the reports from the `scored` subdirectory of the `process.env.REPORTDIR` directory.
411
+ - writes the digested reports to the `digested` subdirectory of the `process.env.REPORTDIR` directory.
414
412
 
415
- The optional third argument to call (`75` in this example) is a report selector. Without the argument, `call` gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory. With the argument, `call` gets only those reports whose names begin with the argument string.
413
+ The optional third argument to call (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory. With the argument, `call` gets only those reports whose names begin with the argument string.
416
414
 
417
415
  The digests created by `digest` are HTML files, and they expect a `style.css` file to exist in their directory. The `reports/digested/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where digested reports are written.
418
416
 
419
417
  ### Validation
420
418
 
421
- To test the `digest` module, in the project directory you can execute the statement `node validation/digest/validate`. All logging statements should begin with “Success” and none should begin with “ERROR”.
419
+ To test the `digest` module, in the project directory you can execute the statement `node validation/digest/validate`. If `digest` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
422
420
 
423
421
  ### Report comparison
424
422
 
@@ -429,7 +427,7 @@ The `compare` module compares the scores in a collection of scored reports. Its
429
427
  - a comparison function
430
428
  - an array of scored reports
431
429
 
432
- The comparison template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by texts, becomes the comparative report. The comparison function defines the rules for replacing the placeholders with texts. The Testilo package contains a `procs/compare` directory, in which there are subdirectories containing pairs of templates and modules that export comparison functions. You can use one of those pairs, or you can create your own.
430
+ The comparison template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by computed values, becomes the comparative report. The comparison function defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/compare` directory, in which there are subdirectories containing pairs of templates and modules that export comparison functions. You can use one of those pairs, or you can create your own.
433
431
 
434
432
  ### Invocation
435
433
 
@@ -442,31 +440,31 @@ A module can invoke `compare` in this way:
442
440
  ```javaScript
443
441
  const fs = require('fs/promises);
444
442
  const {compare} = require('testilo/compare');
445
- const comparerDir = `${process.env.FUNCTIONDIR}/compare/cp25a`;
443
+ const comparerDir = `${process.env.FUNCTIONDIR}/compare/tcp25`;
446
444
  fs.readFile(`${comparerDir}/index.html`)
447
445
  .then(template => {
448
446
  const {comparer} = require(`${comparerDir}/index`);
449
- const comparativeReports = compare(template, comparer, scoredReports);
447
+ const comparativeReport = compare(template, comparer, scoredReports);
450
448
  });
451
449
  ```
452
450
 
453
- The first two arguments to `compare()` are a template and a comparison function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `compare()` is an array of report objects.
451
+ The first two arguments to `compare()` are a template and a comparison function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `compare()` is an array of report objects. The `compare()` function returns a comparative report. The invoking module can further dispose of the comparative report as needed.
454
452
 
455
453
  #### By a user
456
454
 
457
455
  A user can invoke `compare` in this way:
458
456
 
459
457
  ```bash
460
- node call compare cp25a legislators
458
+ node call compare tcp25 legislators
461
459
  ```
462
460
 
463
461
  When a user invokes `compare` in this example, the `call` module:
464
- - gets the template and the comparison module from subdirectory `cp25a` of the subdirectory `compare` in the `process.env.FUNCTIONDIR` directory
465
- - gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory
466
- - writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `process.env.REPORTDIR` directory
462
+ - gets the template and the comparison module from subdirectory `tcp25` of the subdirectory `compare` in the `process.env.FUNCTIONDIR` directory.
463
+ - gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory.
464
+ - writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `process.env.REPORTDIR` directory.
467
465
 
468
- The comparative reports created by `compare` are HTML files, and they expect a `style.css` file to exist in their directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
466
+ The comparative report created by `compare` is an HTML file, and it expects a `style.css` file to exist in its directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
469
467
 
470
468
  ### Validation
471
469
 
472
- To test the `compare` module, in the project directory you can execute the statement `node validation/compare/validate`. All logging statements should begin with “Success” and none should begin with “ERROR”.
470
+ To test the `compare` module, in the project directory you can execute the statement `node validation/compare/validate`. If `compare` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
package/batch.js ADDED
@@ -0,0 +1,60 @@
1
+ /*
2
+ batch.js
3
+ Converts a target list to a batch.
4
+ Arguments:
5
+ 0. batch ID
6
+ 1. batch description
7
+ 2. target list
8
+ */
9
+
10
+ // ########## FUNCTIONS
11
+
12
+ // Converts a target list to a batch and returns the batch.
13
+ exports.batch = (id, what, targetList) => {
14
+ // If the arguments are valid:
15
+ if (
16
+ typeof id === 'string'
17
+ && id.length
18
+ && typeof what === 'string'
19
+ && what.length
20
+ && Array.isArray(targetList)
21
+ && targetList.length
22
+ && targetList.every(
23
+ target => Array.isArray(target) && target.every(item => typeof item === string && item.length)
24
+ )
25
+ ) {
26
+ // Initialize the batch.
27
+ const batch = {
28
+ id,
29
+ what,
30
+ targets: []
31
+ };
32
+ // For each target:
33
+ targetList.forEach(target => {
34
+ // Add it to the batch.
35
+ batch.targets.push({
36
+ id: target[0],
37
+ what: target[1],
38
+ acts: {
39
+ main: [
40
+ {
41
+ type: 'launch'
42
+ },
43
+ {
44
+ type: 'url',
45
+ which: target[2],
46
+ what: target[1]
47
+ }
48
+ ]
49
+ }
50
+ });
51
+ });
52
+ // Return the batch.
53
+ return batch;
54
+ }
55
+ // Otherwise, i.e. if the arguments are invalid:
56
+ else {
57
+ // Return this.
58
+ return null;
59
+ }
60
+ };