testilo 10.3.0 → 11.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +188 -190
- package/batch.js +60 -0
- package/call.js +9 -4
- package/compare.js +2 -2
- package/merge.js +0 -6
- package/package.json +1 -1
- package/procs/compare/cp20sqrt/index.js +7 -10
- package/procs/compare/{cp15b → tcp24}/index.html +5 -5
- package/procs/compare/{cp12a → tcp24}/index.js +13 -20
- package/procs/score/{tic24.js → tic25.js} +40 -27
- package/procs/score/{tic23.js → tic26.js} +264 -229
- package/procs/score/tsp24.js +8 -2
- package/procs/score/tsp26.js +120 -0
- package/score.js +2 -8
- package/procs/compare/cp0/index.html +0 -46
- package/procs/compare/cp0/index.js +0 -70
- package/procs/compare/cp1/index.html +0 -46
- package/procs/compare/cp1/index.js +0 -71
- package/procs/compare/cp12a/index.html +0 -47
- package/procs/compare/cp12b/index.html +0 -47
- package/procs/compare/cp12b/index.js +0 -71
- package/procs/compare/cp14a/index.html +0 -47
- package/procs/compare/cp14a/index.js +0 -71
- package/procs/compare/cp15a/index.html +0 -47
- package/procs/compare/cp15a/index.js +0 -71
- package/procs/compare/cp15b/index.js +0 -74
- package/procs/compare/cp16a/index.html +0 -47
- package/procs/compare/cp16a/index.js +0 -74
- package/procs/compare/cp16b/index.html +0 -47
- package/procs/compare/cp16b/index.js +0 -74
- package/procs/digest/dp10a/index.html +0 -137
- package/procs/digest/dp10a/index.js +0 -402
- package/procs/digest/dp10b/index.html +0 -74
- package/procs/digest/dp10b/index.js +0 -130
- package/procs/digest/dp10c/index.html +0 -55
- package/procs/digest/dp10c/index.js +0 -129
- package/procs/digest/dp11a/index.html +0 -76
- package/procs/digest/dp11a/index.js +0 -127
- package/procs/digest/dp12a/index.html +0 -78
- package/procs/digest/dp12a/index.js +0 -126
- package/procs/digest/dp12b/index.html +0 -78
- package/procs/digest/dp12b/index.js +0 -126
- package/procs/digest/dp13a/index.html +0 -78
- package/procs/digest/dp13a/index.js +0 -126
- package/procs/digest/dp14a/index.html +0 -79
- package/procs/digest/dp14a/index.js +0 -126
- package/procs/digest/dp15a/index.html +0 -80
- package/procs/digest/dp15a/index.js +0 -126
- package/procs/digest/dp16a/index.html +0 -80
- package/procs/digest/dp16a/index.js +0 -126
- package/procs/digest/dp16b/index.html +0 -80
- package/procs/digest/dp16b/index.js +0 -126
- package/procs/digest/dp18a/index.html +0 -80
- package/procs/digest/dp18a/index.js +0 -129
- package/procs/digest/dp20c/index.html +0 -54
- package/procs/digest/dp20c/index.js +0 -117
- package/procs/digest/dp20d/index.html +0 -54
- package/procs/digest/dp20d/index.js +0 -121
- package/procs/digest/tdp09a/index.html +0 -126
- package/procs/digest/tdp09a/index.js +0 -381
- package/procs/digest/tdp22/index.html +0 -54
- package/procs/digest/tdp22/index.js +0 -120
- package/procs/score/tic21.js +0 -6868
- package/procs/score/tic22.js +0 -6911
- package/procs/score/tsp21.js +0 -720
- package/procs/score/tsp22.js +0 -720
- package/procs/score/tsp23.js +0 -720
- package/specs/batches/orgs.json +0 -43
- package/specs/scripts/ts21.json +0 -213
- package/specs/scripts/ts23.json +0 -219
- package/specs/scripts/ts24.json +0 -225
package/README.md
CHANGED
|
@@ -36,214 +36,231 @@ Testaro executes _jobs_. In a job, Testaro performs _acts_ (tests and other oper
|
|
|
36
36
|
You can create a job for Testaro directly, without using Testilo.
|
|
37
37
|
|
|
38
38
|
Testilo can, however, make job preparation more efficient in two scenarios:
|
|
39
|
-
- A common use case is to define a battery of tests and to have those tests performed on multiple targets. In that case,
|
|
40
|
-
- Some tests operate on a copy of a target and modify that copy. Usually, one wants test isolation: The results of a test do not depend on any previously performed tests. To ensure test isolation, a job containing such target-modifying
|
|
39
|
+
- A common use case is to define a battery of tests and to have those same tests performed on multiple targets. In that case, it is efficient to create multiple jobs, one per target. The jobs are identical, except for the target-specific acts (including navigating to targets). If you tell Testilo about the tests and the targets, Testilo can create such collections of jobs for you.
|
|
40
|
+
- Some tests operate on a copy of a target and modify that copy. Usually, one wants test isolation: The results of a test do not depend on any previously performed tests. To ensure test isolation, a job containing such target-modifying test acts must follow them with acts that restore the target to its pre-test state. Testilo can insert the required target-restoring acts into each job after target-modifying test act.
|
|
41
41
|
|
|
42
|
-
The `merge`
|
|
42
|
+
The `batch` and `merge` modules performs these services.
|
|
43
43
|
|
|
44
|
-
###
|
|
44
|
+
### Target lists
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
-
|
|
48
|
-
-
|
|
46
|
+
The simplest version of a list of targets is _target list_. It is stored as a tab-delimited text file, with one line per target. Each line contains 3 items, with tabs between them:
|
|
47
|
+
- An ID for the target
|
|
48
|
+
- A description of the target
|
|
49
|
+
- The URL of the target
|
|
49
50
|
|
|
50
|
-
|
|
51
|
+
For example, a target list (with “→” representing the Tab character) might be:
|
|
51
52
|
|
|
52
|
-
|
|
53
|
+
```text
|
|
54
|
+
w3c→World Wide Web Consortium→https://www.w3.org/
|
|
55
|
+
moz→Mozilla Foundation→https://foundation.mozilla.org/en/
|
|
56
|
+
```
|
|
53
57
|
|
|
54
|
-
|
|
58
|
+
### Batches
|
|
55
59
|
|
|
56
|
-
|
|
60
|
+
Targets can be specified in a more complex way, too. That allows you to create jobs in which particular targets are handled distinctively in particular contexts. The more complex representation of a set of targets is a _batch_. Here is the start of a batch, showing its first target:
|
|
57
61
|
|
|
58
|
-
|
|
62
|
+
```javaScript
|
|
63
|
+
{
|
|
64
|
+
id: 'clothing-stores',
|
|
65
|
+
what: 'clothing stores',
|
|
66
|
+
targets: [
|
|
67
|
+
{
|
|
68
|
+
id: 'acme',
|
|
69
|
+
what: 'Acme Clothes',
|
|
70
|
+
acts: {
|
|
71
|
+
public: [
|
|
72
|
+
{
|
|
73
|
+
type: 'launch'
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
type: 'url',
|
|
77
|
+
which: 'https://acmeclothes.com/',
|
|
78
|
+
what: 'Acme Clothes home page'
|
|
79
|
+
}
|
|
80
|
+
],
|
|
81
|
+
private: [
|
|
82
|
+
{
|
|
83
|
+
type: 'launch'
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
type: 'url',
|
|
87
|
+
which: 'https://acmeclothes.com/login.html',
|
|
88
|
+
what: 'Acme Clothes login page'
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
type: 'text',
|
|
92
|
+
which: 'User Name',
|
|
93
|
+
what: 'tester34'
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
type: 'text',
|
|
97
|
+
which: 'Password',
|
|
98
|
+
what: '__TESTER34_PASSWORD__'
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
type: 'button',
|
|
102
|
+
which: 'Submit',
|
|
103
|
+
what: 'submit the login form'
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
type: 'wait',
|
|
107
|
+
which: 'title',
|
|
108
|
+
what: 'account'
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
…
|
|
114
|
+
]
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
As shown, a batch, unlike a target list, defines named sequences of acts. They can be plugged into jobs, so various complex operations can be performed on each target.
|
|
119
|
+
|
|
120
|
+
### Scripts
|
|
121
|
+
|
|
122
|
+
The generic, target-independent description of a job is _script_. A script can contain _placeholders_ that Testilo replaces with acts from a batch, creating one job per target. Thus, one script plus one batch can generate an unlimited number of jobs.
|
|
123
|
+
|
|
124
|
+
Here is a script:
|
|
59
125
|
|
|
60
126
|
```javaScript
|
|
61
127
|
{
|
|
62
128
|
id: 'ts25',
|
|
63
|
-
what: '
|
|
129
|
+
what: 'Axe and QualWeb on account page',
|
|
64
130
|
strict: true,
|
|
65
131
|
timeLimit: 60,
|
|
66
132
|
acts: [
|
|
67
133
|
{
|
|
68
134
|
type: 'placeholder',
|
|
69
|
-
which: '
|
|
70
|
-
launch: 'webkit'
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
type: 'test',
|
|
74
|
-
which: 'motion',
|
|
75
|
-
what: 'spontaneous change of content; requires webkit',
|
|
76
|
-
delay: 2500,
|
|
77
|
-
interval: 2500,
|
|
78
|
-
count: 5
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
type: 'placeholder',
|
|
82
|
-
which: 'main',
|
|
135
|
+
which: 'private',
|
|
83
136
|
launch: 'chromium'
|
|
84
137
|
},
|
|
85
138
|
{
|
|
86
139
|
type: 'test',
|
|
87
140
|
which: 'axe',
|
|
88
|
-
|
|
141
|
+
detailLevel: 1,
|
|
89
142
|
rules: [],
|
|
90
|
-
what: 'Axe core, all rules
|
|
143
|
+
what: 'Axe core, all rules'
|
|
91
144
|
},
|
|
92
145
|
{
|
|
93
146
|
type: 'test',
|
|
94
|
-
which: '
|
|
95
|
-
|
|
147
|
+
which: 'qualWeb',
|
|
148
|
+
withNewContent: false,
|
|
149
|
+
what: 'QualWeb, all rules'
|
|
96
150
|
}
|
|
97
151
|
]
|
|
98
152
|
}
|
|
99
153
|
```
|
|
100
154
|
|
|
101
|
-
|
|
155
|
+
This script has 2 acts. The first is a placeholder act. The above batch can be merged with this script to create jobs. In that case, the first job would launch a Chromium browser, navigate to the Acme login page, complete and submit the login form, wait for the account page to load, and then run the Axe tests. If the batch contained additional targets, additional jobs would be created, with the login actions for each target specified in the `private` array of the `acts` object of that target.
|
|
102
156
|
|
|
103
|
-
|
|
157
|
+
As shown in this example, when a browser is launched by placeholder substitution, the script can determine the browser type (`chromium`, `firefox`, or `webkit`) by assigning a value to a `launch` property of the placeholder. That is useful, because sometimes it is the actions specified in a script that dictate which browser type is appropriate.
|
|
104
158
|
|
|
105
|
-
|
|
159
|
+
### Target list to batch
|
|
106
160
|
|
|
107
|
-
|
|
108
|
-
{
|
|
109
|
-
id: 'weborgs',
|
|
110
|
-
what: 'Web standards organizations',
|
|
111
|
-
targets: [
|
|
112
|
-
{
|
|
113
|
-
id: 'mozilla',
|
|
114
|
-
what: 'Mozilla Foundation',
|
|
115
|
-
acts: {
|
|
116
|
-
main: [
|
|
117
|
-
{
|
|
118
|
-
type: 'launch'
|
|
119
|
-
},
|
|
120
|
-
{
|
|
121
|
-
type: 'url',
|
|
122
|
-
which: 'https://foundation.mozilla.org/en/',
|
|
123
|
-
what: 'Mozilla Foundation'
|
|
124
|
-
}
|
|
125
|
-
]
|
|
126
|
-
}
|
|
127
|
-
},
|
|
128
|
-
{
|
|
129
|
-
id: 'w3c',
|
|
130
|
-
what: 'World Wide Web Consortium',
|
|
131
|
-
acts: {
|
|
132
|
-
main: [
|
|
133
|
-
{
|
|
134
|
-
type: 'launch'
|
|
135
|
-
},
|
|
136
|
-
{
|
|
137
|
-
type: 'url',
|
|
138
|
-
which: 'https://www.w3.org/',
|
|
139
|
-
what: 'World Wide Web Consortium'
|
|
140
|
-
}
|
|
141
|
-
]
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
]
|
|
145
|
-
}
|
|
146
|
-
```
|
|
161
|
+
If you have a target list, Testilo can convert it to a batch. The batch will contain, for each target, one array of acts named `main`, containing a `launch` act (depending on the script to specify the browser type) and a `url` act.
|
|
147
162
|
|
|
148
|
-
|
|
163
|
+
### Merge
|
|
149
164
|
|
|
150
|
-
|
|
165
|
+
Testilo merges batches with scripts by means of a `merge` module.
|
|
151
166
|
|
|
152
|
-
The `merge` module offers an isolation option. If you exercise it, the `merge` module will act as if the latest placeholder were again inserted after each target-modifying test, except where that test is the last act or the next act after it is a placeholder.
|
|
167
|
+
The `merge` module needs to be given a batch and a script. In addition, `merge` offers an isolation option. If you exercise it, the `merge` module will act as if the latest placeholder were **again** inserted after each target-modifying test act, except where that test act is the last act or where the next act after it is a placeholder.
|
|
153
168
|
|
|
154
169
|
#### Output
|
|
155
170
|
|
|
156
|
-
|
|
171
|
+
##### Without isolation
|
|
172
|
+
|
|
173
|
+
Suppose you ask for a merger of the above batch and script, **without** the isolation option. Then the first job produced by `merge` will look like this:
|
|
157
174
|
|
|
158
175
|
```javaScript
|
|
159
176
|
{
|
|
160
|
-
id: '
|
|
161
|
-
what: '
|
|
177
|
+
id: '7is3i-ts25-acme',
|
|
178
|
+
what: 'Axe on account page',
|
|
162
179
|
strict: true,
|
|
163
180
|
timeLimit: 60,
|
|
164
181
|
acts: [
|
|
165
182
|
{
|
|
166
183
|
type: 'launch',
|
|
167
|
-
which: '
|
|
184
|
+
which: 'chromium'
|
|
168
185
|
},
|
|
169
186
|
{
|
|
170
187
|
type: 'url',
|
|
171
|
-
which: 'https://
|
|
172
|
-
what: '
|
|
173
|
-
}
|
|
188
|
+
which: 'https://acmeclothes.com/login.html',
|
|
189
|
+
what: 'Acme Clothes login page'
|
|
190
|
+
},
|
|
174
191
|
{
|
|
175
|
-
type: '
|
|
176
|
-
which: '
|
|
177
|
-
what: '
|
|
178
|
-
delay: 2500,
|
|
179
|
-
interval: 2500,
|
|
180
|
-
count: 5
|
|
192
|
+
type: 'text',
|
|
193
|
+
which: 'User Name',
|
|
194
|
+
what: 'tester34'
|
|
181
195
|
},
|
|
182
196
|
{
|
|
183
|
-
type: '
|
|
184
|
-
which: '
|
|
197
|
+
type: 'text',
|
|
198
|
+
which: 'Password',
|
|
199
|
+
what: '34SecretTester'
|
|
185
200
|
},
|
|
186
201
|
{
|
|
187
|
-
type: '
|
|
188
|
-
which: '
|
|
189
|
-
what: '
|
|
190
|
-
}
|
|
202
|
+
type: 'button',
|
|
203
|
+
which: 'Submit',
|
|
204
|
+
what: 'submit the login form'
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
type: 'wait',
|
|
208
|
+
which: 'title',
|
|
209
|
+
what: 'account'
|
|
210
|
+
},
|
|
191
211
|
{
|
|
192
212
|
type: 'test',
|
|
193
213
|
which: 'axe',
|
|
194
|
-
|
|
214
|
+
detailLevel: 1,
|
|
195
215
|
rules: [],
|
|
196
|
-
what: 'Axe core, all rules
|
|
216
|
+
what: 'Axe core, all rules'
|
|
197
217
|
},
|
|
198
218
|
{
|
|
199
219
|
type: 'test',
|
|
200
|
-
which: '
|
|
201
|
-
|
|
220
|
+
which: 'qualWeb',
|
|
221
|
+
withNewContent: false,
|
|
222
|
+
what: 'QualWeb, all rules'
|
|
202
223
|
}
|
|
203
224
|
],
|
|
204
225
|
sources: {
|
|
205
226
|
script: 'ts25',
|
|
206
|
-
batch: '
|
|
227
|
+
batch: 'clothing-stores',
|
|
207
228
|
target: {
|
|
208
|
-
id: '
|
|
209
|
-
what: '
|
|
229
|
+
id: 'acme',
|
|
230
|
+
what: 'Acme Clothes'
|
|
210
231
|
},
|
|
211
232
|
requester: 'you@yourdomain.tld'
|
|
212
233
|
},
|
|
213
234
|
creationTime: '2023-11-20T15:50:27',
|
|
214
|
-
timeStamp: '
|
|
235
|
+
timeStamp: '7is3i'
|
|
215
236
|
}
|
|
216
237
|
```
|
|
217
238
|
|
|
218
|
-
|
|
239
|
+
Testilo has substituted the `private` acts from the `acme` target of the batch for the placeholder when creating the job. Testilo also has:
|
|
240
|
+
- let the script determine the browser type of the `launch` act.
|
|
241
|
+
- added a unique timestamp to the job.
|
|
242
|
+
- added the creation time to the job.
|
|
243
|
+
- given the job an ID that combines the timestamp with the script ID and the batch ID.
|
|
244
|
+
- inserted a `sources` property into the job, recording facts about the script, the batch, the target, and the email address given by the user who requested the merger.
|
|
219
245
|
|
|
220
|
-
|
|
246
|
+
This is a valid Testaro job.
|
|
221
247
|
|
|
222
|
-
|
|
223
|
-
- a creation time
|
|
224
|
-
- a timestamp, derived from the creation time
|
|
225
|
-
- a job ID, composed of the timestamp, the script ID, and the target ID
|
|
226
|
-
- a `sources` property, identifying the script, the batch, the target within the batch, and an email address obtained from the environment variable `process.env.REQUESTER`.
|
|
248
|
+
##### With isolation
|
|
227
249
|
|
|
228
|
-
|
|
250
|
+
If, however, you requested a merger **with** isolation, then Testilo would take cognizance of the fact that an `axe` test act is a target-modifying act. Testilo would therefore act as if another instance of the placeholder had been located in the script after the `axe` test act. So, copies of the same 6 acts that precede the `axe` test act would be inserted **after** the `axe` test act, too.
|
|
229
251
|
|
|
230
|
-
|
|
252
|
+
Of the 10 tools providing tests for Testaro, 6 are target-modifying:
|
|
253
|
+
- `axe`
|
|
254
|
+
- `continuum`
|
|
255
|
+
- `htmlcs`
|
|
256
|
+
- `ibm`
|
|
257
|
+
- `wave`
|
|
231
258
|
|
|
232
|
-
|
|
233
|
-
{
|
|
234
|
-
type: 'placeholder',
|
|
235
|
-
which: 'main',
|
|
236
|
-
launch: 'chromium'
|
|
237
|
-
},
|
|
238
|
-
```
|
|
239
|
-
|
|
240
|
-
were located in the script after the `axe` test, because the `axe` test is target-modifying and could therefore change the result of the `bulk` test that follows it. So, additional acts of type `launch` and `url` would appear after the `axe` test in the job.
|
|
241
|
-
|
|
242
|
-
### Invocation
|
|
259
|
+
#### Invocation
|
|
243
260
|
|
|
244
261
|
There are two ways to use the `merge` module.
|
|
245
262
|
|
|
246
|
-
|
|
263
|
+
##### By a module
|
|
247
264
|
|
|
248
265
|
A module can invoke `merge` in this way:
|
|
249
266
|
|
|
@@ -254,7 +271,7 @@ const jobs = merge(script, batch, requester, true);
|
|
|
254
271
|
|
|
255
272
|
This invocation references `script`, `batch`, and `requester` variables that the module must have already defined. The `script` and `batch` variables are a script object and a batch object, respectively. The `requester` variable is an email address. The fourth argument is a boolean, specifying whether to perform isolation; a missing fourth argument is equivalent to `false`. The `merge()` function of the `merge` module generates jobs and returns them in an array. The invoking module can further dispose of the jobs as needed.
|
|
256
273
|
|
|
257
|
-
|
|
274
|
+
##### By a user
|
|
258
275
|
|
|
259
276
|
A user can invoke `merge` in this way:
|
|
260
277
|
|
|
@@ -267,26 +284,16 @@ A user can invoke `merge` in this way:
|
|
|
267
284
|
|
|
268
285
|
In these statements, replace `s` and `b` with the base names of the script and batch files, respectively. For example, if the script file is named `ts25.json`, then replace `s` with `ts25`. Replace `e` with an email address, or with an empty string if the environment variable `process.env.REQUESTER` exists and you want to use it.
|
|
269
286
|
|
|
270
|
-
The first statement will cause a merger with isolation.
|
|
271
|
-
The second and third statements will cause a merger without isolation.
|
|
287
|
+
The first statement will cause a merger **with** isolation.
|
|
288
|
+
The second and third statements will cause a merger **without* isolation.
|
|
272
289
|
|
|
273
290
|
The `call` module will retrieve the named script and batch from their respective directories.
|
|
274
291
|
The `merge` module will create an array of jobs.
|
|
275
|
-
The `call` module will save the jobs in the `todo` subdirectory of the `process.env.JOBDIR` directory.
|
|
276
|
-
|
|
277
|
-
### Validation
|
|
278
|
-
|
|
279
|
-
To test the `merge` module, in the project directory you can execute the statement `node validation/merge/validate`. All logging statements should begin with “Success” and none should begin with “ERROR”.
|
|
280
|
-
|
|
281
|
-
### Examples
|
|
282
|
-
|
|
283
|
-
There are script and batch examples in the `specs` directory.
|
|
292
|
+
The `call` module will save the jobs as JSON files in the `todo` subdirectory of the `process.env.JOBDIR` directory.
|
|
284
293
|
|
|
285
|
-
|
|
294
|
+
#### Validation
|
|
286
295
|
|
|
287
|
-
|
|
288
|
-
- One of them has the URL `https://example.com` and requires each placeholder to be replaced with 2 acts: a `launch` act and a `url` act.
|
|
289
|
-
- The other has the URL `https://www.w3.org/WAI/ARIA/apg/example-index/accordion/accordion` and requires each placeholder to be replaced with 3 acts: a `launch` act, a `url` act, and a `button` act.
|
|
296
|
+
To test the `merge` module, in the project directory you can execute the statement `node validation/merge/validate`. If `merge` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
|
290
297
|
|
|
291
298
|
## Report scoring
|
|
292
299
|
|
|
@@ -307,9 +314,14 @@ Thus, a report produced by Testaro contains these properties:
|
|
|
307
314
|
- `timeStamp`
|
|
308
315
|
- `jobData`
|
|
309
316
|
|
|
310
|
-
Testilo can
|
|
317
|
+
Testilo can enhance such a report in three ways:
|
|
318
|
+
- adding scores
|
|
319
|
+
- creating digests
|
|
320
|
+
- creating comprisons
|
|
311
321
|
|
|
312
|
-
|
|
322
|
+
To add scores to reports, the `score` module of Testilo performs computations on the test results and adds a `score` property to each report.
|
|
323
|
+
|
|
324
|
+
The `score()` function of the `score` module takes two arguments:
|
|
313
325
|
- a scoring function
|
|
314
326
|
- an array of report objects
|
|
315
327
|
|
|
@@ -325,58 +337,44 @@ A module can invoke `score` in this way:
|
|
|
325
337
|
|
|
326
338
|
```javaScript
|
|
327
339
|
const {score} = require('testilo/score');
|
|
328
|
-
const {scorer} = require('testilo/procs/score/
|
|
329
|
-
|
|
340
|
+
const {scorer} = require('testilo/procs/score/tsp25');
|
|
341
|
+
score(scorer, reports);
|
|
330
342
|
```
|
|
331
343
|
|
|
332
|
-
The first argument to `score()` is a scoring function. In this example, it has been obtained from a
|
|
344
|
+
The first argument to `score()` is a scoring function. In this example, it has been obtained from a module in the Testilo package, but it could be a custom function. The second argument to `score()` is an array of report objects. The invoking module can further dispose of the scored reports as needed.
|
|
333
345
|
|
|
334
346
|
#### By a user
|
|
335
347
|
|
|
336
348
|
A user can invoke `score` in this way:
|
|
337
349
|
|
|
338
350
|
```bash
|
|
339
|
-
node call score
|
|
340
|
-
node call score
|
|
351
|
+
node call score tsp25
|
|
352
|
+
node call score tsp25 75m
|
|
341
353
|
```
|
|
342
354
|
|
|
343
355
|
When a user invokes `score` in this example, the `call` module:
|
|
344
|
-
- gets the scoring module `
|
|
345
|
-
- gets the reports from the `raw` subdirectory of the `process.env.REPORTDIR` directory
|
|
346
|
-
- writes the scored reports to the `scored` subdirectory of the `process.env.REPORTDIR` directory
|
|
356
|
+
- gets the scoring module `tsp25` from its JSON file `tsp25.json` in the `score` subdirectory of the `process.env.FUNCTIONDIR` directory.
|
|
357
|
+
- gets the reports from the `raw` subdirectory of the `process.env.REPORTDIR` directory.
|
|
358
|
+
- writes the scored reports in JSON format to the `scored` subdirectory of the `process.env.REPORTDIR` directory.
|
|
347
359
|
|
|
348
|
-
The optional third argument to call (`
|
|
360
|
+
The optional third argument to call (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `raw` subdirectory. With the argument, `call` gets only those reports whose names begin with the argument string.
|
|
349
361
|
|
|
350
362
|
### Validation
|
|
351
363
|
|
|
352
|
-
To test the `score` module, in the project directory you can execute the statement `node validation/score/validate`.
|
|
353
|
-
|
|
354
|
-
### Comprehensive example
|
|
355
|
-
|
|
356
|
-
The module `procs/score/tsp21.js` is an example of a comprehensive score proc. It is designed to score reports produced by Testaro from jobs that Testilo creates from the `ts21` script. That script causes Testaro to perform 1351 tests belonging to Testaro and nine other tools integrated into Testaro as dependencies.
|
|
364
|
+
To test the `score` module, in the project directory you can execute the statement `node validation/score/validate`. If `score` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
|
357
365
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
The `tsp21` score proc imports the `procs/score/tic21.js` module, which classifies all the tests into 255 “issues”. Any two or more tests classified into the same issue are deemed to be approximately equivalent in intent. Using this classification, the score proc tabulates the test results by issue, so that all instances of the issue, regardless of which tool discovered the instances, are reported together.
|
|
361
|
-
|
|
362
|
-
The issue classification in `tic21` assigns weights to the issues to reflect their estimated severities,and the scoring algorithm of `tsp21` adopts a compromise between issue-based and tool-based summation. Thus, if 3 tools discover 20 instances of an issue, the score increases (i.e. becomes worse) by more than it would if only 1 tool had discovered them, but less than 3 times as much. The motivations are that discovery of issue instances by multiple tools increases confidence that the issues are real, and passing accessibility tests is per se beneficial because it decreases risks associated with alleged inaccessibility.
|
|
363
|
-
|
|
364
|
-
The `tsp21` score proc does **not** attempt to identify instances of issues. Thus, if tool A discovers 4 instances and tool B discovers 7 instances of some issue, `tsp21` does not attempt to say which of the 4 discovered by A are a subset of the 7 discovered by B. Instead,`tsp21` naïvely acts as if the 4 are all a subset of the 7, even though in reality the two sets might be entirely disjoint.
|
|
365
|
-
|
|
366
|
-
In the `tsp21` scoring algorithm, the failure of a tool or of a Testaro test to run on a page increases the score of the page, and messages logged by the browser while tests are being performed on a page, especially error messages, also increase the score.
|
|
367
|
-
|
|
368
|
-
## Report digesting
|
|
366
|
+
## Rport digesting
|
|
369
367
|
|
|
370
368
|
### Introduction
|
|
371
369
|
|
|
372
|
-
Reports
|
|
370
|
+
Reports from Testaro are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
|
|
373
371
|
|
|
374
372
|
The `digest` module digests a scored report. Its `digest()` function takes three arguments:
|
|
375
373
|
- a digest template
|
|
376
374
|
- a digesting function
|
|
377
|
-
- an array of report objects
|
|
375
|
+
- an array of scored report objects
|
|
378
376
|
|
|
379
|
-
The digest template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by
|
|
377
|
+
The digest template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by computed values, becomes the digest. The digesting function defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/digest` directory, in which there are subdirectories containing pairs of templates and modules that export digesting functions. You can use one of those pairs, or you can create your own.
|
|
380
378
|
|
|
381
379
|
### Invocation
|
|
382
380
|
|
|
@@ -389,36 +387,36 @@ A module can invoke `digest` in this way:
|
|
|
389
387
|
```javaScript
|
|
390
388
|
const {digest} = require('testilo/digest');
|
|
391
389
|
const digesterDir = `${process.env.FUNCTIONDIR}/digest/dp25a`;
|
|
392
|
-
fs.readFile(`${digesterDir}/index.html
|
|
390
|
+
fs.readFile(`${digesterDir}/index.html`, 'utf8')
|
|
393
391
|
.then(template => {
|
|
394
392
|
const {digester} = require(`${digesterDir}/index`);
|
|
395
393
|
const digestedReports = digest(template, digester, scoredReports);
|
|
396
394
|
});
|
|
397
395
|
```
|
|
398
396
|
|
|
399
|
-
The first two arguments to `digest()` are a digest template and a digesting function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `digest()` is an array of report objects.
|
|
397
|
+
The first two arguments to `digest()` are a digest template and a digesting function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `digest()` is an array of scored report objects. The `digest()` function returns an array of digested reports. The invoking module can further dispose of the digested reports as needed.
|
|
400
398
|
|
|
401
399
|
#### By a user
|
|
402
400
|
|
|
403
401
|
A user can invoke `digest` in this way:
|
|
404
402
|
|
|
405
403
|
```bash
|
|
406
|
-
node call digest
|
|
407
|
-
node call digest
|
|
404
|
+
node call digest tdp25
|
|
405
|
+
node call digest tdp25 75m
|
|
408
406
|
```
|
|
409
407
|
|
|
410
408
|
When a user invokes `digest` in this example, the `call` module:
|
|
411
|
-
- gets the template and the digesting module from subdirectory `
|
|
412
|
-
- gets the reports from the `scored` subdirectory of the `process.env.REPORTDIR` directory
|
|
413
|
-
- writes the digested reports to the `digested` subdirectory of the `process.env.REPORTDIR` directory
|
|
409
|
+
- gets the template and the digesting module from subdirectory `tdp25` in the `digest` subdirectory of the `process.env.FUNCTIONDIR` directory.
|
|
410
|
+
- gets the reports from the `scored` subdirectory of the `process.env.REPORTDIR` directory.
|
|
411
|
+
- writes the digested reports to the `digested` subdirectory of the `process.env.REPORTDIR` directory.
|
|
414
412
|
|
|
415
|
-
The optional third argument to call (`
|
|
413
|
+
The optional third argument to call (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory. With the argument, `call` gets only those reports whose names begin with the argument string.
|
|
416
414
|
|
|
417
415
|
The digests created by `digest` are HTML files, and they expect a `style.css` file to exist in their directory. The `reports/digested/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where digested reports are written.
|
|
418
416
|
|
|
419
417
|
### Validation
|
|
420
418
|
|
|
421
|
-
To test the `digest` module, in the project directory you can execute the statement `node validation/digest/validate`.
|
|
419
|
+
To test the `digest` module, in the project directory you can execute the statement `node validation/digest/validate`. If `digest` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
|
422
420
|
|
|
423
421
|
### Report comparison
|
|
424
422
|
|
|
@@ -429,7 +427,7 @@ The `compare` module compares the scores in a collection of scored reports. Its
|
|
|
429
427
|
- a comparison function
|
|
430
428
|
- an array of scored reports
|
|
431
429
|
|
|
432
|
-
The comparison template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by
|
|
430
|
+
The comparison template is an HTML document containing placeholders. A copy of the template, with its placeholders replaced by computed values, becomes the comparative report. The comparison function defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/compare` directory, in which there are subdirectories containing pairs of templates and modules that export comparison functions. You can use one of those pairs, or you can create your own.
|
|
433
431
|
|
|
434
432
|
### Invocation
|
|
435
433
|
|
|
@@ -442,31 +440,31 @@ A module can invoke `compare` in this way:
|
|
|
442
440
|
```javaScript
|
|
443
441
|
const fs = require('fs/promises);
|
|
444
442
|
const {compare} = require('testilo/compare');
|
|
445
|
-
const comparerDir = `${process.env.FUNCTIONDIR}/compare/
|
|
443
|
+
const comparerDir = `${process.env.FUNCTIONDIR}/compare/tcp25`;
|
|
446
444
|
fs.readFile(`${comparerDir}/index.html`)
|
|
447
445
|
.then(template => {
|
|
448
446
|
const {comparer} = require(`${comparerDir}/index`);
|
|
449
|
-
const
|
|
447
|
+
const comparativeReport = compare(template, comparer, scoredReports);
|
|
450
448
|
});
|
|
451
449
|
```
|
|
452
450
|
|
|
453
|
-
The first two arguments to `compare()` are a template and a comparison function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `compare()` is an array of report objects.
|
|
451
|
+
The first two arguments to `compare()` are a template and a comparison function. In this example, they have been obtained from files in the Testilo package, but they could be custom-made. The third argument to `compare()` is an array of report objects. The `compare()` function returns a comparative report. The invoking module can further dispose of the comparative report as needed.
|
|
454
452
|
|
|
455
453
|
#### By a user
|
|
456
454
|
|
|
457
455
|
A user can invoke `compare` in this way:
|
|
458
456
|
|
|
459
457
|
```bash
|
|
460
|
-
node call compare
|
|
458
|
+
node call compare tcp25 legislators
|
|
461
459
|
```
|
|
462
460
|
|
|
463
461
|
When a user invokes `compare` in this example, the `call` module:
|
|
464
|
-
- gets the template and the comparison module from subdirectory `
|
|
465
|
-
- gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory
|
|
466
|
-
- writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `process.env.REPORTDIR` directory
|
|
462
|
+
- gets the template and the comparison module from subdirectory `tcp25` of the subdirectory `compare` in the `process.env.FUNCTIONDIR` directory.
|
|
463
|
+
- gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory.
|
|
464
|
+
- writes the comparative report as an HTML file named `legislators.html` to the `comparative` subdirectory of the `process.env.REPORTDIR` directory.
|
|
467
465
|
|
|
468
|
-
The comparative
|
|
466
|
+
The comparative report created by `compare` is an HTML file, and it expects a `style.css` file to exist in its directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
|
|
469
467
|
|
|
470
468
|
### Validation
|
|
471
469
|
|
|
472
|
-
To test the `compare` module, in the project directory you can execute the statement `node validation/compare/validate`.
|
|
470
|
+
To test the `compare` module, in the project directory you can execute the statement `node validation/compare/validate`. If `compare` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
package/batch.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/*
|
|
2
|
+
batch.js
|
|
3
|
+
Converts a target list to a batch.
|
|
4
|
+
Arguments:
|
|
5
|
+
0. batch ID
|
|
6
|
+
1. batch description
|
|
7
|
+
2. target list
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
// ########## FUNCTIONS
|
|
11
|
+
|
|
12
|
+
// Converts a target list to a batch and returns the batch.
|
|
13
|
+
exports.batch = (id, what, targetList) => {
|
|
14
|
+
// If the arguments are valid:
|
|
15
|
+
if (
|
|
16
|
+
typeof id === 'string'
|
|
17
|
+
&& id.length
|
|
18
|
+
&& typeof what === 'string'
|
|
19
|
+
&& what.length
|
|
20
|
+
&& Array.isArray(targetList)
|
|
21
|
+
&& targetList.length
|
|
22
|
+
&& targetList.every(
|
|
23
|
+
target => Array.isArray(target) && target.every(item => typeof item === string && item.length)
|
|
24
|
+
)
|
|
25
|
+
) {
|
|
26
|
+
// Initialize the batch.
|
|
27
|
+
const batch = {
|
|
28
|
+
id,
|
|
29
|
+
what,
|
|
30
|
+
targets: []
|
|
31
|
+
};
|
|
32
|
+
// For each target:
|
|
33
|
+
targetList.forEach(target => {
|
|
34
|
+
// Add it to the batch.
|
|
35
|
+
batch.targets.push({
|
|
36
|
+
id: target[0],
|
|
37
|
+
what: target[1],
|
|
38
|
+
acts: {
|
|
39
|
+
main: [
|
|
40
|
+
{
|
|
41
|
+
type: 'launch'
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
type: 'url',
|
|
45
|
+
which: target[2],
|
|
46
|
+
what: target[1]
|
|
47
|
+
}
|
|
48
|
+
]
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
// Return the batch.
|
|
53
|
+
return batch;
|
|
54
|
+
}
|
|
55
|
+
// Otherwise, i.e. if the arguments are invalid:
|
|
56
|
+
else {
|
|
57
|
+
// Return this.
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
};
|