testaro 45.0.13 → 46.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -55
- package/actSpecs.js +2 -1
- package/package.json +1 -1
- package/procs/job.js +1 -6
- package/procs/standardize.js +1 -3
- package/run.js +209 -214
package/README.md
CHANGED
|
@@ -136,7 +136,6 @@ Here is an example of a job:
|
|
|
136
136
|
id: '250110T1200-7f-4',
|
|
137
137
|
what: 'monthly health check',
|
|
138
138
|
strict: true,
|
|
139
|
-
isolate: true,
|
|
140
139
|
standard: 'also',
|
|
141
140
|
observe: false,
|
|
142
141
|
device: {
|
|
@@ -155,7 +154,6 @@ Here is an example of a job:
|
|
|
155
154
|
}
|
|
156
155
|
},
|
|
157
156
|
browserID: 'webkit',
|
|
158
|
-
timeLimit: 80,
|
|
159
157
|
creationTimeStamp: '241229T0537',
|
|
160
158
|
executionTimeStamp: '250110T1200',
|
|
161
159
|
sendReportTo: 'https://abccorp.com/api/report',
|
|
@@ -170,11 +168,9 @@ Here is an example of a job:
|
|
|
170
168
|
requester: 'malavu@abccorp.com'
|
|
171
169
|
},
|
|
172
170
|
acts: [
|
|
173
|
-
{
|
|
174
|
-
type: 'launch'
|
|
175
|
-
},
|
|
176
171
|
{
|
|
177
172
|
type: 'test',
|
|
173
|
+
launch: {},
|
|
178
174
|
which: 'axe',
|
|
179
175
|
detailLevel: 2,
|
|
180
176
|
rules: ['landmark-complementary-is-top-level'],
|
|
@@ -182,6 +178,7 @@ Here is an example of a job:
|
|
|
182
178
|
},
|
|
183
179
|
{
|
|
184
180
|
type: 'test',
|
|
181
|
+
launch: {},
|
|
185
182
|
which: 'qualWeb',
|
|
186
183
|
withNewContent: false,
|
|
187
184
|
rules: ['QW-BP25', 'QW-BP26']
|
|
@@ -191,20 +188,18 @@ Here is an example of a job:
|
|
|
191
188
|
}
|
|
192
189
|
```
|
|
193
190
|
|
|
194
|
-
This job
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
1. Perform the tests for rules `QW-BP25` and `QW-BP26` of the `qualWeb` tool on the existing page.
|
|
191
|
+
This job tells Testaro to perform two acts. One performs one test of the Axe tool wih reporting at detail level 2, and the other performs two tests of the QualWeb tool.
|
|
192
|
+
|
|
193
|
+
Each act includes a `launch` property with a default value. That instructs Testaro, before performing those tests, to launch a new Webkit browser, open a context (window) with some properties of an iPhone 8 and without a reduced-motion setting, create a page (tab), and navigate to a particular page of the `abccorp.com` website.
|
|
198
194
|
|
|
199
195
|
Job properties:
|
|
200
196
|
- `id`: a string uniquely identifying the job.
|
|
201
197
|
- `what`: a description of the job.
|
|
202
198
|
- `strict`: `true` or `false`, indicating whether _substantive redirections_ should be treated as failures. These are redirections that do more than add or subtract a final slash.
|
|
203
|
-
- `standard`:
|
|
199
|
+
- `standard`: whether standardized versions of tool reports are to accompany the original versions (`'also'`), replace the original versions (`'only'`), or not be produced (`'no'`).
|
|
204
200
|
- `observe`: `true` or `false`, indicating whether tool and Testaro-rule invocations are to be reported to the server as they occur, so that the server can update a waiting client.
|
|
205
|
-
- `device`: the ID of a device and the properties of each new browser context (window) that will be set for conformity to that device, unless overridden by
|
|
206
|
-
- `browserID`: the ID of the browser to be used, unless overridden by
|
|
207
|
-
- `timeLimit`: the number of seconds allowed for the execution of the job.
|
|
201
|
+
- `device`: the ID of a device and the properties of each new browser context (window) that will be set for conformity to that device, unless overridden by an act. It must be `'default'` or the ID of one of [about 125 devices recognized by Playwright](https://github.com/microsoft/playwright/blob/main/packages/playwright-core/src/server/deviceDescriptorsSource.json).
|
|
202
|
+
- `browserID`: the ID of the browser to be used, unless overridden by an act. It must be `'chromium'`, `'firefox'`, or `'webkit`'.
|
|
208
203
|
- `creationTimeStamp`: a string in `yymmddThhMM` format, describing when the job was created.
|
|
209
204
|
- `executionTimeStamp`: a string in `yymmddThhMM` format, specifying a date and time before which the job is not to be performed.
|
|
210
205
|
- `sendReportTo`: the URL to which the job report is to be sent, or `''` if not a `netWatch` job.
|
|
@@ -218,19 +213,31 @@ Job properties:
|
|
|
218
213
|
|
|
219
214
|
Each act object has a `type` property and optionally has a `name` property (used in branching, described below). It must or may have other properties, depending on the value of `type`.
|
|
220
215
|
|
|
221
|
-
### Act sequence
|
|
222
|
-
|
|
223
|
-
The first act in any job has the type `launch`, as shown in the example above. It launches a browser and then uses it to visit a URL.
|
|
224
|
-
|
|
225
216
|
### Act types
|
|
226
217
|
|
|
227
|
-
The acts
|
|
228
|
-
- _moves_ (clicks, text inputs, hovers, etc.)
|
|
218
|
+
The acts can tell Testaro to perform any of:
|
|
229
219
|
- _navigations_ (browser launches, visits to URLs, waits for page conditions, etc.)
|
|
220
|
+
- _moves_ (clicks, text inputs, hovers, etc.)
|
|
230
221
|
- _alterations_ (changes to the page)
|
|
231
222
|
- _tests_ (one or more of the tests defined by a tool)
|
|
232
223
|
- _branching_ (continuing from an act other than the next one)
|
|
233
224
|
|
|
225
|
+
#### Navigations
|
|
226
|
+
|
|
227
|
+
An example of a **navigation** is:
|
|
228
|
+
|
|
229
|
+
```json
|
|
230
|
+
{
|
|
231
|
+
"type": "wait",
|
|
232
|
+
"which": "travel",
|
|
233
|
+
"what": "title"
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
In this case, Testaro waits until the page title contains the string “travel” (case-insensitively).
|
|
238
|
+
|
|
239
|
+
There is also a `launch` act. You need it in any job before other acts can be performed, unless the acts are all `test` acts and they include `launch` properties, as in the job example above. That `launch` property is a compact alternative to a `launch` act.
|
|
240
|
+
|
|
234
241
|
#### Moves
|
|
235
242
|
|
|
236
243
|
An example of a **move** is:
|
|
@@ -250,24 +257,6 @@ In identifying the target element for a move, Testaro matches the `which` proper
|
|
|
250
257
|
|
|
251
258
|
When the texts of multiple elements of the same type will contain the same `which` value, you can include an `index` property to specify the index of the target element, among all those that will match.
|
|
252
259
|
|
|
253
|
-
#### Navigations
|
|
254
|
-
|
|
255
|
-
An example of a **navigation** is the act of type `launch` in the job example above. That `launch` act has only a `type` property. If you want a particular `launch` act to use a different browser type or to navigate to a different target URL from the job defaults, the `launch` act can have a `browserID` and/or a `url` property.
|
|
256
|
-
|
|
257
|
-
If any act alters the page, you can restore the page to its original state for the next act by inserting a new `launch` act (and, if necessary, additional page-specific acts) between them.
|
|
258
|
-
|
|
259
|
-
Another navigation example is:
|
|
260
|
-
|
|
261
|
-
```json
|
|
262
|
-
{
|
|
263
|
-
"type": "wait",
|
|
264
|
-
"which": "travel",
|
|
265
|
-
"what": "title"
|
|
266
|
-
}
|
|
267
|
-
```
|
|
268
|
-
|
|
269
|
-
In this case, Testaro waits until the page title contains the string “travel” (case-insensitively).
|
|
270
|
-
|
|
271
260
|
#### Alterations
|
|
272
261
|
|
|
273
262
|
An example of an **alteration** is:
|
|
@@ -306,18 +295,6 @@ An act of type `test` performs the tests of a tool and reports a result. The res
|
|
|
306
295
|
|
|
307
296
|
The `which` property of a `test` act identifies a tool, such as `alfa` or `testaro`.
|
|
308
297
|
|
|
309
|
-
#### Target modification
|
|
310
|
-
|
|
311
|
-
Some tools modify the page, so isolation of tests from one another requires that a browser be relaunched or, at least, navigate to the URL again, after a test act running any of those tools before a test act running another tool.
|
|
312
|
-
|
|
313
|
-
Of the 10 tools, 6 are target-modifying:
|
|
314
|
-
- `alfa`
|
|
315
|
-
- `aslint`
|
|
316
|
-
- `axe`
|
|
317
|
-
- `htmlcs`
|
|
318
|
-
- `ibm`
|
|
319
|
-
- `testaro`
|
|
320
|
-
|
|
321
298
|
##### Configuration
|
|
322
299
|
|
|
323
300
|
Every tool invoked by Testaro must have:
|
|
@@ -331,17 +308,18 @@ test: [
|
|
|
331
308
|
'Perform a test',
|
|
332
309
|
{
|
|
333
310
|
which: [true, 'string', 'isTest', 'test name'],
|
|
311
|
+
launch: [false, 'object', '', 'if new browser to be launched, properties different from target, browserID, and what of the job'],
|
|
334
312
|
rules: [false, 'array', 'areStrings', 'rule IDs or specifications, if not all']
|
|
335
313
|
what: [false, 'string', 'hasLength', 'comment']
|
|
336
314
|
}
|
|
337
315
|
],
|
|
338
316
|
```
|
|
339
317
|
|
|
340
|
-
That means that a test act (i.e. an act with a `type` property having the value `'test'`) must have a string-valued `which` property naming a tool and may optionally have an
|
|
318
|
+
That means that a test act (i.e. an act with a `type` property having the value `'test'`) must have a string-valued `which` property naming a tool and may optionally have an object-valued `launch` property, an array-valued `rules` property, and/or a string-valued `what` property.
|
|
341
319
|
|
|
342
320
|
If a particular test act either must have or may have any other properties, those properties are specified in the `tools` property in `actSpecs.js`.
|
|
343
321
|
|
|
344
|
-
When you include a `rules` property, you limit the tests of the tool that are performed or reported. For some tools (`alfa`, `axe`, `htmlcs`, `qualWeb`, and `
|
|
322
|
+
When you include a `rules` property, you limit the tests of the tool that are performed or reported. For some tools (`alfa`, `axe`, `htmlcs`, `qualWeb`, `testaro`, and `wax`), only the specified tests are performed. Other tools (`aslint`, `ed11y`, `ibm`, `nuVal`, and `wave`) do not allow such a limitation, so, for those tools, all tests are performed but results are reported from only the specified tests.
|
|
345
323
|
|
|
346
324
|
The `nuVal`, `qualWeb`, and `testaro` tools require specific formats for the `rules` property. Those formats are described below in the sections about those tools.
|
|
347
325
|
|
|
@@ -358,7 +336,7 @@ An example of a `test` act is:
|
|
|
358
336
|
}
|
|
359
337
|
```
|
|
360
338
|
|
|
361
|
-
Most tools allow you to decide which of their rules to apply. In effect, this means deciding which of their tests to run, since each test is considered a test of some rule. The act example
|
|
339
|
+
Most tools allow you to decide which of their rules to apply. In effect, this means deciding which of their tests to run, since each test is considered a test of some rule. The act example
|
|
362
340
|
|
|
363
341
|
```javaScript
|
|
364
342
|
{
|
|
@@ -536,7 +514,7 @@ You can add custom rules to the rules of any tool. Testaro provides a template,
|
|
|
536
514
|
|
|
537
515
|
#### WallyAX
|
|
538
516
|
|
|
539
|
-
If a `wax` test act is included in the job, an environment variable named `WAX_KEY` must exist, with your WallyAX API key as its value. You can
|
|
517
|
+
If a `wax` test act is included in the job, an environment variable named `WAX_KEY` must exist, with your WallyAX API key as its value. You can request it from [WallyAX](mailto:technology@wallyax.com).
|
|
540
518
|
|
|
541
519
|
The `wax` tool imposes a limit on the size of a page to be tested. If the page exceeds the limit, Testaro treats the page as preventing `wax` from performing its tests. The limit is less than 500,000 characters.
|
|
542
520
|
|
|
@@ -629,8 +607,7 @@ Testaro also generates some data about the job and adds those data to the job, i
|
|
|
629
607
|
### Contents
|
|
630
608
|
|
|
631
609
|
A report discloses:
|
|
632
|
-
-
|
|
633
|
-
- standardized results of tests conducted by tools
|
|
610
|
+
- results of tests conducted by tools
|
|
634
611
|
- process data, including statistics on:
|
|
635
612
|
- latency (how long a time each tool takes to run its tests)
|
|
636
613
|
- test prevention (the failure of tools to run on particular targets)
|
|
@@ -655,6 +632,8 @@ The standard result includes three properties:
|
|
|
655
632
|
- `totals`: an array of numbers representing how many instances of rule violations at each level of severity the tool reported. There are 4 ordinal severity levels. For example, the array `[3, 0, 14, 10]` would report that there were 3 violations at level 0, 0 at level 1, 14 at level 2, and 10 at level 3.
|
|
656
633
|
- `instances`: an array of objects describing the rule violations. An instance can describe a single violation, usually by one element in the page, or can summarize multiple violations of the same rule.
|
|
657
634
|
|
|
635
|
+
If the value of `prevented` is `true`, the standard result also includes an `error` property describing the reason for the failure.
|
|
636
|
+
|
|
658
637
|
##### Instances
|
|
659
638
|
|
|
660
639
|
Here is an example of a standard instance:
|
|
@@ -686,6 +665,7 @@ The element has no `id` attribute to distinguish it from other `button` elements
|
|
|
686
665
|
- `xpath`: Alfa, ASLint, Equal Access
|
|
687
666
|
- `box` (coordinates, width, and height of the element box): Editoria11y, Testaro
|
|
688
667
|
- none: HTML CodeSniffer
|
|
668
|
+
|
|
689
669
|
The tool also reproduces an excerpt of the element code.
|
|
690
670
|
|
|
691
671
|
##### Element identification
|
package/actSpecs.js
CHANGED
|
@@ -54,7 +54,7 @@ exports.actSpecs = {
|
|
|
54
54
|
launch: [
|
|
55
55
|
'Launch a Playwright browser',
|
|
56
56
|
{
|
|
57
|
-
target: [false, 'object', '', 'target different from
|
|
57
|
+
target: [false, 'object', '', 'target different from target of the job'],
|
|
58
58
|
browserID: [false, 'string', 'isBrowserID', 'browser type different from browserID of the job'],
|
|
59
59
|
what: [false, 'string', 'hasLength', 'comment']
|
|
60
60
|
}
|
|
@@ -144,6 +144,7 @@ exports.actSpecs = {
|
|
|
144
144
|
'Perform tests of a tool',
|
|
145
145
|
{
|
|
146
146
|
which: [true, 'string', 'isTest', 'tool name'],
|
|
147
|
+
launch: [false, 'object', '', 'if new browser to be launched, properties different from target, browserID, and what of the job'],
|
|
147
148
|
rules: [false, 'array', 'areStrings', 'rule IDs or (for nuVal) specifications, if not all']
|
|
148
149
|
}
|
|
149
150
|
],
|
package/package.json
CHANGED
package/procs/job.js
CHANGED
|
@@ -187,7 +187,6 @@ exports.isValidJob = job => {
|
|
|
187
187
|
const {
|
|
188
188
|
id,
|
|
189
189
|
strict,
|
|
190
|
-
isolate,
|
|
191
190
|
standard,
|
|
192
191
|
observe,
|
|
193
192
|
device,
|
|
@@ -207,9 +206,6 @@ exports.isValidJob = job => {
|
|
|
207
206
|
if (typeof strict !== 'boolean') {
|
|
208
207
|
return 'Bad job strict';
|
|
209
208
|
}
|
|
210
|
-
if (typeof isolate !== 'boolean') {
|
|
211
|
-
return 'Bad job isolate';
|
|
212
|
-
}
|
|
213
209
|
if (! ['also', 'only', 'no'].includes(standard)) {
|
|
214
210
|
return 'Bad job standard';
|
|
215
211
|
}
|
|
@@ -247,9 +243,8 @@ exports.isValidJob = job => {
|
|
|
247
243
|
if (
|
|
248
244
|
! acts
|
|
249
245
|
|| ! Array.isArray(acts)
|
|
250
|
-
|| acts.length
|
|
246
|
+
|| ! acts.length
|
|
251
247
|
|| ! acts.every(act => act.type && typeof act.type === 'string')
|
|
252
|
-
|| acts[0].type !== 'launch'
|
|
253
248
|
) {
|
|
254
249
|
return 'Bad job acts';
|
|
255
250
|
}
|
package/procs/standardize.js
CHANGED
|
@@ -647,7 +647,7 @@ const convert = (toolName, data, result, standardResult) => {
|
|
|
647
647
|
// Get its standard instance properties.
|
|
648
648
|
const element = violation.element.replace(/\s+/g, ' ');
|
|
649
649
|
const {message, description, severity} = violation;
|
|
650
|
-
const ordinalSeverity = ['Minor', 'Moderate', '
|
|
650
|
+
const ordinalSeverity = ['Minor', 'Moderate', '', 'Severe'].indexOf(severity);
|
|
651
651
|
const tagNameCandidate = element.replace(/^<| .*$/g, '');
|
|
652
652
|
const tagName = /^[a-zA-Z0-9]+$/.test(tagNameCandidate) ? tagNameCandidate.toUpperCase() : '';
|
|
653
653
|
let id = '';
|
|
@@ -677,8 +677,6 @@ const convert = (toolName, data, result, standardResult) => {
|
|
|
677
677
|
location,
|
|
678
678
|
excerpt: element
|
|
679
679
|
};
|
|
680
|
-
// Add its ordinal severity to the standard result totals.
|
|
681
|
-
standardResult.totals[ordinalSeverity]++;
|
|
682
680
|
// Add the instance to the standard result.
|
|
683
681
|
standardResult.instances.push(instance);
|
|
684
682
|
});
|
package/run.js
CHANGED
|
@@ -88,10 +88,11 @@ const timeLimits = {
|
|
|
88
88
|
|
|
89
89
|
// Facts about the current session.
|
|
90
90
|
let actCount = 0;
|
|
91
|
-
// Facts about the current
|
|
91
|
+
// Facts about the current act.
|
|
92
|
+
let actIndex = 0;
|
|
92
93
|
let browser;
|
|
93
94
|
let browserContext;
|
|
94
|
-
let
|
|
95
|
+
let page;
|
|
95
96
|
let requestedURL = '';
|
|
96
97
|
|
|
97
98
|
// FUNCTIONS
|
|
@@ -192,6 +193,7 @@ const browserClose = async () => {
|
|
|
192
193
|
};
|
|
193
194
|
// Launches a browser, navigates to a URL, and returns browser data.
|
|
194
195
|
const launch = async (report, debug, waits, tempBrowserID, tempURL) => {
|
|
196
|
+
const act = report.acts[actIndex];
|
|
195
197
|
const {device} = report;
|
|
196
198
|
const deviceID = device && device.id;
|
|
197
199
|
const browserID = tempBrowserID || report.browserID || '';
|
|
@@ -211,85 +213,76 @@ const launch = async (report, debug, waits, tempBrowserID, tempURL) => {
|
|
|
211
213
|
};
|
|
212
214
|
browserOptions.headless = ! debug;
|
|
213
215
|
browserOptions.slowMo = waits || 0;
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
//
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
const tail = msgText.slice(150).slice(-150);
|
|
260
|
-
if (msgText.length > 300) {
|
|
261
|
-
parts.push('...');
|
|
262
|
-
}
|
|
263
|
-
parts.push(tail.slice(0, 75));
|
|
264
|
-
if (tail.length > 75) {
|
|
265
|
-
parts.push(tail.slice(75));
|
|
216
|
+
try {
|
|
217
|
+
// Replace the browser with a new one.
|
|
218
|
+
browser = await browserType.launch(browserOptions);
|
|
219
|
+
// Open a context (i.e. browser window).
|
|
220
|
+
const browserContext = await browser.newContext(device.windowOptions);
|
|
221
|
+
// Prevent default timeouts.
|
|
222
|
+
browserContext.setDefaultTimeout(0);
|
|
223
|
+
// When a page (i.e. browser tab) is added to the browser context (i.e. browser window):
|
|
224
|
+
browserContext.on('page', async page => {
|
|
225
|
+
// Ensure the report has a jobData property.
|
|
226
|
+
report.jobData ??= {};
|
|
227
|
+
const {jobData} = report;
|
|
228
|
+
jobData.logCount ??= 0;
|
|
229
|
+
jobData.logSize ??= 0;
|
|
230
|
+
jobData.errorLogCount ??= 0;
|
|
231
|
+
// Add any error events to the count of logging errors.
|
|
232
|
+
page.on('crash', () => {
|
|
233
|
+
jobData.errorLogCount++;
|
|
234
|
+
console.log('Page crashed');
|
|
235
|
+
});
|
|
236
|
+
page.on('pageerror', () => {
|
|
237
|
+
jobData.errorLogCount++;
|
|
238
|
+
});
|
|
239
|
+
page.on('requestfailed', () => {
|
|
240
|
+
jobData.errorLogCount++;
|
|
241
|
+
});
|
|
242
|
+
// If the page emits a message:
|
|
243
|
+
page.on('console', msg => {
|
|
244
|
+
const msgText = msg.text();
|
|
245
|
+
let indentedMsg = '';
|
|
246
|
+
// If debugging is on:
|
|
247
|
+
if (debug) {
|
|
248
|
+
// Log a summary of the message on the console.
|
|
249
|
+
const parts = [msgText.slice(0, 75)];
|
|
250
|
+
if (msgText.length > 75) {
|
|
251
|
+
parts.push(msgText.slice(75, 150));
|
|
252
|
+
if (msgText.length > 150) {
|
|
253
|
+
const tail = msgText.slice(150).slice(-150);
|
|
254
|
+
if (msgText.length > 300) {
|
|
255
|
+
parts.push('...');
|
|
256
|
+
}
|
|
257
|
+
parts.push(tail.slice(0, 75));
|
|
258
|
+
if (tail.length > 75) {
|
|
259
|
+
parts.push(tail.slice(75));
|
|
260
|
+
}
|
|
266
261
|
}
|
|
267
262
|
}
|
|
263
|
+
indentedMsg = parts.map(part => ` | ${part}`).join('\n');
|
|
264
|
+
console.log(`\n${indentedMsg}`);
|
|
268
265
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
)
|
|
286
|
-
jobData.prohibitedCount++;
|
|
287
|
-
}
|
|
266
|
+
// Add statistics on the message to the report.
|
|
267
|
+
const msgTextLC = msgText.toLowerCase();
|
|
268
|
+
const msgLength = msgText.length;
|
|
269
|
+
jobData.logCount++;
|
|
270
|
+
jobData.logSize += msgLength;
|
|
271
|
+
if (errorWords.some(word => msgTextLC.includes(word))) {
|
|
272
|
+
jobData.errorLogCount++;
|
|
273
|
+
jobData.errorLogSize += msgLength;
|
|
274
|
+
}
|
|
275
|
+
const msgLC = msgText.toLowerCase();
|
|
276
|
+
if (
|
|
277
|
+
msgText.includes('403') && (msgLC.includes('status')
|
|
278
|
+
|| msgLC.includes('prohibited'))
|
|
279
|
+
) {
|
|
280
|
+
jobData.prohibitedCount++;
|
|
281
|
+
}
|
|
282
|
+
});
|
|
288
283
|
});
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
const page = await browserContext.newPage();
|
|
292
|
-
try {
|
|
284
|
+
// Replace the page with the first page (tab) of the context (window).
|
|
285
|
+
page = await browserContext.newPage();
|
|
293
286
|
// Wait until it is stable.
|
|
294
287
|
await page.waitForLoadState('domcontentloaded', {timeout: 5000});
|
|
295
288
|
// Navigate to the specified URL.
|
|
@@ -298,41 +291,44 @@ const launch = async (report, debug, waits, tempBrowserID, tempURL) => {
|
|
|
298
291
|
if (navResult.success) {
|
|
299
292
|
// Update the name of the current browser type and store it in the page.
|
|
300
293
|
page.browserID = browserID;
|
|
301
|
-
//
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
294
|
+
// Add the actual URL to the act.
|
|
295
|
+
act.actualURL = page.url();
|
|
296
|
+
// Get the response of the target server.
|
|
297
|
+
const {response} = navResult;
|
|
298
|
+
// Add the script nonce, if any, to the act.
|
|
299
|
+
const scriptNonce = await getNonce(response);
|
|
300
|
+
if (scriptNonce) {
|
|
301
|
+
report.jobData.lastScriptNonce = scriptNonce;
|
|
302
|
+
}
|
|
308
303
|
}
|
|
309
|
-
// Otherwise, if the navigation failed:
|
|
304
|
+
// Otherwise, i.e. if the launch or navigation failed:
|
|
310
305
|
else {
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
306
|
+
// Report this and abort the job.
|
|
307
|
+
actIndex = await addError(
|
|
308
|
+
true, true, report, actIndex, `ERROR: Launch failed (${navResult.error})`
|
|
309
|
+
);
|
|
310
|
+
page = null;
|
|
316
311
|
}
|
|
317
312
|
}
|
|
318
|
-
// If
|
|
313
|
+
// If an error occurred:
|
|
319
314
|
catch(error) {
|
|
320
|
-
//
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
}
|
|
315
|
+
// Report this and abort the job.
|
|
316
|
+
actIndex = await addError(
|
|
317
|
+
true, true, report, actIndex, `ERROR launching or navigating ${error.message}`
|
|
318
|
+
);
|
|
319
|
+
page = null;
|
|
320
|
+
};
|
|
327
321
|
}
|
|
328
322
|
// Otherwise, i.e. if the browser or device ID is invalid:
|
|
329
323
|
else {
|
|
330
|
-
//
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
324
|
+
// Report this and abort the job.
|
|
325
|
+
actIndex = await addError(
|
|
326
|
+
true,
|
|
327
|
+
true,
|
|
328
|
+
report,
|
|
329
|
+
actIndex,
|
|
330
|
+
`ERROR: Browser ${browserID}, device ${deviceID}, or URL ${url} invalid`
|
|
331
|
+
);
|
|
336
332
|
}
|
|
337
333
|
};
|
|
338
334
|
// Returns a string representing the date and time.
|
|
@@ -544,7 +540,7 @@ const addError = async(alsoLog, alsoAbort, report, actIndex, message) => {
|
|
|
544
540
|
}
|
|
545
541
|
};
|
|
546
542
|
// Recursively performs the acts in a report.
|
|
547
|
-
const doActs = async (report, actIndex
|
|
543
|
+
const doActs = async (report, actIndex) => {
|
|
548
544
|
const {acts} = report;
|
|
549
545
|
// If any more acts are to be performed:
|
|
550
546
|
if (actIndex > -1 && actIndex < acts.length) {
|
|
@@ -606,34 +602,119 @@ const doActs = async (report, actIndex, page) => {
|
|
|
606
602
|
}
|
|
607
603
|
// Otherwise, if the act is a launch:
|
|
608
604
|
else if (type === 'launch') {
|
|
609
|
-
// Launch
|
|
610
|
-
|
|
605
|
+
// Launch a browser, navigate to a page, and add the result to the act.
|
|
606
|
+
await launch(
|
|
611
607
|
report,
|
|
612
608
|
debug,
|
|
613
609
|
waits,
|
|
614
610
|
act.browserID || report.browserID || '',
|
|
615
|
-
act.url || report.target && report.target.url || ''
|
|
611
|
+
act.target && act.target.url || report.target && report.target.url || ''
|
|
616
612
|
);
|
|
617
|
-
// If
|
|
618
|
-
if (
|
|
619
|
-
//
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
613
|
+
// If this failed:
|
|
614
|
+
if (page.prevented) {
|
|
615
|
+
// Add this to the act.
|
|
616
|
+
act.prevented = true;
|
|
617
|
+
act.error = page.error || '';
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
// Otherwise, if the act performs tests of a tool:
|
|
621
|
+
else if (act.type === 'test') {
|
|
622
|
+
// Add a description of the tool to the act.
|
|
623
|
+
act.what = tools[act.which];
|
|
624
|
+
// Get the start time of the act.
|
|
625
|
+
const startTime = Date.now();
|
|
626
|
+
try {
|
|
627
|
+
// Get the time limit in seconds for the act.
|
|
628
|
+
const timeLimit = timeLimits[act.which] || 15;
|
|
629
|
+
// If a new browser is to be launched:
|
|
630
|
+
if (act.launch) {
|
|
631
|
+
// Launch it, navigate to a URL, and replace the page.
|
|
632
|
+
await launch(
|
|
633
|
+
report,
|
|
634
|
+
debug,
|
|
635
|
+
waits,
|
|
636
|
+
act.launch.browserID || report.browserID,
|
|
637
|
+
act.launch.target && act.launch.target.url || report.target.url
|
|
638
|
+
);
|
|
639
|
+
}
|
|
640
|
+
// If the page has not prevented the tool from testing:
|
|
641
|
+
if (! page.prevented) {
|
|
642
|
+
// Perform the specified tests of the tool.
|
|
643
|
+
const actReport = await require(`./tests/${act.which}`)
|
|
644
|
+
.reporter(page, report, actIndex, timeLimit);
|
|
645
|
+
// Add the data and result to the act.
|
|
646
|
+
act.data = actReport.data;
|
|
647
|
+
act.result = actReport.result;
|
|
648
|
+
// If the tool reported that the page prevented testing:
|
|
649
|
+
if (actReport.data.prevented) {
|
|
650
|
+
// Add prevention data to the job data.
|
|
651
|
+
report.jobData.preventions[act.which] = act.data.error;
|
|
652
|
+
}
|
|
629
653
|
}
|
|
630
654
|
}
|
|
631
|
-
//
|
|
632
|
-
|
|
633
|
-
//
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
655
|
+
// If the tool invocation failed:
|
|
656
|
+
catch(error) {
|
|
657
|
+
// Report the failure.
|
|
658
|
+
const message = error.message.slice(0, 400);
|
|
659
|
+
console.log(`ERROR: Test act ${act.which} failed (${message})`);
|
|
660
|
+
act.data.prevented = true;
|
|
661
|
+
act.data.error = act.data.error ? `${act.data.error}; ${message}` : message;
|
|
662
|
+
}
|
|
663
|
+
// Add the elapsed time of the tool to the report.
|
|
664
|
+
const time = Math.round((Date.now() - startTime) / 1000);
|
|
665
|
+
const {toolTimes} = report.jobData;
|
|
666
|
+
if (! toolTimes[act.which]) {
|
|
667
|
+
toolTimes[act.which] = 0;
|
|
668
|
+
}
|
|
669
|
+
toolTimes[act.which] += time;
|
|
670
|
+
const standard = report.standard || 'only';
|
|
671
|
+
// If the act was not prevented and standardization is required:
|
|
672
|
+
if (! act.data.prevented && ['also', 'only'].includes(standard)) {
|
|
673
|
+
// Initialize the standard result.
|
|
674
|
+
act.standardResult = {
|
|
675
|
+
totals: [0, 0, 0, 0],
|
|
676
|
+
instances: []
|
|
677
|
+
};
|
|
678
|
+
// Populate it.
|
|
679
|
+
standardize(act);
|
|
680
|
+
// Add a box ID and a path ID to each of its standard instances if missing.
|
|
681
|
+
for (const instance of act.standardResult.instances) {
|
|
682
|
+
const elementID = await identify(instance, page);
|
|
683
|
+
if (! instance.boxID) {
|
|
684
|
+
instance.boxID = elementID ? elementID.boxID : '';
|
|
685
|
+
}
|
|
686
|
+
if (! instance.pathID) {
|
|
687
|
+
instance.pathID = elementID ? elementID.pathID : '';
|
|
688
|
+
}
|
|
689
|
+
};
|
|
690
|
+
// If the original-format result is not to be included in the report:
|
|
691
|
+
if (standard === 'only') {
|
|
692
|
+
// Remove it.
|
|
693
|
+
delete act.result;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
const expectations = act.expect;
|
|
697
|
+
// If the act was not prevented and has expectations:
|
|
698
|
+
if (! act.data.prevented && expectations) {
|
|
699
|
+
// Initialize whether they were fulfilled.
|
|
700
|
+
act.expectations = [];
|
|
701
|
+
let failureCount = 0;
|
|
702
|
+
// For each expectation:
|
|
703
|
+
expectations.forEach(spec => {
|
|
704
|
+
// Add the its result to the act.
|
|
705
|
+
const truth = isTrue(act, spec);
|
|
706
|
+
act.expectations.push({
|
|
707
|
+
property: spec[0],
|
|
708
|
+
relation: spec[1],
|
|
709
|
+
criterion: spec[2],
|
|
710
|
+
actual: truth[0],
|
|
711
|
+
passed: truth[1]
|
|
712
|
+
});
|
|
713
|
+
if (! truth[1]) {
|
|
714
|
+
failureCount++;
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
act.expectationFailures = failureCount;
|
|
637
718
|
}
|
|
638
719
|
}
|
|
639
720
|
// Otherwise, if a current page exists:
|
|
@@ -805,91 +886,6 @@ const doActs = async (report, actIndex, page) => {
|
|
|
805
886
|
};
|
|
806
887
|
});
|
|
807
888
|
}
|
|
808
|
-
// Otherwise, if the act performs tests of a tool:
|
|
809
|
-
else if (act.type === 'test') {
|
|
810
|
-
// Add a description of the tool to the act.
|
|
811
|
-
act.what = tools[act.which];
|
|
812
|
-
// Get the start time of the act.
|
|
813
|
-
const startTime = Date.now();
|
|
814
|
-
try {
|
|
815
|
-
// Get the time limit in seconds for the act.
|
|
816
|
-
const timeLimit = timeLimits[act.which] || 15;
|
|
817
|
-
// Perform the specified tests of the tool.
|
|
818
|
-
const actReport = await require(`./tests/${act.which}`)
|
|
819
|
-
.reporter(page, report, actIndex, timeLimit);
|
|
820
|
-
// Add the data and result to the act.
|
|
821
|
-
act.data = actReport.data;
|
|
822
|
-
act.result = actReport.result;
|
|
823
|
-
// If the tool reported that the page prevented testing:
|
|
824
|
-
if (actReport.data.prevented) {
|
|
825
|
-
// Add prevention data to the job data.
|
|
826
|
-
report.jobData.preventions[act.which] = act.data.error;
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
// If the tool invocation failed:
|
|
830
|
-
catch(error) {
|
|
831
|
-
// Report the failure.
|
|
832
|
-
const message = error.message.slice(0, 400);
|
|
833
|
-
console.log(`ERROR: Test act ${act.which} failed (${message})`);
|
|
834
|
-
act.data.prevented = true;
|
|
835
|
-
act.data.error = act.data.error ? `${act.data.error}; ${message}` : message;
|
|
836
|
-
}
|
|
837
|
-
// Add the elapsed time of the tool to the report.
|
|
838
|
-
const time = Math.round((Date.now() - startTime) / 1000);
|
|
839
|
-
const {toolTimes} = report.jobData;
|
|
840
|
-
if (! toolTimes[act.which]) {
|
|
841
|
-
toolTimes[act.which] = 0;
|
|
842
|
-
}
|
|
843
|
-
toolTimes[act.which] += time;
|
|
844
|
-
// If the act was not prevented and standardization is required:
|
|
845
|
-
const standard = report.standard || 'only';
|
|
846
|
-
if (! act.data.prevented && ['also', 'only'].includes(standard)) {
|
|
847
|
-
// Initialize it.
|
|
848
|
-
act.standardResult = {
|
|
849
|
-
totals: [0, 0, 0, 0],
|
|
850
|
-
instances: []
|
|
851
|
-
};
|
|
852
|
-
// Populate it.
|
|
853
|
-
standardize(act);
|
|
854
|
-
// Add a box ID and a path ID to each of its standard instances if missing.
|
|
855
|
-
for (const instance of act.standardResult.instances) {
|
|
856
|
-
const elementID = await identify(instance, page);
|
|
857
|
-
if (! instance.boxID) {
|
|
858
|
-
instance.boxID = elementID ? elementID.boxID : '';
|
|
859
|
-
}
|
|
860
|
-
if (! instance.pathID) {
|
|
861
|
-
instance.pathID = elementID ? elementID.pathID : '';
|
|
862
|
-
}
|
|
863
|
-
};
|
|
864
|
-
// If the original-format result is not to be included in the report:
|
|
865
|
-
if (standard === 'only') {
|
|
866
|
-
// Remove it.
|
|
867
|
-
delete act.result;
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
// If the act was not prevented and has expectations:
|
|
871
|
-
const expectations = act.expect;
|
|
872
|
-
if (! act.data.prevented && expectations) {
|
|
873
|
-
// Initialize whether they were fulfilled.
|
|
874
|
-
act.expectations = [];
|
|
875
|
-
let failureCount = 0;
|
|
876
|
-
// For each expectation:
|
|
877
|
-
expectations.forEach(spec => {
|
|
878
|
-
const truth = isTrue(act, spec);
|
|
879
|
-
act.expectations.push({
|
|
880
|
-
property: spec[0],
|
|
881
|
-
relation: spec[1],
|
|
882
|
-
criterion: spec[2],
|
|
883
|
-
actual: truth[0],
|
|
884
|
-
passed: truth[1]
|
|
885
|
-
});
|
|
886
|
-
if (! truth[1]) {
|
|
887
|
-
failureCount++;
|
|
888
|
-
}
|
|
889
|
-
});
|
|
890
|
-
act.expectationFailures = failureCount;
|
|
891
|
-
}
|
|
892
|
-
}
|
|
893
889
|
// Otherwise, if the act is a move:
|
|
894
890
|
else if (moves[act.type]) {
|
|
895
891
|
const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
|
|
@@ -995,8 +991,7 @@ const doActs = async (report, actIndex, page) => {
|
|
|
995
991
|
console.log(`ERROR: Network busy after ${move} (${errorStart(error)})`);
|
|
996
992
|
act.result.idleTimely = false;
|
|
997
993
|
}
|
|
998
|
-
//
|
|
999
|
-
page = currentPage;
|
|
994
|
+
// Add the page URL to the result.
|
|
1000
995
|
act.result.newURL = page.url();
|
|
1001
996
|
}
|
|
1002
997
|
};
|
|
@@ -1337,7 +1332,7 @@ const doActs = async (report, actIndex, page) => {
|
|
|
1337
1332
|
}
|
|
1338
1333
|
act.endTime = Date.now();
|
|
1339
1334
|
// Perform any remaining acts if not aborted.
|
|
1340
|
-
await doActs(report, actIndex + 1
|
|
1335
|
+
await doActs(report, actIndex + 1);
|
|
1341
1336
|
}
|
|
1342
1337
|
// Otherwise, if all acts have been performed and the job succeeded:
|
|
1343
1338
|
else if (! report.jobData.abortTime) {
|