testaro 33.0.2 → 33.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -11
- package/call.js +3 -0
- package/dirWatch.js +6 -1
- package/netWatch.js +10 -7
- package/package.json +1 -1
- package/procs/standardize.js +7 -2
- package/procs/tellServer.js +1 -1
- package/run.js +27 -5
- package/watch.js +2 -3
- package/netWatch-old.js +0 -397
package/README.md
CHANGED
|
@@ -144,7 +144,6 @@ Here is an example of a job:
|
|
|
144
144
|
timeLimit: 65,
|
|
145
145
|
standard: 'only',
|
|
146
146
|
observe: false,
|
|
147
|
-
timeStamp: '231208T1200',
|
|
148
147
|
acts: [
|
|
149
148
|
{
|
|
150
149
|
type: 'launch',
|
|
@@ -164,12 +163,15 @@ Here is an example of a job:
|
|
|
164
163
|
batch: 'weborgs',
|
|
165
164
|
target: {
|
|
166
165
|
id: 'w3c',
|
|
167
|
-
|
|
168
|
-
|
|
166
|
+
what: 'World Wide Web Consortium',
|
|
167
|
+
which: 'https://www.w3c.org'
|
|
169
168
|
},
|
|
170
169
|
requester: 'user@domain.org'
|
|
171
170
|
},
|
|
172
|
-
|
|
171
|
+
timeStamp: '241208T1200',
|
|
172
|
+
creationTimeStamp: '241114T0328',
|
|
173
|
+
sendReportTo: 'https://localhost:3004/testapp/api/report',
|
|
174
|
+
mergeID: Q9
|
|
173
175
|
}
|
|
174
176
|
```
|
|
175
177
|
|
|
@@ -185,13 +187,17 @@ Job properties:
|
|
|
185
187
|
- `standard`: `'also'`, `'only'`, or `'no'`, indicating whether rule-violation instances are to be reported in tool-native formats and also in the Testaro standard format, only in the standard format, or only in the tool-native formats.
|
|
186
188
|
- `observe`: `true` or `false`, indicating whether tool and Testaro-rule invocations are to be reported as they occur to the server.
|
|
187
189
|
- `timeStamp`: a string in `yymmddThhMM` format, specifying a date and time before which the job is not to be performed.
|
|
190
|
+
- `creationTimeStamp`: a string in `yymmddThhMM` format, describing when the job was created.
|
|
188
191
|
- `acts`: an array of the acts to be performed (documented below).
|
|
189
192
|
- `sources`: an object describing where the job came from:
|
|
190
|
-
- `script
|
|
191
|
-
- `batch`
|
|
192
|
-
- `target
|
|
193
|
-
- `requester
|
|
194
|
-
- `
|
|
193
|
+
- `script`: the ID of the script (as used by Testilo) from which the job was made, or an empty string if none.
|
|
194
|
+
- `batch` : the ID of the batch (as used by Testilo) from which the target of this job was drawn, or an empty string if none.
|
|
195
|
+
- `target`: an object whose `id`, `what`, and `which` properties describe the target being tested by this job, or, if there was no batch, have empty strings as values.
|
|
196
|
+
- `requester`: the email address that should receive a notice of completion of the job, or an empty string if no notice is to be sent.
|
|
197
|
+
- `creationTimeStamp`: the date and time in `yymmddThhMM` format when the job was created.
|
|
198
|
+
- `timeStamp`: the date and time in `yymmddThhMM` format before which the job is not to be assigned.
|
|
199
|
+
- `sendReportTo`: the URL to which the report of the job is to be sent, or an empty string if the report is not to be sent to a server.
|
|
200
|
+
- `mergeID`: a randomly generated alphanumeric ID for the process (such as a merger, as performed by Testilo, of a script and a batch) that created the job, or an empty string if none.
|
|
195
201
|
|
|
196
202
|
### Reports
|
|
197
203
|
|
|
@@ -757,7 +763,7 @@ node call run be76p
|
|
|
757
763
|
|
|
758
764
|
In the second example, `be76p` is the initial characters of the ID of a job saved as a JSON file in the `todo` subdirectory of the `process.env.JOBDIR` directory.
|
|
759
765
|
|
|
760
|
-
The `call` module will find the first job file with a matching name if an argument is given, or the first job file if not. Then the module will execute the `doJob` function of the `run` module on the job, save the report in the `raw` subdirectory of the `
|
|
766
|
+
The `call` module will find the first job file with a matching name if an argument is given, or the first job file if not. Then the module will execute the `doJob` function of the `run` module on the job, save the report in the `raw` subdirectory of the `REPORTDIR` directory, and archive the job file in the `done` subdirectory of the `JOBDIR` directory.
|
|
761
767
|
|
|
762
768
|
#### Watch
|
|
763
769
|
|
|
@@ -796,7 +802,7 @@ An instance of Testaro is an _agent_ and has an identifier specified by `process
|
|
|
796
802
|
|
|
797
803
|
The URLs polled by Testaro are specified by `process.env.JOB_URLS`. The format of that environment variable is a `+`-delimited list of URLs, including schemes. If one of the URLs is `https://testrunner.org/a11ytest/api/job`, and if a Testaro instance has the agent ID `tester3`, then a job request is a `GET` request to `https://testrunner.org/a11ytest/api/job?agent=tester3`.
|
|
798
804
|
|
|
799
|
-
Once a Testaro instance obtains a network job, Testaro performs it and adds the result data to the job, which then becomes the job report. Testaro sends the report in a `POST` request to the URL specified by the `
|
|
805
|
+
Once a Testaro instance obtains a network job, Testaro performs it and adds the result data to the job, which then becomes the job report. Testaro sends the report in a `POST` request to the URL specified by the `sendReportTo` property of the job.
|
|
800
806
|
|
|
801
807
|
Network watching can be repeated or 1-job. One-job watching stops after 1 job has been performed.
|
|
802
808
|
|
package/call.js
CHANGED
|
@@ -69,6 +69,9 @@ const callRun = async jobIDStart => {
|
|
|
69
69
|
// Get it.
|
|
70
70
|
const jobJSON = await fs.readFile(`${todoDir}/${jobFileName}`, 'utf8');
|
|
71
71
|
const report = JSON.parse(jobJSON);
|
|
72
|
+
// Ensure it does not specify server properties.
|
|
73
|
+
report.observe = false;
|
|
74
|
+
report.sendReportTo = '';
|
|
72
75
|
// Run it.
|
|
73
76
|
await doJob(report);
|
|
74
77
|
// Archive it.
|
package/dirWatch.js
CHANGED
|
@@ -44,7 +44,7 @@ const reportDir = process.env.REPORTDIR;
|
|
|
44
44
|
// Gets a segment of a timestamp.
|
|
45
45
|
const tsPart = (timeStamp, startIndex) => timeStamp.slice(startIndex, startIndex + 2);
|
|
46
46
|
// Returns a string representing the date and time.
|
|
47
|
-
const nowString = () => (new Date()).toISOString().slice(
|
|
47
|
+
const nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
48
48
|
// Gets date of a timestamp.
|
|
49
49
|
const dateOf = ts => {
|
|
50
50
|
const dateString = `20${tsPart(ts, 0)}-${tsPart(ts, 2)}-${tsPart(ts, 4)}`;
|
|
@@ -117,6 +117,11 @@ exports.dirWatch = async (isForever, intervalInSeconds) => {
|
|
|
117
117
|
try {
|
|
118
118
|
const job = JSON.parse(jobJSON);
|
|
119
119
|
const report = JSON.parse(jobJSON);
|
|
120
|
+
// Ensure it has no server properties.
|
|
121
|
+
job.observe = false;
|
|
122
|
+
job.sendReportTo = '';
|
|
123
|
+
report.observe = false;
|
|
124
|
+
report.sendReportTo = '';
|
|
120
125
|
const {id} = job;
|
|
121
126
|
console.log(`Directory job ${id} ready to do (${nowString()})`);
|
|
122
127
|
// Perform it.
|
package/netWatch.js
CHANGED
|
@@ -45,7 +45,7 @@ const agent = process.env.AGENT;
|
|
|
45
45
|
// FUNCTIONS
|
|
46
46
|
|
|
47
47
|
// Returns a string representing the date and time.
|
|
48
|
-
const nowString = () => (new Date()).toISOString().slice(2,
|
|
48
|
+
const nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
49
49
|
// Waits.
|
|
50
50
|
const wait = ms => {
|
|
51
51
|
return new Promise(resolve => {
|
|
@@ -141,7 +141,7 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
|
|
|
141
141
|
}
|
|
142
142
|
// Otherwise, i.e. if there was a job or a message:
|
|
143
143
|
else {
|
|
144
|
-
const {message,
|
|
144
|
+
const {id, message, sendReportTo, sources} = contentObj;
|
|
145
145
|
// If the server sent a message, not a job:
|
|
146
146
|
if (message) {
|
|
147
147
|
// Report it.
|
|
@@ -157,11 +157,10 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
|
|
|
157
157
|
// Add the agent to the job.
|
|
158
158
|
sources.agent = agent;
|
|
159
159
|
// If the job specifies a report destination:
|
|
160
|
-
const {sendReportTo} = sources;
|
|
161
160
|
if (sendReportTo) {
|
|
162
161
|
// Perform the job, adding result data to it.
|
|
163
162
|
const target = sources.target.which;
|
|
164
|
-
console.log(`${logStart}job ${id} (${nowString()}`);
|
|
163
|
+
console.log(`${logStart}job ${id} (${nowString()})`);
|
|
165
164
|
console.log(`>> It will test ${target}`);
|
|
166
165
|
console.log(`>> It will send report to ${sendReportTo}`);
|
|
167
166
|
await doJob(contentObj);
|
|
@@ -220,7 +219,9 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
|
|
|
220
219
|
// If the report submission throws an error:
|
|
221
220
|
.on('error', async error => {
|
|
222
221
|
// Report this.
|
|
223
|
-
console.log(
|
|
222
|
+
console.log(
|
|
223
|
+
`ERROR in report submission: ${reportLogStart}error message ${error.message}\n`
|
|
224
|
+
);
|
|
224
225
|
resolve(true);
|
|
225
226
|
})
|
|
226
227
|
// Finish submitting the report.
|
|
@@ -230,8 +231,8 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
|
|
|
230
231
|
else {
|
|
231
232
|
// Report this.
|
|
232
233
|
const message = `ERROR: ${logStart}job with no report destination`;
|
|
233
|
-
serveObject({message}, response);
|
|
234
234
|
console.log(message);
|
|
235
|
+
serveObject({message}, response);
|
|
235
236
|
resolve(true);
|
|
236
237
|
}
|
|
237
238
|
}
|
|
@@ -249,7 +250,9 @@ exports.netWatch = async (isForever, intervalInSeconds, isCertTolerant = true) =
|
|
|
249
250
|
// If processing the server response throws an error:
|
|
250
251
|
catch(error) {
|
|
251
252
|
// Report this.
|
|
252
|
-
console.log(
|
|
253
|
+
console.log(
|
|
254
|
+
`ERROR processing server response: ${error.message} (response ${content.slice(0, 1000)})`
|
|
255
|
+
);
|
|
253
256
|
resolve(true);
|
|
254
257
|
}
|
|
255
258
|
});
|
package/package.json
CHANGED
package/procs/standardize.js
CHANGED
|
@@ -395,7 +395,6 @@ const convert = (toolName, data, result, standardResult) => {
|
|
|
395
395
|
&& ruleResult.message.actual
|
|
396
396
|
&& ruleResult.message.actual.description
|
|
397
397
|
) {
|
|
398
|
-
|
|
399
398
|
const what = ruleResult.message.actual.description;
|
|
400
399
|
// Get its differentiated ID if any.
|
|
401
400
|
const ruleData = aslintData[ruleID];
|
|
@@ -409,10 +408,16 @@ const convert = (toolName, data, result, standardResult) => {
|
|
|
409
408
|
}
|
|
410
409
|
}
|
|
411
410
|
const xpath = ruleResult.element && ruleResult.element.xpath || '';
|
|
412
|
-
|
|
411
|
+
let tagName = xpath
|
|
413
412
|
&& xpath.replace(/^.*\//, '').replace(/[^-\w].*$/, '').toUpperCase()
|
|
414
413
|
|| '';
|
|
414
|
+
if (! tagName && finalRuleID.endsWith('_svg')) {
|
|
415
|
+
tagName = 'SVG';
|
|
416
|
+
}
|
|
415
417
|
const excerpt = ruleResult.element && ruleResult.element.html || '';
|
|
418
|
+
if (! tagName && /^<[a-z]+[ >]/.test(excerpt)) {
|
|
419
|
+
tagName = excerpt.slice(1).replace(/[ >]+/, '').toUpperCase();
|
|
420
|
+
}
|
|
416
421
|
const idDraft = excerpt && excerpt.replace(/^[^[>]+id="/, 'id=').replace(/".*$/, '');
|
|
417
422
|
const id = idDraft && idDraft.length > 3 && idDraft.startsWith('id=')
|
|
418
423
|
? idDraft.slice(3)
|
package/procs/tellServer.js
CHANGED
|
@@ -35,7 +35,7 @@ const agent = process.env.AGENT;
|
|
|
35
35
|
|
|
36
36
|
// Sends a notification to an observer.
|
|
37
37
|
exports.tellServer = (report, messageParams, logMessage) => {
|
|
38
|
-
const observer = report.
|
|
38
|
+
const observer = report.sendReportTo.replace(/report$/, 'granular');
|
|
39
39
|
const whoParams = `agent=${agent}&jobID=${report.id || ''}`;
|
|
40
40
|
const wholeURL = `${observer}?${whoParams}&${messageParams}`;
|
|
41
41
|
const client = wholeURL.startsWith('https://') ? httpsClient : httpClient;
|
package/run.js
CHANGED
|
@@ -224,11 +224,31 @@ const isValidAct = act => {
|
|
|
224
224
|
return false;
|
|
225
225
|
}
|
|
226
226
|
};
|
|
227
|
+
// Inserts a character periodically in a string.
|
|
228
|
+
const punctuate = (string, insertion, chunkSize) => {
|
|
229
|
+
const segments = [];
|
|
230
|
+
let startIndex = 0;
|
|
231
|
+
while (startIndex < string.length) {
|
|
232
|
+
segments.push(string.slice(startIndex, startIndex + chunkSize));
|
|
233
|
+
startIndex += chunkSize;
|
|
234
|
+
}
|
|
235
|
+
return segments.join(insertion);
|
|
236
|
+
};
|
|
237
|
+
// Converts a compact timestamp to a date.
|
|
238
|
+
const dateOf = timeStamp => {
|
|
239
|
+
if (/^\d{6}T\d{4}$/.test(timeStamp)) {
|
|
240
|
+
const dateString = punctuate(timeStamp.slice(0, 6), '-', 2);
|
|
241
|
+
const timeString = punctuate(timeStamp.slice(7, 11), ':', 2);
|
|
242
|
+
return new Date(`20${dateString}T${timeString}Z`);
|
|
243
|
+
} else {
|
|
244
|
+
return null;
|
|
245
|
+
}
|
|
246
|
+
};
|
|
227
247
|
// Validates a report object.
|
|
228
248
|
const isValidReport = report => {
|
|
229
249
|
if (report) {
|
|
230
250
|
// Return whether the report is valid.
|
|
231
|
-
const {id, what, strict, timeLimit, acts, sources,
|
|
251
|
+
const {id, what, strict, timeLimit, acts, sources, creationTimeStamp, timeStamp} = report;
|
|
232
252
|
if (! id || typeof id !== 'string') {
|
|
233
253
|
return 'Bad report ID';
|
|
234
254
|
}
|
|
@@ -278,11 +298,13 @@ const isValidReport = report => {
|
|
|
278
298
|
if (typeof sources.script !== 'string') {
|
|
279
299
|
return 'Bad source script';
|
|
280
300
|
}
|
|
281
|
-
if (
|
|
282
|
-
|
|
301
|
+
if (
|
|
302
|
+
! (creationTimeStamp && typeof creationTimeStamp === 'string' && dateOf(creationTimeStamp))
|
|
303
|
+
) {
|
|
304
|
+
return 'bad job creation time stamp';
|
|
283
305
|
}
|
|
284
306
|
if (! (timeStamp && typeof timeStamp === 'string')) {
|
|
285
|
-
return 'bad report
|
|
307
|
+
return 'bad report time stamp';
|
|
286
308
|
}
|
|
287
309
|
return '';
|
|
288
310
|
}
|
|
@@ -530,7 +552,7 @@ const launch = async (report, typeName, url, debug, waits, isLowMotion = false)
|
|
|
530
552
|
}
|
|
531
553
|
};
|
|
532
554
|
// Returns a string representing the date and time.
|
|
533
|
-
const nowString = () => (new Date()).toISOString().slice(
|
|
555
|
+
const nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
534
556
|
// Returns the first line of an error message.
|
|
535
557
|
const errorStart = error => error.message.replace(/\n.+/s, '');
|
|
536
558
|
// Normalizes spacing characters and cases in a string.
|
package/watch.js
CHANGED
|
@@ -47,7 +47,7 @@ const agent = process.env.AGENT;
|
|
|
47
47
|
// ########## FUNCTIONS
|
|
48
48
|
|
|
49
49
|
// Returns a string representing the date and time.
|
|
50
|
-
const nowString = () => (new Date()).toISOString().slice(
|
|
50
|
+
const nowString = () => (new Date()).toISOString().slice(2, 16);
|
|
51
51
|
// Waits.
|
|
52
52
|
const wait = ms => {
|
|
53
53
|
return new Promise(resolve => {
|
|
@@ -205,7 +205,7 @@ const checkNetJob = async (servers, serverIndex, isForever, interval, noJobCount
|
|
|
205
205
|
}
|
|
206
206
|
// Otherwise, i.e. if there was a job or a message:
|
|
207
207
|
else {
|
|
208
|
-
const {message,
|
|
208
|
+
const {id, message, sendReportTo, sources} = contentObj;
|
|
209
209
|
// If the server sent a message, not a job:
|
|
210
210
|
if (message) {
|
|
211
211
|
// Report it.
|
|
@@ -218,7 +218,6 @@ const checkNetJob = async (servers, serverIndex, isForever, interval, noJobCount
|
|
|
218
218
|
// Add the agent to it.
|
|
219
219
|
sources.agent = agent;
|
|
220
220
|
// If the job specifies a report destination:
|
|
221
|
-
const {sendReportTo} = sources;
|
|
222
221
|
if (sendReportTo) {
|
|
223
222
|
// Perform the job, adding result data to it.
|
|
224
223
|
const testee = sources.target.which;
|
package/netWatch-old.js
DELETED
|
@@ -1,397 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
© 2022–2023 CVS Health and/or one of its affiliates. All rights reserved.
|
|
3
|
-
|
|
4
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
6
|
-
in the Software without restriction, including without limitation the rights
|
|
7
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
9
|
-
furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in all
|
|
12
|
-
copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
20
|
-
SOFTWARE.
|
|
21
|
-
*/
|
|
22
|
-
|
|
23
|
-
/*
|
|
24
|
-
netWatch.js
|
|
25
|
-
Module for watching for a network job and running it when found.
|
|
26
|
-
*/
|
|
27
|
-
|
|
28
|
-
// ########## IMPORTS
|
|
29
|
-
|
|
30
|
-
// Module to keep secrets.
|
|
31
|
-
require('dotenv').config();
|
|
32
|
-
// Module to read and write files.
|
|
33
|
-
const fs = require('fs/promises');
|
|
34
|
-
// Module to make requests to servers.
|
|
35
|
-
const httpClient = require('http');
|
|
36
|
-
const httpsClient = require('https');
|
|
37
|
-
// Module to perform jobs.
|
|
38
|
-
const {doJob} = require('./run');
|
|
39
|
-
|
|
40
|
-
// CONSTANTS
|
|
41
|
-
|
|
42
|
-
const jobDir = process.env.JOBDIR;
|
|
43
|
-
const jobURLs = process.env.JOB_URLS;
|
|
44
|
-
const reportDir = process.env.REPORTDIR;
|
|
45
|
-
const agent = process.env.AGENT;
|
|
46
|
-
|
|
47
|
-
// ########## FUNCTIONS
|
|
48
|
-
|
|
49
|
-
// Returns a string representing the date and time.
|
|
50
|
-
const nowString = () => (new Date()).toISOString().slice(0, 19);
|
|
51
|
-
// Waits.
|
|
52
|
-
const wait = ms => {
|
|
53
|
-
return new Promise(resolve => {
|
|
54
|
-
setTimeout(() => {
|
|
55
|
-
resolve('');
|
|
56
|
-
}, ms);
|
|
57
|
-
});
|
|
58
|
-
};
|
|
59
|
-
// Serves an object in JSON format.
|
|
60
|
-
const serveObject = (object, response) => {
|
|
61
|
-
response.setHeader('Content-Type', 'application/json; charset=utf-8');
|
|
62
|
-
response.end(JSON.stringify(object));
|
|
63
|
-
};
|
|
64
|
-
// Writes a directory report.
|
|
65
|
-
const writeDirReport = async report => {
|
|
66
|
-
const jobID = report && report.id;
|
|
67
|
-
if (jobID) {
|
|
68
|
-
try {
|
|
69
|
-
const reportJSON = JSON.stringify(report, null, 2);
|
|
70
|
-
const reportName = `${jobID}.json`;
|
|
71
|
-
await fs.mkdir(reportDir, {recursive: true});
|
|
72
|
-
await fs.writeFile(`${reportDir}/${reportName}`, reportJSON);
|
|
73
|
-
console.log(`Report ${reportName} saved in ${reportDir}`);
|
|
74
|
-
}
|
|
75
|
-
catch(error) {
|
|
76
|
-
console.log(`ERROR: Failed to write report ${jobID} in ${reportDir} (${error.message})`);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
else {
|
|
80
|
-
console.log('ERROR: Job has no ID');
|
|
81
|
-
}
|
|
82
|
-
};
|
|
83
|
-
// Archives a job.
|
|
84
|
-
const archiveJob = async (job, isFile) => {
|
|
85
|
-
// Save the job in the done subdirectory.
|
|
86
|
-
const {id} = job;
|
|
87
|
-
const jobJSON = JSON.stringify(job, null, 2);
|
|
88
|
-
await fs.mkdir(`${jobDir}/done`, {recursive: true});
|
|
89
|
-
await fs.writeFile(`${jobDir}/done/${id}.json`, jobJSON);
|
|
90
|
-
// If the job had been saved as a file in the todo subdirectory:
|
|
91
|
-
if (isFile) {
|
|
92
|
-
// Delete the file.
|
|
93
|
-
await fs.rm(`${jobDir}/todo/${id}.json`);
|
|
94
|
-
}
|
|
95
|
-
};
|
|
96
|
-
// Checks for a directory job and, if found, performs and reports it, once or repeatedly.
|
|
97
|
-
const checkDirJob = async (isForever, interval) => {
|
|
98
|
-
try {
|
|
99
|
-
// If there are any jobs in the watched directory:
|
|
100
|
-
const toDoFileNames = await fs.readdir(`${jobDir}/todo`);
|
|
101
|
-
const jobFileNames = toDoFileNames.filter(fileName => fileName.endsWith('.json'));
|
|
102
|
-
if (jobFileNames.length) {
|
|
103
|
-
// If the first one is ready to do:
|
|
104
|
-
const firstJobTime = jobFileNames[0].replace(/-.+$/, '');
|
|
105
|
-
if (Date.now() > dateOf(firstJobTime)) {
|
|
106
|
-
// Perform it.
|
|
107
|
-
const jobJSON = await fs.readFile(`${jobDir}/todo/${jobFileNames[0]}`, 'utf8');
|
|
108
|
-
try {
|
|
109
|
-
const job = JSON.parse(jobJSON, null, 2);
|
|
110
|
-
const {id} = job;
|
|
111
|
-
console.log(`Directory job ${id} found (${nowString()})`);
|
|
112
|
-
await doJob(job);
|
|
113
|
-
console.log(`Job ${id} finished (${nowString()})`);
|
|
114
|
-
// Report it.
|
|
115
|
-
await writeDirReport(job);
|
|
116
|
-
// Archive it.
|
|
117
|
-
await archiveJob(job, true);
|
|
118
|
-
console.log(`Job ${id} archived in ${jobDir} (${nowString()})`);
|
|
119
|
-
// If watching is repetitive:
|
|
120
|
-
if (isForever) {
|
|
121
|
-
// Wait 2 seconds.
|
|
122
|
-
await wait(2000);
|
|
123
|
-
// Check the directory again.
|
|
124
|
-
checkDirJob(true, interval);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
catch(error) {
|
|
128
|
-
console.log(`ERROR processing directory job (${error.message})`);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
// Otherwise, i.e. if the first one is not yet ready to do:
|
|
132
|
-
else {
|
|
133
|
-
// Report this.
|
|
134
|
-
console.log(`All jobs in ${jobDir} not yet ready to do (${nowString()})`);
|
|
135
|
-
// Wait for the specified interval.
|
|
136
|
-
await wait(1000 * interval);
|
|
137
|
-
// Check the directory again.
|
|
138
|
-
await checkDirJob(true, interval);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
// Otherwise, i.e. if there are no more jobs in the watched directory:
|
|
142
|
-
else {
|
|
143
|
-
console.log(`No job in ${jobDir} (${nowString()})`);
|
|
144
|
-
// Wait for the specified interval.
|
|
145
|
-
await wait(1000 * interval);
|
|
146
|
-
// Check the directory again.
|
|
147
|
-
await checkDirJob(true, interval);
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
catch(error) {
|
|
151
|
-
console.log(`ERROR: Directory watching failed (${error.message})`);
|
|
152
|
-
}
|
|
153
|
-
};
|
|
154
|
-
// Checks servers for a network job.
|
|
155
|
-
const checkNetJob = async (servers, serverIndex, isForever, interval, noJobCount) => {
|
|
156
|
-
// If all servers are jobless:
|
|
157
|
-
if (noJobCount === servers.length) {
|
|
158
|
-
// Wait for the specified interval.
|
|
159
|
-
await wait(1000 * interval);
|
|
160
|
-
// Reset the count of jobless servers.
|
|
161
|
-
noJobCount = 0;
|
|
162
|
-
}
|
|
163
|
-
// Otherwise, i.e. if any server may still have a job:
|
|
164
|
-
else {
|
|
165
|
-
// Wait 2 seconds.
|
|
166
|
-
await wait(2000);
|
|
167
|
-
}
|
|
168
|
-
// If the last server has been checked:
|
|
169
|
-
serverIndex = serverIndex % servers.length;
|
|
170
|
-
if (serverIndex === 0) {
|
|
171
|
-
// Report this.
|
|
172
|
-
console.log('--');
|
|
173
|
-
}
|
|
174
|
-
// Check the next server.
|
|
175
|
-
const server = servers[serverIndex];
|
|
176
|
-
const client = server.startsWith('https://') ? httpsClient : httpClient;
|
|
177
|
-
const fullURL = `${server}?agent=${agent}`;
|
|
178
|
-
const logStart = `Requested job from server ${server} and got `;
|
|
179
|
-
// Tolerate unrecognized certificate authorities if the environment specifies.
|
|
180
|
-
const ruOpt = process.env.REJECT_UNAUTHORIZED === 'false' ? {rejectUnauthorized: false} : {};
|
|
181
|
-
client.request(fullURL, ruOpt, response => {
|
|
182
|
-
const chunks = [];
|
|
183
|
-
response
|
|
184
|
-
// If the response to the job request threw an error:
|
|
185
|
-
.on('error', async error => {
|
|
186
|
-
// Report it.
|
|
187
|
-
console.log(`${logStart}error message ${error.message}`);
|
|
188
|
-
// Check the next server.
|
|
189
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
190
|
-
})
|
|
191
|
-
.on('data', chunk => {
|
|
192
|
-
chunks.push(chunk);
|
|
193
|
-
})
|
|
194
|
-
// When the response arrives:
|
|
195
|
-
.on('end', async () => {
|
|
196
|
-
const content = chunks.join('');
|
|
197
|
-
try {
|
|
198
|
-
// If there was no job to do:
|
|
199
|
-
let contentObj = JSON.parse(content);
|
|
200
|
-
if (! Object.keys(contentObj).length) {
|
|
201
|
-
// Report this.
|
|
202
|
-
console.log(`No job to do at ${server}`);
|
|
203
|
-
// Check the next server.
|
|
204
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
205
|
-
}
|
|
206
|
-
// Otherwise, i.e. if there was a job or a message:
|
|
207
|
-
else {
|
|
208
|
-
const {message, id, sources} = contentObj;
|
|
209
|
-
// If the server sent a message, not a job:
|
|
210
|
-
if (message) {
|
|
211
|
-
// Report it.
|
|
212
|
-
console.log(`${logStart}${message}`);
|
|
213
|
-
// Check the next server.
|
|
214
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
215
|
-
}
|
|
216
|
-
// Otherwise, if the server sent a valid job:
|
|
217
|
-
else if (id && sources && sources.target && sources.target.which) {
|
|
218
|
-
// Add the agent to it.
|
|
219
|
-
sources.agent = agent;
|
|
220
|
-
// If the job specifies a report destination:
|
|
221
|
-
const {sendReportTo} = sources;
|
|
222
|
-
if (sendReportTo) {
|
|
223
|
-
// Perform the job, adding result data to it.
|
|
224
|
-
const testee = sources.target.which;
|
|
225
|
-
console.log(
|
|
226
|
-
`${logStart}job ${id} (${nowString()})\n>> It will test ${testee}\n>> It will send report to ${sendReportTo}\n`
|
|
227
|
-
);
|
|
228
|
-
await doJob(contentObj);
|
|
229
|
-
let reportJSON = JSON.stringify(contentObj, null, 2);
|
|
230
|
-
console.log(`Job ${id} finished (${nowString()})`);
|
|
231
|
-
// Send the report to the specified server.
|
|
232
|
-
console.log(`Sending report ${id} to ${sendReportTo}`);
|
|
233
|
-
const reportClient = sendReportTo.startsWith('https://') ? httpsClient : httpClient;
|
|
234
|
-
const reportLogStart = `Sent report ${id} to ${sendReportTo} and got `;
|
|
235
|
-
reportClient.request(sendReportTo, {method: 'POST'}, repResponse => {
|
|
236
|
-
const chunks = [];
|
|
237
|
-
repResponse
|
|
238
|
-
// If the response to the report threw an error:
|
|
239
|
-
.on('error', async error => {
|
|
240
|
-
// Report this.
|
|
241
|
-
console.log(`${reportLogStart}error message ${error.message}\n`);
|
|
242
|
-
// Check the next server.
|
|
243
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
244
|
-
})
|
|
245
|
-
.on('data', chunk => {
|
|
246
|
-
chunks.push(chunk);
|
|
247
|
-
})
|
|
248
|
-
// When the response arrives:
|
|
249
|
-
.on('end', async () => {
|
|
250
|
-
const content = chunks.join('');
|
|
251
|
-
try {
|
|
252
|
-
// If the server sent a message, as expected:
|
|
253
|
-
const ackObj = JSON.parse(content);
|
|
254
|
-
const {message} = ackObj;
|
|
255
|
-
if (message) {
|
|
256
|
-
// Report it.
|
|
257
|
-
console.log(`${reportLogStart}${message}\n`);
|
|
258
|
-
// Free the memory used by the report.
|
|
259
|
-
reportJSON = '';
|
|
260
|
-
contentObj = {};
|
|
261
|
-
// Check the next server.
|
|
262
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, 0);
|
|
263
|
-
}
|
|
264
|
-
// Otherwise, i.e. if the server sent anything else:
|
|
265
|
-
else {
|
|
266
|
-
// Report it.
|
|
267
|
-
console.log(
|
|
268
|
-
`ERROR: ${reportLogStart}status ${repResponse.statusCode} and error message ${JSON.stringify(ackObj, null, 2)}\n`
|
|
269
|
-
);
|
|
270
|
-
// Check the next server, disregarding the failed job.
|
|
271
|
-
await checkNetJob(
|
|
272
|
-
servers, serverIndex + 1, isForever, interval, noJobCount + 1
|
|
273
|
-
);
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
// If processing the report threw an error:
|
|
277
|
-
catch(error) {
|
|
278
|
-
// Report it.
|
|
279
|
-
console.log(
|
|
280
|
-
`ERROR: ${reportLogStart}status ${repResponse.statusCode}, error message ${error.message}, and response ${content.slice(0, 1000)}\n`
|
|
281
|
-
);
|
|
282
|
-
// Check the next server, disregarding the failed job.
|
|
283
|
-
await checkNetJob(
|
|
284
|
-
servers, serverIndex + 1, isForever, interval, noJobCount + 1
|
|
285
|
-
);
|
|
286
|
-
}
|
|
287
|
-
});
|
|
288
|
-
})
|
|
289
|
-
// If the report submission throws an error:
|
|
290
|
-
.on('error', async error => {
|
|
291
|
-
// Report this.
|
|
292
|
-
console.log(`ERROR: ${reportLogStart}error message ${error.message}\n`);
|
|
293
|
-
// Check the next server, disregarding the failed job.
|
|
294
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
295
|
-
})
|
|
296
|
-
// Finish submitting the report.
|
|
297
|
-
.end(reportJSON);
|
|
298
|
-
}
|
|
299
|
-
// Otherwise, i.e. if the job specifies no report destination:
|
|
300
|
-
else {
|
|
301
|
-
// Report this.
|
|
302
|
-
const message = `ERROR: ${logStart}job with no report destination`;
|
|
303
|
-
serveObject({message}, response);
|
|
304
|
-
console.log(message);
|
|
305
|
-
// Check the next server, disregarding the defective job.
|
|
306
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
// Otherwise, if the server sent an invalid job:
|
|
310
|
-
else {
|
|
311
|
-
// Report this.
|
|
312
|
-
const message
|
|
313
|
-
= `ERROR: ${logStart}invalid job:\n${JSON.stringify(contentObj, null, 2)}`;
|
|
314
|
-
console.log(message);
|
|
315
|
-
serveObject({message}, response);
|
|
316
|
-
// Check the next server, disregarding the defective job.
|
|
317
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
// If an error is thrown:
|
|
322
|
-
catch(error) {
|
|
323
|
-
// Report this.
|
|
324
|
-
console.log(`ERROR: ${error.message} (response ${content.slice(0, 1000)})`);
|
|
325
|
-
// Check the next server.
|
|
326
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
327
|
-
}
|
|
328
|
-
});
|
|
329
|
-
})
|
|
330
|
-
// If the job request throws an error:
|
|
331
|
-
.on('error', async error => {
|
|
332
|
-
// If it was a refusal to connect:
|
|
333
|
-
const {message} = error;
|
|
334
|
-
if (message.includes('ECONNREFUSED')) {
|
|
335
|
-
// Report this.
|
|
336
|
-
console.log(`${logStart}no connection`);
|
|
337
|
-
// Check the next server.
|
|
338
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
339
|
-
}
|
|
340
|
-
// Otherwise, if it was a DNS failure:
|
|
341
|
-
else if (message.includes('ENOTFOUND')) {
|
|
342
|
-
// Report this.
|
|
343
|
-
console.log(`${logStart}no domain name resolution`);
|
|
344
|
-
// Check the next server.
|
|
345
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
346
|
-
}
|
|
347
|
-
// Otherwise, i.e. if it was any other error:
|
|
348
|
-
else {
|
|
349
|
-
// Report this.
|
|
350
|
-
console.log(
|
|
351
|
-
`ERROR: ${logStart}no response, but got error message ${error.message.slice(0, 200)}`
|
|
352
|
-
);
|
|
353
|
-
// Check the next server.
|
|
354
|
-
await checkNetJob(servers, serverIndex + 1, isForever, interval, noJobCount + 1);
|
|
355
|
-
}
|
|
356
|
-
})
|
|
357
|
-
// Finish sending the request.
|
|
358
|
-
.end();
|
|
359
|
-
};
|
|
360
|
-
// Composes an interval description.
|
|
361
|
-
const intervalSpec = interval => {
|
|
362
|
-
if (interval > -1) {
|
|
363
|
-
return `repeatedly, with ${interval}-second intervals `;
|
|
364
|
-
}
|
|
365
|
-
else {
|
|
366
|
-
return '';
|
|
367
|
-
}
|
|
368
|
-
};
|
|
369
|
-
// Checks for a directory job, performs it, and submits a report, once or repeatedly.
|
|
370
|
-
exports.dirWatch = async (isForever, interval = 300) => {
|
|
371
|
-
console.log(`Directory watching started ${intervalSpec(interval)}(${nowString()})\n`);
|
|
372
|
-
// Start the checking.
|
|
373
|
-
await checkDirJob(isForever, interval);
|
|
374
|
-
};
|
|
375
|
-
// Checks for a network job, performs it, and submits a report, once or repeatedly.
|
|
376
|
-
exports.netWatch = async (isForever, interval = 300) => {
|
|
377
|
-
console.log('Starting netWatch');
|
|
378
|
-
// If the servers to be checked are valid:
|
|
379
|
-
const servers = jobURLs
|
|
380
|
-
.split('+')
|
|
381
|
-
.map(url => [Math.random(), url])
|
|
382
|
-
.sort((a, b) => a[0] - b[0])
|
|
383
|
-
.map(pair => pair[1]);
|
|
384
|
-
if (
|
|
385
|
-
servers
|
|
386
|
-
&& servers.length
|
|
387
|
-
&& servers
|
|
388
|
-
.every(server => ['http://', 'https://'].some(prefix => server.startsWith(prefix)))
|
|
389
|
-
) {
|
|
390
|
-
console.log(`Network watching started ${intervalSpec(interval)}(${nowString()})\n`);
|
|
391
|
-
// Start checking.
|
|
392
|
-
await checkNetJob(servers, 0, isForever, interval, 0);
|
|
393
|
-
}
|
|
394
|
-
else {
|
|
395
|
-
console.log('ERROR: List of job URLs invalid');
|
|
396
|
-
}
|
|
397
|
-
};
|