testilo 28.0.0 → 29.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -93
- package/call.js +205 -140
- package/compare.js +6 -6
- package/{procs/analyze/credit.js → credit.js} +6 -17
- package/digest.js +8 -15
- package/merge.js +12 -0
- package/package.json +1 -1
- package/procs/analyze/ruleCounts.js +2 -2
- package/procs/compare/tcp40/index.html +3 -2
- package/procs/compare/tcp40/index.js +22 -41
- package/procs/score/tic40.js +7 -0
- package/procs/track/ttp40/index.html +1 -1
- package/procs/track/ttp40/index.js +0 -2
- package/score.js +6 -11
- package/script.js +2 -2
- package/summarize.js +11 -20
package/README.md
CHANGED
|
@@ -58,7 +58,7 @@ Testilo can, however, make job preparation more efficient in these scenarios:
|
|
|
58
58
|
|
|
59
59
|
### Target lists
|
|
60
60
|
|
|
61
|
-
The simplest version of a list of targets is a _target list_. It is an array of arrays defining 1 or more targets. It can be stored as a
|
|
61
|
+
The simplest version of a list of targets is a _target list_. It is an array of arrays defining 1 or more targets. It can be stored as a text file.
|
|
62
62
|
|
|
63
63
|
A target is defined by 2 items:
|
|
64
64
|
- A description
|
|
@@ -73,11 +73,11 @@ For example, a target list might be:
|
|
|
73
73
|
]
|
|
74
74
|
```
|
|
75
75
|
|
|
76
|
-
If this target list were stored as a file, its content would be this
|
|
76
|
+
If this target list were stored as a file, its content would be this, with vertical bars separating the URLs from the descriptions:
|
|
77
77
|
|
|
78
78
|
```text
|
|
79
|
-
World Wide Web Consortium
|
|
80
|
-
Mozilla Foundation
|
|
79
|
+
World Wide Web Consortium|https://www.w3.org/
|
|
80
|
+
Mozilla Foundation|https://foundation.mozilla.org/en/
|
|
81
81
|
```
|
|
82
82
|
|
|
83
83
|
### Batches
|
|
@@ -221,7 +221,7 @@ The invoking module can further dispose of the batch as needed.
|
|
|
221
221
|
|
|
222
222
|
A user can invoke `batch()` in this way:
|
|
223
223
|
|
|
224
|
-
- Create a target list and save it as a text file (with tab-delimited items in newline-delimited lines) in the `targetLists` subdirectory of the `SPECDIR` directory. Name the file `x.
|
|
224
|
+
- Create a target list and save it as a text file (with tab-delimited items in newline-delimited lines) in the `targetLists` subdirectory of the `SPECDIR` directory. Name the file `x.txt`, where `x` is the list ID.
|
|
225
225
|
- In the Testilo project directory, execute the statement `node call batch id what`.
|
|
226
226
|
|
|
227
227
|
In this statement, replace `id` with the list ID and `what` with a string describing the batch. Example: `node call batch divns 'ABC company divisions'`.
|
|
@@ -249,25 +249,25 @@ A module can invoke `script()` in this way:
|
|
|
249
249
|
```javaScript
|
|
250
250
|
const {script} = require('testilo/script');
|
|
251
251
|
const {issues} = require('testilo/procs/score/tic99');
|
|
252
|
-
const scriptObj = script('monthly', issues, 'regionNoText', 'mainNot1');
|
|
252
|
+
const scriptObj = script('monthly', 'landmarks', issues, 'regionNoText', 'mainNot1');
|
|
253
253
|
```
|
|
254
254
|
|
|
255
|
-
In this example, the script will have `'monthly'` as its ID. It will tell Testaro to test for all, and only, the rules that are classified into either the `regionNoText` or the `mainNot1` issue.
|
|
255
|
+
In this example, the script will have `'monthly'` as its ID and `'landmarks'` as its description. It will tell Testaro to test for all, and only, the rules that are classified into either the `regionNoText` or the `mainNot1` issue.
|
|
256
256
|
|
|
257
257
|
The invoking module can further modify and use the script (`scriptObj`) as needed.
|
|
258
258
|
|
|
259
|
-
To create a script **without** issue restrictions, a module can call `script()` with only the first (ID)
|
|
259
|
+
To create a script **without** issue restrictions, a module can call `script()` with only the first two arguments (the ID and the description).
|
|
260
260
|
|
|
261
261
|
##### By a user
|
|
262
262
|
|
|
263
263
|
A user can invoke `script()` by executing one of these statements in the Testilo project directory:
|
|
264
264
|
|
|
265
265
|
```javascript
|
|
266
|
-
node call script id ticnn issuea issueb …
|
|
267
|
-
node call script id
|
|
266
|
+
node call script id what ticnn issuea issueb …
|
|
267
|
+
node call script id what
|
|
268
268
|
```
|
|
269
269
|
|
|
270
|
-
In this statement, replace `id` with an ID for the script, such as `
|
|
270
|
+
In this statement, replace `id` with an ID for the script, such as `headings1`, consisting of ASCII alphanumeric characters.
|
|
271
271
|
|
|
272
272
|
If specifying issues:
|
|
273
273
|
- Replace `ticnn` with the base, such as `tic99`, of the name of an issue classification file in the `score` subdirectory of the `FUNCTIONDIR` directory.
|
|
@@ -275,7 +275,7 @@ If specifying issues:
|
|
|
275
275
|
|
|
276
276
|
The `call` module will retrieve the named classification, if any.
|
|
277
277
|
The `script` module will create a script.
|
|
278
|
-
The `call` module will save the script as a JSON file in the `scripts` subdirectory of the `SPECDIR` directory.
|
|
278
|
+
The `call` module will save the script as a JSON file in the `scripts` subdirectory of the `SPECDIR` directory, using the `id` value as the base of the file name.
|
|
279
279
|
|
|
280
280
|
#### Options
|
|
281
281
|
|
|
@@ -310,7 +310,9 @@ The first two arguments are a script and a batch obtained from files or from pri
|
|
|
310
310
|
|
|
311
311
|
The `standard` argument specifies how to handle standardization. If `also`, jobs will tell Testaro to include in its reports both the original results of the tests of tools and the Testaro-standardized results. If `only`, reports are to include only the standardized test results. If `no`, reports are to include only the original results, without standardization.
|
|
312
312
|
|
|
313
|
-
The `observe` argument tells Testaro whether the jobs should allow granular observation. If `false`, Testaro will not report job progress, but will
|
|
313
|
+
The `observe` argument tells Testaro whether the jobs should allow granular observation. If `true`, it will. If `false`, Testaro will not report job progress, but will send reports to the server only when the reports are completed. It is generally user-friendly to allow granular observation, and for user applications to implement it, if they make users wait while jobs are assigned and performed, since that process typically takes a few minutes.
|
|
314
|
+
|
|
315
|
+
The `requester` argument is an email address to which any notices about the job are to be sent.
|
|
314
316
|
|
|
315
317
|
The `timeStamp` argument specifies the earliest UTC date and time when the jobs may be assigned, or it may be an empty string if now.
|
|
316
318
|
|
|
@@ -428,11 +430,11 @@ Testilo can enhance such a report by:
|
|
|
428
430
|
|
|
429
431
|
### Scoring
|
|
430
432
|
|
|
431
|
-
|
|
433
|
+
The `score` module of Testilo performs computations on test results and adds a `score` property to a report.
|
|
432
434
|
|
|
433
435
|
The `score()` function of the `score` module takes two arguments:
|
|
434
436
|
- a scoring function
|
|
435
|
-
-
|
|
437
|
+
- a report object
|
|
436
438
|
|
|
437
439
|
A scoring function defines scoring rules. The Testilo package contains a `procs/score` directory, in which there are modules that export scoring functions. You can use one of those scoring functions, or you can create your own.
|
|
438
440
|
|
|
@@ -493,15 +495,15 @@ A module can invoke `score()` in this way:
|
|
|
493
495
|
```javaScript
|
|
494
496
|
const {score} = require('testilo/score');
|
|
495
497
|
const {scorer} = require('testilo/procs/score/tsp99');
|
|
496
|
-
const
|
|
497
|
-
score(scorer,
|
|
498
|
+
const report = …;
|
|
499
|
+
score(scorer, report);
|
|
498
500
|
```
|
|
499
501
|
|
|
500
502
|
The first argument to `score()` is a scoring function. In this example, it has been obtained from a module in the Testilo package, but it could be a custom function.
|
|
501
503
|
|
|
502
|
-
The second argument to `score()` is
|
|
504
|
+
The second argument to `score()` is a report object. It may have been read from a JSON file and parsed, or parsed from the body of a `POST` request received from a Testaro agent.
|
|
503
505
|
|
|
504
|
-
The invoking module can further dispose of the scored
|
|
506
|
+
The invoking module can further dispose of the scored report as needed.
|
|
505
507
|
|
|
506
508
|
##### By a user
|
|
507
509
|
|
|
@@ -509,15 +511,14 @@ A user can invoke `score()` in this way:
|
|
|
509
511
|
|
|
510
512
|
```bash
|
|
511
513
|
node call score tsp99
|
|
512
|
-
node call score tsp99
|
|
514
|
+
node call score tsp99 240922
|
|
513
515
|
```
|
|
514
516
|
|
|
515
517
|
When a user invokes `score()` in this example, the `call` module:
|
|
516
518
|
- gets the scoring module `tsp99` from its JSON file `tsp99.json` in the `score` subdirectory of the `FUNCTIONDIR` directory.
|
|
517
|
-
- gets the reports from the `raw` subdirectory of the `REPORTDIR` directory.
|
|
518
|
-
-
|
|
519
|
-
|
|
520
|
-
The optional third argument to `call()` (`75m` in this example) is a report selector. Without the argument, `call()` gets all the reports in the `raw` subdirectory. With the argument, `call()` gets only those reports whose names begin with the argument string.
|
|
519
|
+
- gets all reports, or if the third argument to `call()` exists the reports whose file names begin with `'240922'`, from the `raw` subdirectory of the `REPORTDIR` directory.
|
|
520
|
+
- adds score data to each report.
|
|
521
|
+
- writes each scored report in JSON format to the `scored` subdirectory of the `REPORTDIR` directory.
|
|
521
522
|
|
|
522
523
|
#### Validation
|
|
523
524
|
|
|
@@ -530,11 +531,11 @@ To test the `score` module, in the project directory you can execute the stateme
|
|
|
530
531
|
Reports from Testaro are JavaScript objects. When represented as JSON, they are human-readable, but not human-friendly. They are basically designed for machine tractability. This is equally true for reports that have been scored by Testilo. But Testilo can _digest_ a scored report, converting it to a human-oriented HTML document, or _digest_.
|
|
531
532
|
|
|
532
533
|
The `digest` module digests a scored report. Its `digest()` function takes three arguments:
|
|
533
|
-
- a digesting function
|
|
534
|
-
-
|
|
534
|
+
- a digester (a digesting function)
|
|
535
|
+
- a scored report object
|
|
535
536
|
- the URL of a directory containing the scored reports
|
|
536
537
|
|
|
537
|
-
The
|
|
538
|
+
The digester populates an HTML digest template. A copy of the template, with its placeholders replaced by computed values, becomes the digest. The digester defines the rules for replacing the placeholders with values. The Testilo package contains a `procs/digest` directory, in which there are subdirectories, each containing a template and a module that exports a digester. You can use one of those modules, or you can create your own.
|
|
538
539
|
|
|
539
540
|
The included templates format placeholders with leading and trailing underscore pairs (such as `__issueCount__`).
|
|
540
541
|
|
|
@@ -550,19 +551,19 @@ A module can invoke `digest()` in this way:
|
|
|
550
551
|
const {digest} = require('testilo/digest');
|
|
551
552
|
const digesterDir = `${process.env.FUNCTIONDIR}/digest/tdp99a`;
|
|
552
553
|
const {digester} = require(`${digesterDir}/index`);
|
|
553
|
-
const
|
|
554
|
+
const scoredReport = …;
|
|
554
555
|
const reportDirURL = 'https://xyz.org/a11yTesting/reports';
|
|
555
|
-
digest(digester,
|
|
556
|
-
.then(
|
|
556
|
+
digest(digester, scoredReport, reportDirURL)
|
|
557
|
+
.then(digestedReport => {…});
|
|
557
558
|
```
|
|
558
559
|
|
|
559
|
-
The first argument to `digest()` is a
|
|
560
|
+
The first argument to `digest()` is a digester. In this example, it has been obtained from a file in the Testilo package, but it could be custom-made.
|
|
560
561
|
|
|
561
|
-
The second argument to `digest()` is
|
|
562
|
+
The second argument to `digest()` is a scored report object. It may have been read from a JSON file and parsed, or may be a scored report output by `score()`.
|
|
562
563
|
|
|
563
|
-
The third argument is the absolute or relative URL of a directory where the reports being digested are located. The `digest()` function needs that URL because a digest includes a link to the full report. The link concatenates the directory URL with the report ID and a `.json` suffix.
|
|
564
|
+
The third argument is the absolute or relative URL of a directory where the reports being digested are located. The `digest()` function needs that URL because a digest includes a link to the full scored report. The link concatenates the directory URL with the report ID and a `.json` suffix.
|
|
564
565
|
|
|
565
|
-
The `digest()` function returns
|
|
566
|
+
The `digest()` function returns a promise resolved with a digest. The invoking module can further dispose of the digest as needed.
|
|
566
567
|
|
|
567
568
|
##### By a user
|
|
568
569
|
|
|
@@ -570,18 +571,15 @@ A user can invoke `digest()` in this way:
|
|
|
570
571
|
|
|
571
572
|
```bash
|
|
572
573
|
node call digest tdp99
|
|
573
|
-
node call digest tdp99
|
|
574
|
+
node call digest tdp99 241105
|
|
574
575
|
```
|
|
575
576
|
|
|
576
577
|
When a user invokes `digest()` in this example, the `call` module:
|
|
577
578
|
- gets the template and the digesting module from subdirectory `tdp99` in the `digest` subdirectory of the `FUNCTIONDIR` directory.
|
|
578
|
-
- gets the reports from the `scored` subdirectory of the `REPORTDIR` directory.
|
|
579
|
+
- gets all reports, or if the third argument to `call()` exists all reports whose file names begin with `'241105'`, from the `scored` subdirectory of the `REPORTDIR` directory.
|
|
580
|
+
- digests each report.
|
|
579
581
|
- writes the digested reports to the `digested` subdirectory of the `REPORTDIR` directory.
|
|
580
|
-
- includes in each digest a link to the scored report, with the link destination being based on `
|
|
581
|
-
|
|
582
|
-
The third argument to `call()` can be an absolute URL, as shown, or a URL that is relative to the URL of the digest. For example, if it is known that the scored reports and the digests will inhabit the same directory and be retrievable with identical URLs except for the file extensions, then the third argument can be `./`.
|
|
583
|
-
|
|
584
|
-
The optional fourth argument to `call()` (`75m` in this example) is a report selector. Without the argument, `call` gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory. With the argument, `call` gets only those reports whose names begin with the argument string.
|
|
582
|
+
- includes in each digest a link to the scored report, with the link destination being based on `SCORED_REPORT_URL`.
|
|
585
583
|
|
|
586
584
|
The digests created by `digest()` are HTML files, and they expect a `style.css` file to exist in their directory. The `reports/digested/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where digested reports are written.
|
|
587
585
|
|
|
@@ -646,13 +644,48 @@ Difgests expect a `style.css` file to exist in their directory, as digests do.
|
|
|
646
644
|
|
|
647
645
|
To test the `digest` module, in the project directory you can execute the statement `node validation/digest/validate`. If `digest` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
|
|
648
646
|
|
|
649
|
-
###
|
|
647
|
+
### Summarization
|
|
648
|
+
|
|
649
|
+
The `summarize` module of Testilo can summarize a scored report. The summary contains, insofar as they exist in the report, its ID, end time, order ID, target data, and total score.
|
|
650
|
+
|
|
651
|
+
#### Invocation
|
|
652
|
+
|
|
653
|
+
##### By a module
|
|
654
|
+
|
|
655
|
+
A module can invoke `summarize()` in this way:
|
|
656
|
+
|
|
657
|
+
```javaScript
|
|
658
|
+
const {summarize} = require('testilo/summarize');
|
|
659
|
+
const report = …;
|
|
660
|
+
const summary = summarize(report);
|
|
661
|
+
…
|
|
662
|
+
```
|
|
663
|
+
|
|
664
|
+
The `report` argument is a scored report. The `summary` constant is an object. The module can further dispose of `summary` as needed.
|
|
665
|
+
|
|
666
|
+
##### By a user
|
|
667
|
+
|
|
668
|
+
A user can invoke `summarize()` in either of these two ways:
|
|
669
|
+
|
|
670
|
+
```javaScript
|
|
671
|
+
node call summarize divisions
|
|
672
|
+
node call summarize divisions 2411
|
|
673
|
+
```
|
|
674
|
+
|
|
675
|
+
When a user invokes `summarize` in this example, the `call` module:
|
|
676
|
+
- gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory, or (if the third argument is present) all those whose file names begin with `2411`.
|
|
677
|
+
- creates a summary of each report.
|
|
678
|
+
- combines the summaries into an array.
|
|
679
|
+
- creates a _summary report_, an object containing three properties: an ID, a description (here `'divisions'`), and the array of summaries.
|
|
680
|
+
- writes the summary report in JSON format to the `summarized` subdirectory of the `REPORTDIR` directory, using the ID as the base of the file name.
|
|
681
|
+
|
|
682
|
+
### Comparison
|
|
650
683
|
|
|
651
|
-
If you use Testilo to perform a battery of tests on multiple targets, you may want a single report that compares the total scores received by the targets. Testilo can produce such a
|
|
684
|
+
If you use Testilo to perform a battery of tests on multiple targets, you may want a single report that compares the total scores received by the targets. Testilo can produce such a _comparison_.
|
|
652
685
|
|
|
653
|
-
The `compare` module compares the scores in a
|
|
686
|
+
The `compare` module compares the scores in a summary of scored reports. Its `compare()` function takes two arguments:
|
|
654
687
|
- a comparison function
|
|
655
|
-
-
|
|
688
|
+
- a summary report
|
|
656
689
|
|
|
657
690
|
The comparison function defines the rules for generating an HTML file comparing the scored reports. The Testilo package contains a `procs/compare` directory, in which there are subdirectories containing modules that export comparison functions. You can use one of those functions, or you can create your own.
|
|
658
691
|
|
|
@@ -668,25 +701,27 @@ A module can invoke `compare()` in this way:
|
|
|
668
701
|
const {compare} = require('testilo/compare');
|
|
669
702
|
const comparerDir = `${process.env.FUNCTIONDIR}/compare/tcp99`;
|
|
670
703
|
const {comparer} = require(`${comparerDir}/index`);
|
|
671
|
-
|
|
704
|
+
const id = …;
|
|
705
|
+
compare(id, comparer, summaryReport)
|
|
672
706
|
.then(comparison => {…});
|
|
673
707
|
```
|
|
674
708
|
|
|
675
|
-
The first argument to `compare()` is a comparison function. In this example, it been obtained from a file in the Testilo package, but it could be custom-made. The
|
|
709
|
+
The first argument to `compare()` is an ID that will be named in the comparison. The second argument is a comparison function. In this example, it been obtained from a file in the Testilo package, but it could be custom-made. The third argument is a summary report. The `compare()` function returns a comparison. The invoking module can further dispose of the comparison as needed.
|
|
676
710
|
|
|
677
711
|
##### By a user
|
|
678
712
|
|
|
679
713
|
A user can invoke `compare()` in this way:
|
|
680
714
|
|
|
681
715
|
```bash
|
|
682
|
-
node call compare tcp99
|
|
683
|
-
node call compare tcp99 legislators 23pl
|
|
716
|
+
node call compare 'state legislators' tcp99 240813
|
|
684
717
|
```
|
|
685
718
|
|
|
686
719
|
When a user invokes `compare` in this example, the `call` module:
|
|
687
720
|
- gets the comparison module from subdirectory `tcp99` of the subdirectory `compare` in the `FUNCTIONDIR` directory.
|
|
688
|
-
- gets
|
|
689
|
-
-
|
|
721
|
+
- gets the summary report whose file name begins with `'240813'` from the `summarized` subdirectory of the `REPORTDIR` directory.
|
|
722
|
+
- creates an ID for the comparison.
|
|
723
|
+
- creates the comparison as an HTML document.
|
|
724
|
+
- writes the comparison in the `comparative` subdirectory of the `REPORTDIR` directory, with `state legislators` as a description and the ID as the base of the file name.
|
|
690
725
|
|
|
691
726
|
The comparative report created by `compare` is an HTML file, and it expects a `style.css` file to exist in its directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
|
|
692
727
|
|
|
@@ -698,7 +733,7 @@ To test the `compare` module, in the project directory you can execute the state
|
|
|
698
733
|
|
|
699
734
|
If you use Testaro to perform all the tests of all the tools on multiple targets and score the reports with a score proc that maps tool rules onto tool-agnostic issues, you may want to tabulate the comparative efficacy of each tool in discovering instances of issues. Testilo can help you do this by producing a _credit report_.
|
|
700
735
|
|
|
701
|
-
The `credit` module tabulates the contribution of each tool to the discovery of issue instances in a collection of scored reports. Its `credit()` function takes
|
|
736
|
+
The `credit` module tabulates the contribution of each tool to the discovery of issue instances in a collection of scored reports. Its `credit()` function takes two arguments: a report description and an array of `score` properties of scored reports.
|
|
702
737
|
|
|
703
738
|
The credit report contains four sections:
|
|
704
739
|
- `counts`: for each issue, how many instances each tool reported
|
|
@@ -718,57 +753,27 @@ A module can invoke `credit()` in this way:
|
|
|
718
753
|
|
|
719
754
|
```javaScript
|
|
720
755
|
const {credit} = require('testilo/credit');
|
|
721
|
-
|
|
756
|
+
const reportScores = […];
|
|
757
|
+
credit('June 2025', reportScores)
|
|
722
758
|
.then(creditReport => {…});
|
|
723
759
|
```
|
|
724
760
|
|
|
725
|
-
The argument to `credit()` is an array of scored report objects. The `credit()` function returns a credit report. The invoking module can further dispose of the credit report as needed.
|
|
761
|
+
The first argument to `credit()` is a description to be included in the credit report. The second argument is an array of `score` properties of scored report objects. The `credit()` function returns a credit report. The invoking module can further dispose of the credit report as needed.
|
|
726
762
|
|
|
727
763
|
##### By a user
|
|
728
764
|
|
|
729
|
-
A user can invoke `credit()` in
|
|
765
|
+
A user can invoke `credit()` in one of these ways:
|
|
730
766
|
|
|
731
767
|
```bash
|
|
732
|
-
node call credit legislators
|
|
768
|
+
node call credit legislators
|
|
769
|
+
node call credit legislators 241106
|
|
733
770
|
```
|
|
734
771
|
|
|
735
772
|
When a user invokes `credit` in this example, the `call` module:
|
|
736
|
-
- gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory
|
|
737
|
-
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
### Summarization
|
|
742
|
-
|
|
743
|
-
The `summarize` module of Testilo can summarize a collection of scored reports. The summary of each report contains, insofar as they exist in the report, its ID, end time, order ID, target data, and total score.
|
|
744
|
-
|
|
745
|
-
#### Invocation
|
|
746
|
-
|
|
747
|
-
##### By a module
|
|
748
|
-
|
|
749
|
-
A module can invoke `summarize()` in this way:
|
|
750
|
-
|
|
751
|
-
```javaScript
|
|
752
|
-
const {summarize} = require('testilo/summarize');
|
|
753
|
-
const reports = […];
|
|
754
|
-
const summary = summarize(reports);
|
|
755
|
-
…
|
|
756
|
-
```
|
|
757
|
-
|
|
758
|
-
The `reports` argument is an array of scored reports. The `summary` constant is an object. The module can further dispose of `summary` as needed.
|
|
759
|
-
|
|
760
|
-
##### By a user
|
|
761
|
-
|
|
762
|
-
A user can invoke `summarize()` in either of these two ways:
|
|
763
|
-
|
|
764
|
-
```javaScript
|
|
765
|
-
node call summarize divisions
|
|
766
|
-
node call summarize divisions 2411
|
|
767
|
-
```
|
|
768
|
-
|
|
769
|
-
When a user invokes `summarize` in this example, the `call` module:
|
|
770
|
-
- gets all the reports in the `scored` subdirectory of the `REPORTDIR` directory, or (if the third argument is present) all those whose file names begin with `2411`.
|
|
771
|
-
- writes the summary, with `divisions` as its description, in JSON format to the `summarized` subdirectory of the `REPORTDIR` directory.
|
|
773
|
+
- gets all reports, or if the third argument to `call()` exists all reports whose file names begin with `'241106'`, in the `scored` subdirectory of the `REPORTDIR` directory.
|
|
774
|
+
- gets the `score` properties of those reports.
|
|
775
|
+
- creates an ID for the credit report.
|
|
776
|
+
- writes the credit report as a JSON file, with the ID as the base of its file name and `legislators` as its description, to the `credit` subdirectory of the `REPORTDIR` directory.
|
|
772
777
|
|
|
773
778
|
### Track
|
|
774
779
|
|
|
@@ -800,7 +805,7 @@ node call track ttp99a 241016T2045-Uf-0 4 'ABC Foundation'
|
|
|
800
805
|
|
|
801
806
|
When a user invokes `track()` in this example, the `call` module:
|
|
802
807
|
- gets the summary from the `241016T2045-Uf-0.json` file in the `summarized` subdirectory of the `REPORTDIR` directory.
|
|
803
|
-
- selects the summarized data for all
|
|
808
|
+
- selects the summarized data for all results with the `order` value of `'4'` and the `target.what` value of `'ABC Foundation'`. If the third or fourth argument to `call()` is `null` (or omitted), then `call()` does not select results by `order` or by `target.what`, respectively.
|
|
804
809
|
- uses tracker `ttp99a` to create a tracking report.
|
|
805
810
|
- writes the tracking report to the `tracking` subdirectory of the `REPORTDIR` directory.
|
|
806
811
|
|
package/call.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// Module to keep secrets.
|
|
18
18
|
require('dotenv').config();
|
|
19
19
|
// Module to perform common operations.
|
|
20
|
-
const {getNowStamp, getRandomString} = require('./procs/util');
|
|
20
|
+
const {getFileID, getNowStamp, getRandomString} = require('./procs/util');
|
|
21
21
|
// Function to process files.
|
|
22
22
|
const fs = require('fs/promises');
|
|
23
23
|
// Function to process a list-to-batch conversion.
|
|
@@ -34,6 +34,8 @@ const {digest} = require('./digest');
|
|
|
34
34
|
const {difgest} = require('./difgest');
|
|
35
35
|
// Function to compare scores.
|
|
36
36
|
const {compare} = require('./compare');
|
|
37
|
+
// Function to credit tools for issues in reports.
|
|
38
|
+
const {credit} = require('./credit');
|
|
37
39
|
// Function to summarize reports.
|
|
38
40
|
const {summarize} = require('./summarize');
|
|
39
41
|
// Function to track audits.
|
|
@@ -50,35 +52,66 @@ const fnArgs = process.argv.slice(3);
|
|
|
50
52
|
|
|
51
53
|
// ########## FUNCTIONS
|
|
52
54
|
|
|
55
|
+
// Gets a summary report.
|
|
56
|
+
const getSummaryReport = async selector => {
|
|
57
|
+
const summaryDir = `${reportDir}/summarized`;
|
|
58
|
+
const summaryReportNames = await fs.readdir(summaryDir);
|
|
59
|
+
const summaryReportName = summaryReportNames.find(reportName => reportName.startsWith(selector));
|
|
60
|
+
if (summaryReportName) {
|
|
61
|
+
const summaryReportJSON = await fs.readFile(`${summaryDir}/${summaryReportName}`, 'utf8');
|
|
62
|
+
const summaryReport = JSON.parse(summaryReportJSON);
|
|
63
|
+
return summaryReport;
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
throw new Error(`ERROR: No summary report name starts with ${selector}`);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
53
69
|
// Converts a target list to a batch.
|
|
54
70
|
const callBatch = async (id, what) => {
|
|
55
71
|
// Get the target list.
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
72
|
+
try {
|
|
73
|
+
const listString = await fs.readFile(`${specDir}/targetLists/${id}.txt`, 'utf8');
|
|
74
|
+
const list = listString
|
|
75
|
+
.split('\n')
|
|
76
|
+
.filter(target => target.length)
|
|
77
|
+
.map(target => target.split('|'));
|
|
78
|
+
// Convert it to a batch.
|
|
79
|
+
const batchObj = batch(id, what, list);
|
|
80
|
+
// Save the batch.
|
|
81
|
+
if (batchObj) {
|
|
82
|
+
const batchJSON = JSON.stringify(batchObj, null, 2);
|
|
83
|
+
const batchPath = `${specDir}/batches/${id}.json`;
|
|
84
|
+
await fs.writeFile(batchPath, `${batchJSON}\n`);
|
|
85
|
+
console.log(`Target list ${id} converted to a batch and saved as ${batchPath}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
catch(error) {
|
|
89
|
+
console.log(`ERROR converting target list to batch (${error.message})`);
|
|
68
90
|
}
|
|
69
91
|
};
|
|
70
92
|
// Fulfills a script-creation request.
|
|
71
|
-
const callScript = async (scriptID, classificationID = null, ... issueIDs) => {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
93
|
+
const callScript = async (scriptID, what, classificationID = null, ... issueIDs) => {
|
|
94
|
+
try {
|
|
95
|
+
// Get any issue classification.
|
|
96
|
+
const issues = classificationID
|
|
97
|
+
? require(`${functionDir}/score/${classificationID}`).issues
|
|
98
|
+
: null;
|
|
99
|
+
// Sanitize the ID.
|
|
100
|
+
scriptID = scriptID.replace(/[^a-zA-Z0-9]/g, '');
|
|
101
|
+
if (scriptID === '') {
|
|
102
|
+
scriptID = `script-${getRandomString(2)}`;
|
|
103
|
+
}
|
|
104
|
+
// Create a script.
|
|
105
|
+
const scriptObj = script(scriptID, what, issues, ... issueIDs);
|
|
106
|
+
// Save the script.
|
|
107
|
+
const scriptJSON = JSON.stringify(scriptObj, null, 2);
|
|
108
|
+
const scriptPath = `${specDir}/scripts/${scriptID}.json`;
|
|
109
|
+
await fs.writeFile(scriptPath, `${scriptJSON}\n`);
|
|
110
|
+
console.log(`Script created and saved as ${scriptPath}`);
|
|
111
|
+
}
|
|
112
|
+
catch(error) {
|
|
113
|
+
console.log(`ERROR creating script (${error.message})`);
|
|
114
|
+
}
|
|
82
115
|
};
|
|
83
116
|
// Fulfills a merging request.
|
|
84
117
|
const callMerge = async (
|
|
@@ -90,58 +123,67 @@ const callMerge = async (
|
|
|
90
123
|
timeStamp,
|
|
91
124
|
todoDir
|
|
92
125
|
) => {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const
|
|
104
|
-
|
|
126
|
+
try {
|
|
127
|
+
// If the todoDir argument is invalid:
|
|
128
|
+
if (! ['true', 'false'].includes(todoDir)) {
|
|
129
|
+
// Report this.
|
|
130
|
+
throw new Error('Invalid todoDir configuration');
|
|
131
|
+
}
|
|
132
|
+
// Get the script and the batch.
|
|
133
|
+
const scriptJSON = await fs.readFile(`${specDir}/scripts/${scriptID}.json`, 'utf8');
|
|
134
|
+
const script = JSON.parse(scriptJSON);
|
|
135
|
+
const batchJSON = await fs.readFile(`${specDir}/batches/${batchID}.json`, 'utf8');
|
|
136
|
+
const batch = JSON.parse(batchJSON);
|
|
137
|
+
// Merge them into an array of jobs.
|
|
138
|
+
const jobs = merge(script, batch, standard, observe === 'true', requester, timeStamp);
|
|
139
|
+
// Save the jobs.
|
|
140
|
+
const subdir = `${jobDir}/${todoDir === 'true' ? 'todo' : 'pending'}`;
|
|
141
|
+
for (const job of jobs) {
|
|
142
|
+
const jobJSON = JSON.stringify(job, null, 2);
|
|
143
|
+
await fs.writeFile(`${subdir}/${job.id}.json`, `${jobJSON}\n`);
|
|
144
|
+
}
|
|
145
|
+
const truncatedID = `${jobs[0].timeStamp}-${jobs[0].mergeID}-…`;
|
|
146
|
+
console.log(`Script ${scriptID} and batch ${batchID} merged as ${truncatedID} in ${subdir}`);
|
|
147
|
+
}
|
|
148
|
+
catch(error) {
|
|
149
|
+
console.log(`ERROR merging script and batch (${error.message})`);
|
|
105
150
|
}
|
|
106
|
-
const truncatedID = `${jobs[0].timeStamp}-${jobs[0].mergeID}-…`;
|
|
107
|
-
console.log(`Script ${scriptID} and batch ${batchID} merged as ${truncatedID} in ${subdir}`);
|
|
108
151
|
};
|
|
109
|
-
// Gets selected reports.
|
|
110
|
-
const
|
|
152
|
+
// Gets the file base names (equal to the IDs) of the selected reports.
|
|
153
|
+
const getReportIDs = async (type, selector = '') => {
|
|
111
154
|
const allFileNames = await fs.readdir(`${reportDir}/${type}`);
|
|
112
155
|
const reportIDs = allFileNames
|
|
113
156
|
.filter(fileName => fileName.endsWith('.json'))
|
|
114
157
|
.filter(fileName => fileName.startsWith(selector))
|
|
115
158
|
.map(fileName => fileName.slice(0, -5));
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
return
|
|
159
|
+
return reportIDs;
|
|
160
|
+
};
|
|
161
|
+
// Gets and returns a report.
|
|
162
|
+
const getReport = async (type, id) => {
|
|
163
|
+
const reportJSON = await fs.readFile(`${reportDir}/${type}/${id}.json`, 'utf8');
|
|
164
|
+
const report = JSON.parse(reportJSON);
|
|
165
|
+
return report;
|
|
123
166
|
};
|
|
124
167
|
// Fulfills a scoring request.
|
|
125
168
|
const callScore = async (scorerID, selector = '') => {
|
|
126
169
|
// Get the raw reports to be scored.
|
|
127
|
-
const
|
|
170
|
+
const reportIDs = await getReportIDs('raw', selector);
|
|
128
171
|
// If any exist:
|
|
129
|
-
if (
|
|
172
|
+
if (reportIDs.length) {
|
|
130
173
|
// Get the scorer.
|
|
131
174
|
const {scorer} = require(`${functionDir}/score/${scorerID}`);
|
|
132
|
-
// Score the reports.
|
|
133
|
-
score(scorer, reports);
|
|
175
|
+
// Score and save the reports.
|
|
134
176
|
const scoredReportDir = `${reportDir}/scored`;
|
|
135
177
|
await fs.mkdir(scoredReportDir, {recursive: true});
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
178
|
+
for (const reportID of reportIDs) {
|
|
179
|
+
const report = await getReport('raw', reportID);
|
|
180
|
+
score(scorer, report);
|
|
139
181
|
await fs.writeFile(
|
|
140
|
-
`${scoredReportDir}/${
|
|
182
|
+
`${scoredReportDir}/${reportID}.json`, `${JSON.stringify(report, null, 2)}\n`
|
|
141
183
|
);
|
|
142
|
-
};
|
|
143
|
-
console.log(`Reports scored and saved in ${scoredReportDir}`);
|
|
144
184
|
}
|
|
185
|
+
console.log(`Reports scored and saved in ${scoredReportDir}`);
|
|
186
|
+
}
|
|
145
187
|
// Otherwise, i.e. if no raw reports are to be scored:
|
|
146
188
|
else {
|
|
147
189
|
// Report this.
|
|
@@ -150,22 +192,26 @@ const callScore = async (scorerID, selector = '') => {
|
|
|
150
192
|
};
|
|
151
193
|
// Fulfills a digesting request.
|
|
152
194
|
const callDigest = async (digesterID, selector = '') => {
|
|
153
|
-
// Get the scored reports to be digested.
|
|
154
|
-
const
|
|
195
|
+
// Get the base base names (equal to the IDs) of the scored reports to be digested.
|
|
196
|
+
const reportIDs = await getReportIDs('scored', selector);
|
|
155
197
|
// If any exist:
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
198
|
+
if (reportIDs.length) {
|
|
199
|
+
try {
|
|
200
|
+
// Get the digester.
|
|
201
|
+
const {digester} = require(`${functionDir}/digest/${digesterID}/index`);
|
|
202
|
+
// Digest and save the reports.
|
|
203
|
+
const digestDir = `${reportDir}/digested`;
|
|
204
|
+
await fs.mkdir(digestDir, {recursive: true});
|
|
205
|
+
for (const reportID of reportIDs) {
|
|
206
|
+
const report = await getReport('scored', reportID);
|
|
207
|
+
const digestedReport = await digest(digester, report);
|
|
208
|
+
await fs.writeFile(`${digestDir}/${reportID}.html`, digestedReport);
|
|
209
|
+
};
|
|
210
|
+
console.log(`Reports digested and saved in ${digestDir}`);
|
|
211
|
+
}
|
|
212
|
+
catch(error) {
|
|
213
|
+
console.log(`ERROR digesting reports (${error.message})`);
|
|
214
|
+
}
|
|
169
215
|
}
|
|
170
216
|
// Otherwise, i.e. if no scored reports are to be digested:
|
|
171
217
|
else {
|
|
@@ -176,10 +222,8 @@ const callDigest = async (digesterID, selector = '') => {
|
|
|
176
222
|
// Fulfills a difgesting request.
|
|
177
223
|
const callDifgest = async (difgesterID, reportAID, reportBID) => {
|
|
178
224
|
// Get the scored reports to be difgested.
|
|
179
|
-
const
|
|
180
|
-
const
|
|
181
|
-
const reportA = reportAArray[0];
|
|
182
|
-
const reportB = reportBArray[0];
|
|
225
|
+
const reportA = await getReport('scored', reportAID);
|
|
226
|
+
const reportB = await getReport('scored', reportBID);
|
|
183
227
|
// If both exist:
|
|
184
228
|
if (reportAID && reportBID) {
|
|
185
229
|
// Get the difgester.
|
|
@@ -201,69 +245,90 @@ const callDifgest = async (difgesterID, reportAID, reportBID) => {
|
|
|
201
245
|
console.log('ERROR: No pair of scored reports to be digested');
|
|
202
246
|
}
|
|
203
247
|
};
|
|
204
|
-
// Fulfills a
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
const
|
|
208
|
-
// If any exist:
|
|
209
|
-
if (reports.length) {
|
|
210
|
-
// Get the comparer.
|
|
211
|
-
const comparerDir = `${functionDir}/compare/${compareProcID}`;
|
|
212
|
-
const {comparer} = require(`${comparerDir}/index`);
|
|
213
|
-
// Compare the reports.
|
|
214
|
-
const comparison = await compare(comparer, reports);
|
|
215
|
-
// Save the comparison.
|
|
216
|
-
const comparisonDir = `${reportDir}/comparative`;
|
|
217
|
-
await fs.mkdir(comparisonDir, {recursive: true});
|
|
218
|
-
await fs.writeFile(`${comparisonDir}/${comparisonNameBase}.html`, comparison);
|
|
219
|
-
console.log(`Comparison completed and saved in ${comparisonDir}`);
|
|
220
|
-
}
|
|
221
|
-
};
|
|
222
|
-
// Fulfills a credit request.
|
|
223
|
-
const callCredit = async (tallyID, selector = '') => {
|
|
224
|
-
// Get the scored reports to be tallied.
|
|
225
|
-
const reports = await getReports('scored', selector);
|
|
248
|
+
// Fulfills a summarization request.
|
|
249
|
+
const callSummarize = async (what, selector = '') => {
|
|
250
|
+
// Get the IDs of the scored reports to be summarized.
|
|
251
|
+
const reportIDs = await getReportIDs('scored', selector);
|
|
226
252
|
// If any exist:
|
|
227
|
-
if (
|
|
228
|
-
//
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
253
|
+
if (reportIDs.length) {
|
|
254
|
+
// Initialize a summary report.
|
|
255
|
+
const summaryReport = {
|
|
256
|
+
id: getFileID(2),
|
|
257
|
+
what,
|
|
258
|
+
data: []
|
|
259
|
+
};
|
|
260
|
+
// For each report to be summarized:
|
|
261
|
+
for (const reportID of reportIDs) {
|
|
262
|
+
// Get it.
|
|
263
|
+
const report = await getReport('scored', reportID);
|
|
264
|
+
// Add a summary of it to the summary report.
|
|
265
|
+
const summary = summarize(report);
|
|
266
|
+
summaryReport.data.push(summary);
|
|
267
|
+
};
|
|
268
|
+
// Save the summary report.
|
|
269
|
+
const summaryDir = `${reportDir}/summarized`;
|
|
270
|
+
await fs.mkdir(summaryDir, {recursive: true});
|
|
271
|
+
const filePath = `${summaryDir}/${summaryReport.id}.json`;
|
|
272
|
+
await fs.writeFile(filePath, `${JSON.stringify(summaryReport, null, 2)}\n`);
|
|
273
|
+
console.log(`Reports summarized and summary report saved as ${filePath}`);
|
|
237
274
|
}
|
|
238
|
-
// Otherwise, i.e. if no scored reports are to be
|
|
275
|
+
// Otherwise, i.e. if no scored reports are to be summarized:
|
|
239
276
|
else {
|
|
240
277
|
// Report this.
|
|
241
|
-
console.log('ERROR: No scored reports to be
|
|
278
|
+
console.log('ERROR: No scored reports to be summarized');
|
|
242
279
|
}
|
|
243
280
|
};
|
|
244
|
-
// Fulfills a
|
|
245
|
-
const
|
|
246
|
-
// Get the
|
|
247
|
-
const
|
|
281
|
+
// Fulfills a comparison request.
|
|
282
|
+
const callCompare = async (what, compareProcID, selector) => {
|
|
283
|
+
// Get the specified summary report.
|
|
284
|
+
const summaryReport = await getSummaryReport(selector);
|
|
285
|
+
// If it exists:
|
|
286
|
+
if (summaryReport) {
|
|
287
|
+
try {
|
|
288
|
+
// Get the comparer.
|
|
289
|
+
const comparerDir = `${functionDir}/compare/${compareProcID}`;
|
|
290
|
+
const {comparer} = require(`${comparerDir}/index`);
|
|
291
|
+
// Compare the reports and save the comparison.
|
|
292
|
+
const comparisonDir = `${reportDir}/comparative`;
|
|
293
|
+
await fs.mkdir(comparisonDir, {recursive: true});
|
|
294
|
+
const id = getFileID(2);
|
|
295
|
+
const comparison = await compare(id, what, comparer, summaryReport);
|
|
296
|
+
const comparisonPath = `${comparisonDir}/${id}.html`;
|
|
297
|
+
await fs.writeFile(comparisonPath, comparison);
|
|
298
|
+
console.log(`Comparison completed and saved as ${comparisonPath}`);
|
|
299
|
+
}
|
|
300
|
+
catch(error) {
|
|
301
|
+
console.log(`ERROR comparing scores (${error.message})`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
// Fulfills a credit request.
|
|
306
|
+
const callCredit = async (what, selector = '') => {
|
|
307
|
+
// Get the IDs of the scored reports to be credited.
|
|
308
|
+
const reportIDs = await getReportIDs('scored', selector);
|
|
248
309
|
// If any exist:
|
|
249
|
-
if (
|
|
250
|
-
//
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
310
|
+
if (reportIDs.length) {
|
|
311
|
+
// Get an array of the score properties of the reports to be credited.
|
|
312
|
+
const reportScores = [];
|
|
313
|
+
for (const id of reportIDs) {
|
|
314
|
+
const report = await getReport('scored', id);
|
|
315
|
+
reportScores.push(report.score);
|
|
255
316
|
}
|
|
256
|
-
//
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
const
|
|
260
|
-
await fs.
|
|
261
|
-
|
|
317
|
+
// Credit the reports.
|
|
318
|
+
const tally = credit(what, reportScores);
|
|
319
|
+
// Save the credit report.
|
|
320
|
+
const creditDir = `${reportDir}/credit`;
|
|
321
|
+
await fs.mkdir(creditDir, {recursive: true});
|
|
322
|
+
const creditReportID = getFileID(2);
|
|
323
|
+
tally.id = creditReportID;
|
|
324
|
+
const reportPath = `${creditDir}/${creditReportID}.json`;
|
|
325
|
+
await fs.writeFile(reportPath, `${JSON.stringify(tally, null, 2)}\n`);
|
|
326
|
+
console.log(`Reports credited and credit report saved as ${reportPath}`);
|
|
262
327
|
}
|
|
263
|
-
// Otherwise, i.e. if no scored reports are to be
|
|
328
|
+
// Otherwise, i.e. if no scored reports are to be credited:
|
|
264
329
|
else {
|
|
265
330
|
// Report this.
|
|
266
|
-
console.log('ERROR: No scored reports to be
|
|
331
|
+
console.log('ERROR: No scored reports to be credited');
|
|
267
332
|
}
|
|
268
333
|
};
|
|
269
334
|
// Fulfills a tracking request.
|
|
@@ -273,20 +338,20 @@ const callTrack = async (trackerID, summaryID, orderID, targetWhat) => {
|
|
|
273
338
|
const summaryJSON = await fs.readFile(`${reportDir}/summarized/${summaryID}.json`, 'utf8');
|
|
274
339
|
const summary = JSON.parse(summaryJSON);
|
|
275
340
|
// Remove unwanted audits from it.
|
|
276
|
-
summary.data = summary.data.filter(
|
|
277
|
-
if (orderID &&
|
|
341
|
+
summary.data = summary.data.filter(result => {
|
|
342
|
+
if (orderID && result.order !== orderID) {
|
|
278
343
|
return false;
|
|
279
344
|
}
|
|
280
|
-
if (targetWhat &&
|
|
345
|
+
if (targetWhat && result.target && result.target.what !== targetWhat) {
|
|
281
346
|
return false;
|
|
282
347
|
}
|
|
283
348
|
return true;
|
|
284
349
|
});
|
|
285
|
-
// If any
|
|
350
|
+
// If any results remain:
|
|
286
351
|
if (summary.data.length) {
|
|
287
352
|
// Get the tracker.
|
|
288
353
|
const {tracker} = require(`${functionDir}/track/${trackerID}/index`);
|
|
289
|
-
// Track the
|
|
354
|
+
// Track the results.
|
|
290
355
|
const [reportID, trackingReport] = await track(tracker, summary);
|
|
291
356
|
// Save the tracking report.
|
|
292
357
|
await fs.mkdir(`${reportDir}/tracking`, {recursive: true});
|
|
@@ -313,7 +378,7 @@ if (fn === 'batch' && fnArgs.length === 2) {
|
|
|
313
378
|
console.log('Execution completed');
|
|
314
379
|
});
|
|
315
380
|
}
|
|
316
|
-
else if (fn === 'script' && (fnArgs.length ===
|
|
381
|
+
else if (fn === 'script' && (fnArgs.length === 2 || fnArgs.length > 3)) {
|
|
317
382
|
callScript(... fnArgs)
|
|
318
383
|
.then(() => {
|
|
319
384
|
console.log('Execution completed');
|
|
@@ -349,20 +414,20 @@ else if (fn === 'difgest' && fnArgs.length === 3) {
|
|
|
349
414
|
console.log('Execution completed');
|
|
350
415
|
});
|
|
351
416
|
}
|
|
352
|
-
else if (fn === '
|
|
353
|
-
|
|
417
|
+
else if (fn === 'summarize' && fnArgs.length > 0 && fnArgs.length < 3) {
|
|
418
|
+
callSummarize(... fnArgs)
|
|
354
419
|
.then(() => {
|
|
355
420
|
console.log('Execution completed');
|
|
356
421
|
});
|
|
357
422
|
}
|
|
358
|
-
else if (fn === '
|
|
359
|
-
|
|
423
|
+
else if (fn === 'compare' && fnArgs.length === 3) {
|
|
424
|
+
callCompare(... fnArgs)
|
|
360
425
|
.then(() => {
|
|
361
426
|
console.log('Execution completed');
|
|
362
427
|
});
|
|
363
428
|
}
|
|
364
|
-
else if (fn === '
|
|
365
|
-
|
|
429
|
+
else if (fn === 'credit' && fnArgs.length > 0 && fnArgs.length < 3) {
|
|
430
|
+
callCredit(... fnArgs)
|
|
366
431
|
.then(() => {
|
|
367
432
|
console.log('Execution completed');
|
|
368
433
|
});
|
package/compare.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
/*
|
|
2
2
|
compare.js
|
|
3
|
-
Creates a comparison from
|
|
3
|
+
Creates a comparison from a summary report.
|
|
4
4
|
Arguments:
|
|
5
5
|
0. Comparing function.
|
|
6
|
-
1.
|
|
6
|
+
1. Summary report.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
// ########## FUNCTIONS
|
|
10
10
|
|
|
11
|
-
// Compares the
|
|
12
|
-
exports.compare = async (comparer,
|
|
11
|
+
// Compares the summarized reports and returns a comparison.
|
|
12
|
+
exports.compare = async (id, what, comparer, summaryReport) => {
|
|
13
13
|
// Return the comparison.
|
|
14
|
-
console.log(`Comparison complete. Report count: ${
|
|
15
|
-
return comparer(
|
|
14
|
+
console.log(`Comparison complete. Report count: ${summaryReport.data.length}`);
|
|
15
|
+
return comparer(id, what, summaryReport);
|
|
16
16
|
};
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/*
|
|
2
2
|
credit.js
|
|
3
|
-
Analyzes tool coverages of issues in
|
|
3
|
+
Analyzes tool coverages of issues in an array of score properties of scored reports.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// Returns a tabulation of the instance counts of issues reported by tools in scored reports.
|
|
7
|
-
exports.credit =
|
|
7
|
+
exports.credit = (what, reportScores) => {
|
|
8
8
|
const tally = {
|
|
9
|
+
what,
|
|
9
10
|
instanceCounts: {},
|
|
10
11
|
onlies: {},
|
|
11
12
|
mosts: {},
|
|
@@ -19,11 +20,11 @@ exports.credit = reports => {
|
|
|
19
20
|
};
|
|
20
21
|
const {instanceCounts, onlies, mosts, tools, issueCounts} = tally;
|
|
21
22
|
// For each report:
|
|
22
|
-
|
|
23
|
+
reportScores.forEach(reportScore => {
|
|
23
24
|
// If it is valid:
|
|
24
|
-
if (
|
|
25
|
+
if (reportScore && reportScore.details && reportScore.details.issue) {
|
|
25
26
|
// For each issue:
|
|
26
|
-
const issues =
|
|
27
|
+
const issues = reportScore.details && reportScore.details.issue;
|
|
27
28
|
Object.keys(issues).forEach(issueID => {
|
|
28
29
|
// For each tool with any complaints about it:
|
|
29
30
|
if (! instanceCounts[issueID]) {
|
|
@@ -51,13 +52,6 @@ exports.credit = reports => {
|
|
|
51
52
|
}
|
|
52
53
|
instanceCounts[issueID][toolID].rules[ruleID] += complaints.countTotal;
|
|
53
54
|
}
|
|
54
|
-
// Otherwise, i.e. if no instance count was recorded:
|
|
55
|
-
else {
|
|
56
|
-
// Report this.
|
|
57
|
-
console.log(
|
|
58
|
-
`ERROR: Report ${report.id} missing countTotal for ${toolID} in ${issueID}`
|
|
59
|
-
);
|
|
60
|
-
}
|
|
61
55
|
});
|
|
62
56
|
});
|
|
63
57
|
}
|
|
@@ -66,11 +60,6 @@ exports.credit = reports => {
|
|
|
66
60
|
}
|
|
67
61
|
});
|
|
68
62
|
}
|
|
69
|
-
// Otherwise, i.e. if it is invalid:
|
|
70
|
-
else {
|
|
71
|
-
// Report this.
|
|
72
|
-
console.log(`ERROR: Report ${report.id} missing score data`);
|
|
73
|
-
}
|
|
74
63
|
});
|
|
75
64
|
// Populate the total and initial non-only issue counts.
|
|
76
65
|
issueCounts.nonOnlies = issueCounts.total = Object.keys(instanceCounts).length;
|
package/digest.js
CHANGED
|
@@ -1,25 +1,18 @@
|
|
|
1
1
|
/*
|
|
2
2
|
digest.js
|
|
3
|
-
Creates
|
|
3
|
+
Creates a digest from a scored report.
|
|
4
4
|
Arguments:
|
|
5
5
|
0. Digesting function.
|
|
6
|
-
1.
|
|
6
|
+
1. Scored report.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
// ########## FUNCTIONS
|
|
10
10
|
|
|
11
11
|
// Digests the scored reports and returns them, digested.
|
|
12
|
-
exports.digest = async (digester,
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
// Add the digest to the array of digests.
|
|
19
|
-
digests[report.id] = digestedReport;
|
|
20
|
-
console.log(`Report ${report.id} digested`);
|
|
21
|
-
};
|
|
22
|
-
// Return the digests.
|
|
23
|
-
console.log(`Digesting complete; report count ${reports.length}`);
|
|
24
|
-
return digests;
|
|
12
|
+
exports.digest = async (digester, report) => {
|
|
13
|
+
// Create a digest.
|
|
14
|
+
const digest = await digester(report);
|
|
15
|
+
console.log(`Report ${report.id} digested`);
|
|
16
|
+
// Return the digest.
|
|
17
|
+
return digest;
|
|
25
18
|
};
|
package/merge.js
CHANGED
|
@@ -28,6 +28,18 @@ const mergeIDLength = 2;
|
|
|
28
28
|
|
|
29
29
|
// Merges a script and a batch and returns jobs.
|
|
30
30
|
exports.merge = (script, batch, standard, observe, requester, timeStamp) => {
|
|
31
|
+
// If standard is invalid:
|
|
32
|
+
if (! ['also', 'only', 'no'].includes(standard)) {
|
|
33
|
+
// Report this and quit.
|
|
34
|
+
console.log('ERROR: Invalid standard treatment specified');
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
// If observe is invalid:
|
|
38
|
+
if (! [true, false].includes(observe)) {
|
|
39
|
+
// Report this and quit.
|
|
40
|
+
console.log('ERROR: Invalid observe configuration specified');
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
31
43
|
// If a time stamp was specified:
|
|
32
44
|
if (timeStamp) {
|
|
33
45
|
// If it is invalid:
|
package/package.json
CHANGED
|
@@ -8,16 +8,17 @@
|
|
|
8
8
|
<meta name="publisher" name="Testilo">
|
|
9
9
|
<meta name="description" content="comparison of accessibility scores">
|
|
10
10
|
<meta name="keywords" content="accessibility a11y web testing">
|
|
11
|
-
<title>Accessibility score comparison</title>
|
|
11
|
+
<title>Accessibility score comparison: __what__</title>
|
|
12
12
|
<link rel="icon" href="favicon.ico">
|
|
13
13
|
<link rel="stylesheet" href="style.css">
|
|
14
14
|
</head>
|
|
15
15
|
<body>
|
|
16
16
|
<main>
|
|
17
17
|
<header>
|
|
18
|
-
<h1>Accessibility score comparison</h1>
|
|
18
|
+
<h1>Accessibility score comparison: __what__</h1>
|
|
19
19
|
</header>
|
|
20
20
|
<h2>Introduction</h2>
|
|
21
|
+
<p>This is comparison __id__.</p>
|
|
21
22
|
<p>The table below compares __pageCount__ web pages on <a href="https://www.w3.org/WAI/fundamentals/accessibility-intro/">accessibility</a>. The page names are links to the pages on the web. The scores are links to digests that explain in detail how the scores were computed.</p>
|
|
22
23
|
<p>The pages were tested by <a href="https://www.npmjs.com/package/testaro">Testaro</a>. Testaro used ten tools (Alfa, ASLint, Axe, Editoria11y, Equal Access, HTML CodeSniffer, Nu Html Checker, QualWeb, Testaro, and WAVE) to perform about 900 automated accessibility tests.</p>
|
|
23
24
|
<p><a href="https://www.npmjs.com/package/testilo">Testilo</a> classified the problems found by these tests into <dfn>issues</dfn> and assigned a <dfn>score</dfn> to each page. A perfect score would be 0. Higher scores indicate more issues, more instances of them, more serious issues, and more of the tools reporting instances of issues.</p>
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/*
|
|
2
2
|
index
|
|
3
|
-
Compares scores in
|
|
3
|
+
Compares scores in a summary report.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// ########## IMPORTS
|
|
@@ -11,46 +11,26 @@ const {getBarCell, getNowDate, getNowDateSlash} = require('../../util');
|
|
|
11
11
|
|
|
12
12
|
// CONSTANTS
|
|
13
13
|
|
|
14
|
-
// Comparer ID.
|
|
15
|
-
const id = 'tcp39';
|
|
16
14
|
// Newlines with indentations.
|
|
17
15
|
const innestJoiner = '\n ';
|
|
18
16
|
|
|
19
17
|
// ########## FUNCTIONS
|
|
20
18
|
|
|
21
|
-
// Returns data on the targets.
|
|
22
|
-
const getData = async scoredReports => {
|
|
23
|
-
// For each scored report:
|
|
24
|
-
const bodyData = [];
|
|
25
|
-
for (const report of scoredReports) {
|
|
26
|
-
// Get data.
|
|
27
|
-
const {id, sources, score} = report;
|
|
28
|
-
if (id && sources && sources.script && score) {
|
|
29
|
-
bodyData.push({
|
|
30
|
-
id,
|
|
31
|
-
org: sources.target.what,
|
|
32
|
-
url: sources.target.which,
|
|
33
|
-
score: score.summary.total
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
};
|
|
37
|
-
// Return the report count, the script ID of the first report, and the data of all the reports.
|
|
38
|
-
return {
|
|
39
|
-
pageCount: scoredReports.length,
|
|
40
|
-
bodyData
|
|
41
|
-
}
|
|
42
|
-
};
|
|
43
19
|
// Returns the maximum score.
|
|
44
|
-
const getMaxScore =
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
20
|
+
const getMaxScore = summaryReport => summaryReport.data.reduce(
|
|
21
|
+
(max, result) => Math.max(max, result.score), 0
|
|
22
|
+
);
|
|
23
|
+
// Converts summary report data to a table body.
|
|
24
|
+
const getTableBody = async summaryReport => {
|
|
25
|
+
const maxScore = getMaxScore(summaryReport);
|
|
26
|
+
const rows = summaryReport.data
|
|
49
27
|
.sort((a, b) => a.score - b.score)
|
|
50
|
-
.map(
|
|
51
|
-
const {id,
|
|
52
|
-
const
|
|
53
|
-
const
|
|
28
|
+
.map(result => {
|
|
29
|
+
const {id, target, score} = result;
|
|
30
|
+
const {what, which} = target;
|
|
31
|
+
const pageCell = `<th scope="row"><a href="${which}">${what}</a></th>`;
|
|
32
|
+
const scoreDestination = process.env.DIGEST_URL.replace('__id__', id);
|
|
33
|
+
const numCell = `<td><a href="${scoreDestination}">${score}</a></td>`;
|
|
54
34
|
// Make the bar width proportional.
|
|
55
35
|
const barCell = getBarCell(score, maxScore, 25, false);
|
|
56
36
|
const row = `<tr>${pageCell}${numCell}${barCell}</tr>`;
|
|
@@ -59,18 +39,19 @@ const getTableBody = async bodyData => {
|
|
|
59
39
|
return rows.join(innestJoiner);
|
|
60
40
|
};
|
|
61
41
|
// Populates a query for a comparative table.
|
|
62
|
-
const populateQuery = async (
|
|
63
|
-
|
|
64
|
-
query.
|
|
65
|
-
query.
|
|
42
|
+
const populateQuery = async (id, what, summaryReport, query) => {
|
|
43
|
+
query.id = id;
|
|
44
|
+
query.what = what;
|
|
45
|
+
query.pageCount = summaryReport.data.length;
|
|
46
|
+
query.tableBody = await getTableBody(summaryReport);
|
|
66
47
|
query.dateISO = getNowDate();
|
|
67
48
|
query.dateSlash = getNowDateSlash();
|
|
68
49
|
};
|
|
69
|
-
// Returns a
|
|
70
|
-
exports.comparer = async
|
|
50
|
+
// Returns a comparison.
|
|
51
|
+
exports.comparer = async (id, what, summaryReport) => {
|
|
71
52
|
// Create a query to replace placeholders.
|
|
72
53
|
const query = {};
|
|
73
|
-
populateQuery(
|
|
54
|
+
populateQuery(id, what, summaryReport, query);
|
|
74
55
|
// Get the template.
|
|
75
56
|
let template = await fs.readFile(`${__dirname}/index.html`, 'utf8');
|
|
76
57
|
// Replace its placeholders.
|
package/procs/score/tic40.js
CHANGED
|
@@ -5801,6 +5801,13 @@ exports.issues = {
|
|
|
5801
5801
|
wcag: '2.5.5',
|
|
5802
5802
|
weight: 3,
|
|
5803
5803
|
tools: {
|
|
5804
|
+
alfa: {
|
|
5805
|
+
r111: {
|
|
5806
|
+
variable: false,
|
|
5807
|
+
quality: 1,
|
|
5808
|
+
what: 'Target size is substandard'
|
|
5809
|
+
}
|
|
5810
|
+
},
|
|
5804
5811
|
testaro: {
|
|
5805
5812
|
targetSize: {
|
|
5806
5813
|
variable: false,
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
</header>
|
|
20
20
|
<h2>Introduction</h2>
|
|
21
21
|
<p>This is tracking report <code>__id__</code>.</p>
|
|
22
|
-
<p>It tracks accessibility scores over time. A perfect score is 0. The tracking was performed by Testilo procedure __tp__
|
|
22
|
+
<p>It tracks accessibility scores over time. A perfect score is 0. The tracking was performed by Testilo procedure <code>__tp__</code>.</p>
|
|
23
23
|
<p>The results are presented first as a graph, and then as a table.</p>
|
|
24
24
|
<h2>Results as a graph</h2>
|
|
25
25
|
<figure id="graph">
|
|
@@ -29,9 +29,7 @@ const populateQuery = async (id, summary, query) => {
|
|
|
29
29
|
query.dateSlash = getNowDateSlash();
|
|
30
30
|
// JSON of pruned summary.
|
|
31
31
|
summary.data.forEach(result => {
|
|
32
|
-
delete result.id;
|
|
33
32
|
delete result.target.id;
|
|
34
|
-
delete result.target.which;
|
|
35
33
|
});
|
|
36
34
|
query.summaryJSON = JSON.stringify(summary);
|
|
37
35
|
// For each score:
|
package/score.js
CHANGED
|
@@ -1,20 +1,15 @@
|
|
|
1
1
|
/*
|
|
2
2
|
score.js
|
|
3
|
-
Scores Testaro
|
|
3
|
+
Scores a Testaro report.
|
|
4
4
|
Arguments:
|
|
5
5
|
0. Scoring function.
|
|
6
|
-
1.
|
|
6
|
+
1. Report.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
// ########## FUNCTIONS
|
|
10
10
|
|
|
11
|
-
// Scores the specified raw
|
|
12
|
-
exports.score = (scorer,
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
// Score it.
|
|
16
|
-
scorer(report);
|
|
17
|
-
console.log(`Report ${report.id} scored`);
|
|
18
|
-
}
|
|
19
|
-
console.log(`Scoring complete; report count ${reports.length}`);
|
|
11
|
+
// Scores the specified raw report.
|
|
12
|
+
exports.score = (scorer, report) => {
|
|
13
|
+
scorer(report);
|
|
14
|
+
console.log(`Report ${report.id} scored`);
|
|
20
15
|
};
|
package/script.js
CHANGED
|
@@ -18,7 +18,7 @@ let toolIDs = [
|
|
|
18
18
|
// ########## FUNCTIONS
|
|
19
19
|
|
|
20
20
|
// Creates and returns a script.
|
|
21
|
-
exports.script = (id, issues = null, ... issueIDs) => {
|
|
21
|
+
exports.script = (id, what, issues = null, ... issueIDs) => {
|
|
22
22
|
// Initialize data on the tools and their rules for the specified issues, if any.
|
|
23
23
|
const neededTools = {};
|
|
24
24
|
// If an issue classification and any issues were specified:
|
|
@@ -67,7 +67,7 @@ exports.script = (id, issues = null, ... issueIDs) => {
|
|
|
67
67
|
// Initialize a script.
|
|
68
68
|
const scriptObj = {
|
|
69
69
|
id,
|
|
70
|
-
what
|
|
70
|
+
what,
|
|
71
71
|
strict: true,
|
|
72
72
|
isolate: true,
|
|
73
73
|
timeLimit: Math.round(30 + (issueIDs.length || 300) / 2 + 20 * toolIDs.length),
|
package/summarize.js
CHANGED
|
@@ -1,35 +1,26 @@
|
|
|
1
1
|
/*
|
|
2
2
|
summarize.js
|
|
3
|
-
Returns a summary of
|
|
3
|
+
Returns a summary of a report.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// ########## IMPORTS
|
|
7
7
|
|
|
8
8
|
// Module to keep secrets.
|
|
9
9
|
require('dotenv').config();
|
|
10
|
-
// Module to perform common operations.
|
|
11
|
-
const {getFileID} = require('./procs/util');
|
|
12
10
|
|
|
13
11
|
// ########## FUNCTIONS
|
|
14
12
|
|
|
15
|
-
// Returns a summary.
|
|
16
|
-
exports.summarize =
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const target = sources && sources.target || '';
|
|
21
|
-
return {
|
|
22
|
-
id: id || '',
|
|
23
|
-
endTime: jobData && jobData.endTime || '',
|
|
24
|
-
order: order || '',
|
|
25
|
-
target,
|
|
26
|
-
score: score && score.summary && score.summary.total || null
|
|
27
|
-
};
|
|
28
|
-
});
|
|
13
|
+
// Returns a report summary.
|
|
14
|
+
exports.summarize = report => {
|
|
15
|
+
const {id, jobData, score, sources} = report;
|
|
16
|
+
const order = sources && sources.order || '';
|
|
17
|
+
const target = sources && sources.target || '';
|
|
29
18
|
const summary = {
|
|
30
|
-
id:
|
|
31
|
-
|
|
32
|
-
|
|
19
|
+
id: id || '',
|
|
20
|
+
endTime: jobData && jobData.endTime || '',
|
|
21
|
+
order: order || '',
|
|
22
|
+
target,
|
|
23
|
+
score: score && score.summary && score.summary.total || null
|
|
33
24
|
};
|
|
34
25
|
return summary;
|
|
35
26
|
};
|