testilo 19.3.0 → 19.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/procs/analyze/credit.js +29 -18
package/README.md
CHANGED
|
@@ -607,7 +607,7 @@ The comparative report created by `compare` is an HTML file, and it expects a `s
|
|
|
607
607
|
|
|
608
608
|
### Tool crediting
|
|
609
609
|
|
|
610
|
-
If you use
|
|
610
|
+
If you use Testaro to perform all the tests of all the tools on multiple targets and score the reports with a score proc that maps tool rules onto tool-agnostic issues, you may want to tabulate the comparative efficacy of each tool in discovering instances of issues. Testilo can help you do this by producing a _credit report_.
|
|
611
611
|
|
|
612
612
|
The `credit` module tabulates the contribution of each tool to the discovery of issue instances in a collection of scored reports. Its `credit()` function takes one argument: an array of scored reports.
|
|
613
613
|
|
package/package.json
CHANGED
package/procs/analyze/credit.js
CHANGED
|
@@ -6,29 +6,35 @@
|
|
|
6
6
|
// Returns a tabulation of the instance counts of issues reported by tools in scored reports.
|
|
7
7
|
exports.credit = reports => {
|
|
8
8
|
const tally = {
|
|
9
|
-
|
|
9
|
+
instanceCounts: {},
|
|
10
10
|
onlies: {},
|
|
11
11
|
mosts: {},
|
|
12
|
-
tools: {}
|
|
12
|
+
tools: {},
|
|
13
|
+
issueCounts: {
|
|
14
|
+
total: 0,
|
|
15
|
+
onlies: {}
|
|
16
|
+
}
|
|
13
17
|
};
|
|
14
|
-
const {
|
|
18
|
+
const {instanceCounts, onlies, mosts, tools, issueCounts} = tally;
|
|
15
19
|
// For each report:
|
|
16
20
|
reports.forEach(report => {
|
|
17
21
|
// If it is valid:
|
|
18
22
|
if (report.score && report.score.details && report.score.details.issue) {
|
|
19
|
-
//
|
|
23
|
+
// Populate the issue count.
|
|
20
24
|
const issues = report.score.details && report.score.details.issue;
|
|
25
|
+
issueCounts.total = Object.keys(issues).length;
|
|
26
|
+
// For each issue:
|
|
21
27
|
Object.keys(issues).forEach(issueID => {
|
|
22
28
|
// For each tool with any complaints about it:
|
|
23
|
-
if (!
|
|
24
|
-
|
|
29
|
+
if (! instanceCounts[issueID]) {
|
|
30
|
+
instanceCounts[issueID] = {};
|
|
25
31
|
}
|
|
26
32
|
const issueTools = issues[issueID].tools;
|
|
27
33
|
if (issueTools) {
|
|
28
34
|
Object.keys(issueTools).forEach(toolID => {
|
|
29
35
|
// For each rule cited by any of those complaints:
|
|
30
|
-
if (!
|
|
31
|
-
|
|
36
|
+
if (! instanceCounts[issueID][toolID]) {
|
|
37
|
+
instanceCounts[issueID][toolID] = {
|
|
32
38
|
total: 0,
|
|
33
39
|
rules: {}
|
|
34
40
|
};
|
|
@@ -38,12 +44,12 @@ exports.credit = reports => {
|
|
|
38
44
|
const {complaints} = issueTools[toolID][ruleID];
|
|
39
45
|
if (complaints && complaints.countTotal) {
|
|
40
46
|
// Add it to the tally.
|
|
41
|
-
|
|
47
|
+
instanceCounts[issueID][toolID].total += complaints.countTotal;
|
|
42
48
|
// Add a rule itemization to the tally.
|
|
43
|
-
if (!
|
|
44
|
-
|
|
49
|
+
if (! instanceCounts[issueID][toolID].rules[ruleID]) {
|
|
50
|
+
instanceCounts[issueID][toolID].rules[ruleID] = 0;
|
|
45
51
|
}
|
|
46
|
-
|
|
52
|
+
instanceCounts[issueID][toolID].rules[ruleID] += complaints.countTotal;
|
|
47
53
|
}
|
|
48
54
|
// Otherwise, i.e. if no instance count was recorded:
|
|
49
55
|
else {
|
|
@@ -67,21 +73,26 @@ exports.credit = reports => {
|
|
|
67
73
|
}
|
|
68
74
|
});
|
|
69
75
|
// For each tallied issue:
|
|
70
|
-
Object.keys(
|
|
76
|
+
Object.keys(instanceCounts).forEach(issueID => {
|
|
71
77
|
// If only 1 tool complained about it:
|
|
72
|
-
const toolIDs = Object.keys(
|
|
78
|
+
const toolIDs = Object.keys(instanceCounts[issueID])
|
|
73
79
|
if (toolIDs.length === 1) {
|
|
74
80
|
// Add this to the tally.
|
|
75
|
-
|
|
81
|
+
const toolID = toolIDs[0];
|
|
82
|
+
onlies[issueID] = toolID;
|
|
83
|
+
if (! issueCounts.onlies[toolID]) {
|
|
84
|
+
issueCounts.onlies[toolID] = 0;
|
|
85
|
+
}
|
|
86
|
+
issueCounts.onlies[toolID]++;
|
|
76
87
|
}
|
|
77
88
|
// Otherwise, i.e. if multiple tools complained about it:
|
|
78
89
|
else {
|
|
79
90
|
// Add the tools with the maximum instance count to the tally.
|
|
80
91
|
const maxCount = Object
|
|
81
|
-
.values(
|
|
92
|
+
.values(instanceCounts[issueID])
|
|
82
93
|
.reduce((max, current) => Math.max(max, current ? current.total : 0), 0);
|
|
83
|
-
Object.keys(
|
|
84
|
-
if (maxCount &&
|
|
94
|
+
Object.keys(instanceCounts[issueID]).forEach(toolID => {
|
|
95
|
+
if (maxCount && instanceCounts[issueID][toolID].total === maxCount) {
|
|
85
96
|
if (! mosts[issueID]) {
|
|
86
97
|
mosts[issueID] = [];
|
|
87
98
|
}
|