testilo 13.7.0 → 13.8.0

package/README.md

@@ -446,7 +446,7 @@ The optional third argument to call (`75m` in this example) is a report selector
 
 To test the `score` module, in the project directory you can execute the statement `node validation/score/validate`. If `score` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.
 
-## Rport digesting
+## Report digesting
 
 ### Introduction
 
@@ -544,6 +544,50 @@ The fourth argument to `call` (`23pl` in this example) is optional. If it is omi
 
 The comparative report created by `compare` is an HTML file, and it expects a `style.css` file to exist in its directory. The `reports/comparative/style.css` file in Testilo is an appropriate stylesheet to be copied into the directory where comparative reports are written.
 
+### Tool crediting
+
+If you use Testilo to perform all the tests of all the tools on multiple targets and score the reports with a score proc that maps tool rules onto tool-agnostic issues, you may want to tabulate the comparative efficacy of each tool in discovering instances of issues. Testilo can help you do this by producing a _credit report_.
+
+The `credit` module tabulates the contribution of each tool to the discovery of issue instances in a collection of scored reports. Its `credit()` function takes one argument: an array of scored reports.
+
+The credit report contains four sections:
+- `counts`: for each issue, how many instances each tool reported
+- `onlies`: for each issue of which only 1 tool reported instances, which tool it was
+- `mosts`: for each issue of which at least 2 tools reported instances, which tool(s) reported the maximum instance count
+- `tools`: for each tool, two sections:
+    - `onlies`: a list of the issues that only the tool reported instances of
+    - `mosts`: a list of the issues for which the instance count of the tool was not surpassed by that of any other tool
+
+### Invocation
+
+There are two ways to use the `credit` module.
+
+#### By a module
+
+A module can invoke `credit` in this way:
+
+```javaScript
+const {credit} = require('testilo/credit');
+credit(scoredReports)
+.then(creditReport => {…});
+```
+
+The argument to `credit()` is an array of scored report objects. The `credit()` function returns a credit report. The invoking module can further dispose of the credit report as needed.
+
+#### By a user
+
+A user can invoke `credit` in this way:
+
+```bash
+node call credit legislators 23pl
+```
+
+When a user invokes `credit` in this example, the `call` module:
+- gets all the reports in the `scored` subdirectory of the `process.env.REPORTDIR` directory whose file names begin with `23pl`.
+- writes the credit report as JSON file named `legislators.json` to the `credit` subdirectory of the `process.env.REPORTDIR` directory.
+
+The third argument to `call` (`23pl` in this example) is optional. If it is omitted, `call` will get and `credit()` will tabulate all the reports in the `scored` directory.
+
 ### Validation
 
 To test the `compare` module, in the project directory you can execute the statement `node validation/compare/validate`. If `compare` is valid, all logging statements will begin with “Success” and none will begin with “ERROR”.

package/call.js

@@ -167,6 +167,26 @@ const callCompare = async (compareProcID, comparisonNameBase, selector = '') =>
     console.log(`Comparison completed and saved in ${comparisonDir}`);
   }
 };
+// Fulfills a credit request.
+const callCredit = async (tallyID, selector = '') => {
+  // Get the scored reports to be tallied.
+  const reports = await getReports('scored', selector);
+  // If any exist:
+  if (reports.length) {
+    // Get the tallier.
+    const {credit} = require(`${functionDir}/analyze/credit`);
+    // Tally the reports.
+    const tally = credit(reports);
+    // Save the tally.
+    await fs.writeFile(`${reportDir}/credit/${tallyID}.json`, JSON.stringify(tally, null, 2));
+    console.log(`Reports tallied and credit report saved in ${reportDir}/credit`);
+  }
+  // Otherwise, i.e. if no scored reports are to be tallied:
+  else {
+    // Report this.
+    console.log('ERROR: No scored reports to be tallied');
+  }
+};
 
 // ########## OPERATION
 
@@ -225,6 +245,12 @@ else if (fn === 'compare' && fnArgs.length > 1 && fnArgs.length < 4) {
     console.log('Execution completed');
   });
 }
+else if (fn === 'credit' && fnArgs.length === 2) {
+  callCredit(... fnArgs)
+  .then(() => {
+    console.log('Execution completed');
+  });
+}
 else {
   console.log('ERROR: Invalid statement');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testilo",
-  "version": "13.7.0",
+  "version": "13.8.0",
   "description": "Prepares and processes Testaro reports",
   "main": "aim.js",
   "scripts": {

package/procs/analyze/credit.js

@@ -0,0 +1,97 @@
+/*
+  credit.js
+  Analyzes tool coverages of issues in a set of reports.
+*/
+
+// Returns a tabulation of the instance counts of issues reported by tools in scored reports.
+exports.credit = reports => {
+  const tally = {
+    counts: {},
+    onlies: {},
+    mosts: {},
+    tools: {}
+  };
+  const {counts, onlies, mosts, tools} = tally;
+  // For each report:
+  reports.forEach(report => {
+    // If it is valid:
+    if (report.score && report.score.details && report.score.details.issue) {
+      // For each issue:
+      const issues = report.score.details.issue;
+      Object.keys(issues).forEach(issueID => {
+        // For each tool with any complaints about it:
+        if (! counts[issueID]) {
+          counts[issueID] = {};
+        }
+        const issueTools = issues[issueID].tools;
+        if (issueTools) {
+          Object.keys(issueTools).forEach(toolID => {
+            // For each rule cited by any of those complaints:
+            if (! counts[issueID][toolID]) {
+              counts[issueID][toolID] = 0;
+            }
+            Object.keys(issueTools[toolID]).forEach(ruleID => {
+              // If an instance count was recorded:
+              const {complaints} = issueTools[toolID][ruleID];
+              if (complaints && complaints.countTotal) {
+                // Add it to the tally.
+                counts[issueID][toolID] += complaints.countTotal;
+              }
+              // Otherwise, i.e. if no instance count was recorded:
+              else {
+                // Report this.
+                console.log(`ERROR: Missing countTotal for ${toolID} in ${issueID}`);
+              }
+            });
+          });
+        }
+        else {
+          console.log(`ERROR: Missing tools for ${issueID}`);
+        }
+      });
+    }
+    // Otherwise, i.e. if it is invalid:
+    else {
+      // Report this.
+      console.log(`ERROR: Report ${report.id} missing score data`);
+    }
+  });
+  // For each tallied issue:
+  Object.keys(counts).forEach(issueID => {
+    // If only 1 tool complained about it:
+    const toolIDs = Object.keys(counts[issueID])
+    if (toolIDs.length === 1) {
+      // Add this to the tally.
+      onlies[issueID] = toolIDs[0];
+    }
+    // Otherwise, i.e. if multiple tools complained about it:
+    else {
+      // Add the tools with the maximum instance count to the tally.
+      const maxCount = Object
+      .values(counts[issueID])
+      .reduce((max, current) => Math.max(max, current));
+      Object.keys(counts[issueID]).forEach(toolID => {
+        if (counts[issueID][toolID] === maxCount) {
+          if (! mosts[issueID]) {
+            mosts[issueID] = [];
+          }
+          mosts[issueID].push(toolID);
+        }
+      });
+    }
+  });
+  // Add the onlies and mosts to the tool tabulation in the tally.
+  [[onlies, 'onlies'], [mosts, 'mosts']].forEach(qualityPair => {
+    Object.keys(qualityPair[0]).forEach(issueID => {
+      const toolID = qualityPair[0][issueID];
+      if (! tools[toolID]) {
+        tools[toolID] = {
+          onlies: [],
+          mosts: []
+        };
+      }
+      tools[toolID][qualityPair[1]].push(issueID);
+    });
+  });
+  return tally;
+};