testaro 2.1.2 → 2.2.0

package/README.md

@@ -8,9 +8,9 @@ Testaro is a collection of web accessibility tests.
 
 The purpose of Testaro is to provide programmatic access to over 600 accessibility tests defined in several test packages and in Testaro itself.
 
-Calling Testaro requires telling it which operations (including tests) to perform, which URLs to perform them on, and where to write its reports.
+Running Testaro requires telling it which operations (including tests) to perform and which URLs to perform them on, and giving Testaro an object to put its output into.
 
-Testaro outputs progress messages and a list of reports to the standard output. It writes the reports in JSON format.
+Testaro outputs progress messages to the standard output. It populates the object with log information and test reports.
 
 ## System requirements
 
@@ -47,7 +47,7 @@ The main directories containing code files are:
 - package root: main code files
 - `tests`: files containing the code defining particular tests
 - `procs`: shared procedures
-- `validation`: code and artifacts for the validation of tests
+- `validation`: code and artifacts for the validation of Testaro
 
 ## Installation
 
@@ -256,12 +256,6 @@ An example of a **Testaro-defined** test is:
 
 In this case, Testaro runs the `motion` test with the specified parameters.
 
-###### Validation
-
-For each of the non-package tests defined in Testaro, a validating script and some files tested by it are also provided. They are in the `validation` directory. A module that you can use to execute those scripts is also there, named `validator.js`.
-
-For example, to execute the `focOp.json` validation script, you can enter `node validation/validator focOp`. The `validator.js` module outputs the report to the console.
-
 ##### Scoring
 
 An example of a **scoring** command is:
@@ -442,6 +436,17 @@ If a `wave` test is included in the script, an environment variable named `TESTA
 
 Before executing a Testaro script, you can optionally also set the environment variables `TESTARO_DEBUG` (to `'true'` or anything else) and/or `TESTARO_WAITS` (to a non-negative integer). The effects of these variables are described in the `index.js` file.
 
+## Validation
+
+Three _executors_ for Testaro validation are located in the `validation` directory. An executor is a commonJS JavaScript module that runs Testaro and reports whether the results are correct.
+
+The executors are:
+- `appNoBatch.js`: Reports whether Testaro runs correctly with a no-batch script.
+- `appBatch.js`: Reports whether Testaro runs correctly with a script and a batch.
+- `tests.js`: Runs Testaro with each custom test and reports whether the results are correct.
+
+To execute any executor `xyz.js`, call it with the statement `node validation/executors/xyz`. The results will appear in the standard output.
+
 ## Contribution
 
 You can define additional Testaro commands and functionality. Contributions are welcome.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "2.1.2",
+  "version": "2.2.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/scoring/correlation.js

@@ -5,9 +5,9 @@
 */
 const fs = require('fs');
 const compile = () => {
-  const issuesJSON = fs.readFileSync(`${__dirname}/scoring/package/issues.json`, 'utf8');
+  const issuesJSON = fs.readFileSync(`${__dirname}/package/issues.json`, 'utf8');
   const issues = JSON.parse(issuesJSON);
-  const dataJSON = fs.readFileSync(`${__dirname}/scoring/package/data.json`, 'utf8');
+  const dataJSON = fs.readFileSync(`${__dirname}/package/data.json`, 'utf8');
   const reportData = JSON.parse(dataJSON);
   const reports = Object.values(reportData);
   // Initialize the list.
@@ -72,5 +72,5 @@ const compile = () => {
   return data;
 };
 fs.writeFileSync(
-  `${__dirname}/scoring/package/correlations.json`, JSON.stringify(compile(), null, 2)
+  `${__dirname}/package/correlations.json`, JSON.stringify(compile(), null, 2)
 );

package/scoring/dupCounts.js

@@ -7,7 +7,7 @@
 */
 const fs = require('fs');
 const compile = () => {
-  const dupsJSON = fs.readFileSync(`${__dirname}/scoring/package/duplications.json`, 'utf8');
+  const dupsJSON = fs.readFileSync(`${__dirname}/package/duplications.json`, 'utf8');
   const dups = JSON.parse(dupsJSON);
   // Initialize the object.
   const data = {
@@ -36,4 +36,4 @@ const compile = () => {
   });
   return data;
 };
-fs.writeFileSync(`${__dirname}/scoring/package/dupCounts.json`, JSON.stringify(compile(), null, 2));
+fs.writeFileSync(`${__dirname}/package/dupCounts.json`, JSON.stringify(compile(), null, 2));

package/scoring/packageData.js

@@ -120,7 +120,7 @@ const compilers = {
 };
 const repo = process.argv[2];
 const compile = repo => {
-  const dirPath = `${__dirname}/../${repo}/reports/script`;
+  const dirPath = `${__dirname}/../../${repo}/reports/script`;
   const batchDirNames = fs
   .readdirSync(dirPath, {withFileTypes: true})
   .filter(dirEnt => dirEnt.isDirectory())
@@ -168,4 +168,4 @@ const compile = repo => {
   });
   return data;
 };
-fs.writeFileSync(`${__dirname}/scoring/package/data.json`, JSON.stringify(compile(repo), null, 2));
+fs.writeFileSync(`${__dirname}/package/data.json`, JSON.stringify(compile(repo), null, 2));

package/tests/radioSet.js

@@ -12,7 +12,7 @@ exports.reporter = async (page, withItems) => {
   // If itemization is required:
   if (withItems) {
     // Add the body of the textOf function as a string to the array.
-    const textOfBody = await fs.readFile(`${__dirname}/procs/test/textOf.txt`, 'utf8');
+    const textOfBody = await fs.readFile(`${__dirname}/../procs/test/textOf.txt`, 'utf8');
     args.push(textOfBody);
   }
   // Get the result data.

package/validation/executors/appBatch.js

@@ -0,0 +1,87 @@
+// app.js
+// Validator for Testaro application with a script and a batch.
+
+const options = {
+  script: {
+    what: 'Sample Testaro executor with 1 test',
+    strict: true,
+    commands: [
+      {
+        type: 'launch',
+        which: 'chromium',
+        what: 'Chromium browser'
+      },
+      {
+        type: 'url',
+        which: 'https://*',
+        what: 'URL to be replaced with URLs in the batch'
+      },
+      {
+        type: 'test',
+        which: 'bulk',
+        what: 'bulk'
+      },
+      {
+        type: 'test',
+        which: 'noSuchTest',
+        what: 'test that does not exist'
+      }
+    ]
+  },
+  batch: {
+    what: 'Two websites',
+    hosts: [
+      {
+        which: 'https://www.w3.org/',
+        what: 'W3C'
+      },
+      {
+        which: 'https://www.wikimedia.org/',
+        what: 'Wikimedia'
+      }
+    ]
+  },
+  log: [],
+  reports: []
+};
+const {handleRequest} = require(`${__dirname}/../../index`);
+const isValidReport = (reports, index) => {
+  const isValid = reports[index].acts
+  && reports[index].acts.length === 4
+  && reports[index].acts[2].result
+  && reports[index].acts[2].result.visibleElements
+  && typeof reports[index].acts[2].result.visibleElements === 'number'
+  && reports[index].acts[3].result
+  && typeof reports[index].acts[3].result === 'string'
+  && reports[index].acts[3].result.startsWith('ERROR');
+  return isValid;
+};
+handleRequest(options)
+.then(
+  () => {
+    const {log, reports} = options;
+    if (
+      log.length === 4
+      && log[1].event === 'timeStamp'
+      && /^[a-z0-9]+$/.test(log[1].value)
+      && log[2].event === 'batchSize'
+      && log[2].value === 2
+    ) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (reports.length === 2 && isValidReport(reports, 0) && isValidReport(reports, 1)) {
+      console.log('Success: Reports have been correctly populated');
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  },
+  rejection => {
+    console.log(`Failure: ${rejection}`);
+  }
+);

package/validation/executors/appNoBatch.js

@@ -0,0 +1,67 @@
+// app.js
+// Validator for Testaro application with a no-batch script.
+
+const options = {
+  script: {
+    what: 'Sample Testaro executor with 1 test',
+    strict: true,
+    commands: [
+      {
+        type: 'launch',
+        which: 'chromium',
+        what: 'Chromium browser'
+      },
+      {
+        type: 'url',
+        which: 'https://example.com/',
+        what: 'simple page, replaced if there is a batch'
+      },
+      {
+        type: 'test',
+        which: 'bulk',
+        what: 'bulk'
+      },
+      {
+        type: 'test',
+        which: 'noSuchTest',
+        what: 'test that does not exist'
+      }
+    ]
+  },
+  log: [],
+  reports: []
+};
+const {handleRequest} = require(`${__dirname}/../../index`);
+handleRequest(options)
+.then(
+  () => {
+    const {log, reports} = options;
+    if (log.length === 3 && log[1].event === 'timeStamp' && /^[a-z0-9]+$/.test(log[1].value)) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (
+      reports.length === 1
+      && reports[0].acts
+      && reports[0].acts.length === 4
+      && reports[0].acts[2].result
+      && reports[0].acts[2].result.visibleElements
+      && typeof reports[0].acts[2].result.visibleElements === 'number'
+      && reports[0].acts[3].result
+      && typeof reports[0].acts[3].result === 'string'
+      && reports[0].acts[3].result.startsWith('ERROR')
+    ) {
+      console.log('Success: Reports have been correctly populated');
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  },
+  rejection => {
+    console.log(`Failure: ${rejection}`);
+  }
+);

package/validation/executors/tests.js

@@ -0,0 +1,53 @@
+// app.js
+// Validator for Testaro tests.
+
+const fs = require('fs').promises;
+const {handleRequest} = require(`${__dirname}/../../index`);
+const validateTests = async () => {
+  const scriptFileNames = await fs.readdir(`${__dirname}/../tests/scripts`);
+  for (const scriptFileName of scriptFileNames) {
+    const rawScriptJSON = await fs
+    .readFile(`${__dirname}/../tests/scripts/${scriptFileName}`, 'utf8');
+    const scriptJSON = rawScriptJSON
+    .replace(/__targets__/g, `file://${__dirname}/../tests/targets`);
+    const script = JSON.parse(scriptJSON);
+    const options = {script};
+    options.log = [];
+    options.reports = [];
+    await handleRequest(options);
+    const {log, reports} = options;
+    if (log.length === 3 && log[1].event === 'timeStamp' && /^[a-z0-9]+$/.test(log[1].value)) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (
+      reports.length === 1
+      && reports[0].acts
+      && reports[0].acts.length === script.commands.length
+      && reports[0].acts.every(
+        act => act.type && act.type === 'test'
+          ? act.result && act.result.failureCount !== undefined 
+          : true
+      )
+    ) {
+      console.log('Success: Reports have been correctly populated');
+      if (reports[0].acts.every(
+        act => act.type === 'test' ? act.result.failureCount === 0 : true
+      )) {
+        console.log('Success: No failures');
+      }
+      else {
+        console.log('Failure: At least one test has at least one failure');
+        console.log(JSON.stringify(reports, null, 2));
+      }
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  }
+};
+validateTests();

package/validation/{scripts/test → tests/scripts}/bulk.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/bulk/good.html",
+      "which": "__targets__/bulk/good.html",
       "what": "small page"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/bulk/bad.html",
+      "which": "__targets__/bulk/bad.html",
       "what": "large page"
     },
     {

package/validation/{scripts/test → tests/scripts}/embAc.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/embAc/good.html",
+      "which": "__targets__/embAc/good.html",
       "what": "page without embedding in links or buttons"
     },
     {
@@ -26,7 +26,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/embAc/bad.html",
+      "which": "__targets__/embAc/bad.html",
       "what": "page with embeddings in links and buttons"
     },
     {

package/validation/{scripts/test → tests/scripts}/focAll.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/good.html",
+      "which": "__targets__/focAll/good.html",
       "what": "fully Tab-focusable page"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/less.html",
+      "which": "__targets__/focAll/less.html",
       "what": "partly Tab-focusable page"
     },
     {
@@ -41,7 +41,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/more.html",
+      "which": "__targets__/focAll/more.html",
       "what": "page with added Tab-focusability"
     },
     {

package/validation/{scripts/test → tests/scripts}/focInd.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focInd/good.html",
+      "which": "__targets__/focInd/good.html",
       "what": "page with outline focus indication"
     },
     {
@@ -30,7 +30,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focInd/bad.html",
+      "which": "__targets__/focInd/bad.html",
       "what": "page with mixed focus indication"
     },
     {

package/validation/{scripts/test → tests/scripts}/focOp.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focOp/good.html",
+      "which": "__targets__/focOp/good.html",
       "what": "page with standard focusability and operability"
     },
     {
@@ -29,7 +29,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focOp/bad.html",
+      "which": "__targets__/focOp/bad.html",
       "what": "page with deviant focusability and operability"
     },
     {

package/validation/{scripts/test → tests/scripts}/hover.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/hover/good.html",
+      "which": "__targets__/hover/good.html",
       "what": "page with standard hover behavior"
     },
     {
@@ -27,7 +27,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/hover/bad.html",
+      "which": "__targets__/hover/bad.html",
       "what": "page with deviant hover behavior"
     },
     {

package/validation/{scripts/test → tests/scripts}/labClash.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/labClash/good.html",
+      "which": "__targets__/labClash/good.html",
       "what": "page with standard labeling"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/labClash/bad.html",
+      "which": "__targets__/labClash/bad.html",
       "what": "page with deviant labeling"
     },
     {

package/validation/{scripts/test → tests/scripts}/linkUl.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/good.html",
+      "which": "__targets__/linkUl/good.html",
       "what": "page with underlined inline links"
     },
     {
@@ -26,7 +26,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/bad.html",
+      "which": "__targets__/linkUl/bad.html",
       "what": "page without underlined inline links"
     },
     {
@@ -43,7 +43,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/na.html",
+      "which": "__targets__/linkUl/na.html",
       "what": "page without inline links"
     },
     {

package/validation/{scripts/test → tests/scripts}/menuNav.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/menuNav/good.html",
+      "which": "__targets__/menuNav/good.html",
       "what": "page with standard menu navigation"
     },
     {
@@ -52,7 +52,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/menuNav/bad.html",
+      "which": "__targets__/menuNav/bad.html",
       "what": "page with deviant menu navigation"
     },
     {

package/validation/{scripts/test → tests/scripts}/motion.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/motion/good.html",
+      "which": "__targets__/motion/good.html",
       "what": "page without motion"
     },
     {
@@ -30,7 +30,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/motion/bad.html",
+      "which": "__targets__/motion/bad.html",
       "what": "page with motion"
     },
     {

package/validation/{scripts/test → tests/scripts}/radioSet.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/radioSet/good.html",
+      "which": "__targets__/radioSet/good.html",
       "what": "page with standard radio-button grouping"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/radioSet/bad.html",
+      "which": "__targets__/radioSet/bad.html",
       "what": "page with deviant radio-button grouping"
     },
     {

package/validation/{scripts/test → tests/scripts}/role.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/role/good.html",
+      "which": "__targets__/role/good.html",
       "what": "page with standard roles"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/role/bad.html",
+      "which": "__targets__/role/bad.html",
       "what": "page with deviant roles"
     },
     {

package/validation/{scripts/test → tests/scripts}/styleDiff.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/styleDiff/good.html",
+      "which": "__targets__/styleDiff/good.html",
       "what": "page with consistent styles"
     },
     {
@@ -32,7 +32,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/styleDiff/bad.html",
+      "which": "__targets__/styleDiff/bad.html",
       "what": "page with consistent styles"
     },
     {

package/validation/{scripts/test → tests/scripts}/tabNav.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/tabNav/good.html",
+      "which": "__targets__/tabNav/good.html",
       "what": "page with standard menu navigation"
     },
     {
@@ -52,7 +52,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/tabNav/bad.html",
+      "which": "__targets__/tabNav/bad.html",
       "what": "page with deviant menu navigation"
     },
     {

package/validation/{scripts/test → tests/scripts}/zIndex.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/zIndex/good.html",
+      "which": "__targets__/zIndex/good.html",
       "what": "page with no explicit z-index attributes"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/zIndex/bad.html",
+      "which": "__targets__/zIndex/bad.html",
       "what": "page with explicit z-index attributes roles"
     },
     {

package/validation/batches/sample.json

@@ -1,13 +0,0 @@
-{
-  "what": "Sample batch",
-  "hosts": [
-    {
-      "which": "https://www.w3.org/",
-      "what": "W3C"
-    },
-    {
-      "which": "https://www.wikimedia.org/",
-      "what": "Wikimedia"
-    }
-  ]
-}

package/validation/executors/sample.js

@@ -1,11 +0,0 @@
-// sample.js
-// Sample executor for Testaro.
-
-const options = {
-  reports: `${__dirname}/../reports`,
-  // To use the sample batch, uncomment the following line.
-  batches: `${__dirname}/../batches/sample.json`,
-  script: `${__dirname}/../scripts/app/sample.json`
-};
-const {handleRequest} = require('../../index');
-handleRequest(options);

package/validation/scripts/app/sample.json

@@ -1,21 +0,0 @@
-{
-  "what": "Sample Testaro executor with 1 test",
-  "strict": true,
-  "commands": [
-    {
-      "type": "launch",
-      "which": "chromium",
-      "what": "Chromium browser"
-    },
-    {
-      "type": "url",
-      "which": "https://example.com/",
-      "what": "simple page, replaced if there is a batch"
-    },
-    {
-      "type": "test",
-      "which": "bulk",
-      "what": "bulk"
-    }
-  ]
-}