testaro 2.1.3 → 2.2.1

package/README.md

@@ -8,9 +8,9 @@ Testaro is a collection of web accessibility tests.
 
 The purpose of Testaro is to provide programmatic access to over 600 accessibility tests defined in several test packages and in Testaro itself.
 
-Calling Testaro requires telling it which operations (including tests) to perform, which URLs to perform them on, and where to write its reports.
+Running Testaro requires telling it which operations (including tests) to perform and which URLs to perform them on, and giving Testaro an object to put its output into.
 
-Testaro outputs progress messages and a list of reports to the standard output. It writes the reports in JSON format.
+Testaro outputs progress messages to the standard output. It populates the object with log information and test reports.
 
 ## System requirements
 
@@ -47,7 +47,7 @@ The main directories containing code files are:
 - package root: main code files
 - `tests`: files containing the code defining particular tests
 - `procs`: shared procedures
-- `validation`: code and artifacts for the validation of tests
+- `validation`: code and artifacts for the validation of Testaro
 
 ## Installation
 
@@ -256,12 +256,6 @@ An example of a **Testaro-defined** test is:
 
 In this case, Testaro runs the `motion` test with the specified parameters.
 
-###### Validation
-
-For each of the non-package tests defined in Testaro, a validating script and some files tested by it are also provided. They are in the `validation` directory. A module that you can use to execute those scripts is also there, named `validator.js`.
-
-For example, to execute the `focOp.json` validation script, you can enter `node validation/validator focOp`. The `validator.js` module outputs the report to the console.
-
 ##### Scoring
 
 An example of a **scoring** command is:
@@ -383,6 +377,8 @@ The second item in each array, if there are 3 items in the array, is an operator
 - `>`: greater than
 - `!`: unequal to
 
+A typical use for an `expect` property is checking the correctness of a Testaro test. Thus, the validation scripts in the `validation/tests/scripts` directory all contain `test` commands with `expect` properties. See the “Validation” section below.
+
 ## Batches
 
 There are two ways to use a script to give instructions to Testaro:
@@ -442,6 +438,19 @@ If a `wave` test is included in the script, an environment variable named `TESTA
 
 Before executing a Testaro script, you can optionally also set the environment variables `TESTARO_DEBUG` (to `'true'` or anything else) and/or `TESTARO_WAITS` (to a non-negative integer). The effects of these variables are described in the `index.js` file.
 
+## Validation
+
+Three _executors_ for Testaro validation are located in the `validation` directory. An executor is a commonJS JavaScript module that runs Testaro and reports whether the results are correct.
+
+The executors are:
+- `appNoBatch.js`: Reports whether Testaro runs correctly with a no-batch script.
+- `appBatch.js`: Reports whether Testaro runs correctly with a script and a batch.
+- `tests.js`: Runs Testaro with each custom test and reports whether the results are correct.
+
+To execute any executor `xyz.js`, call it with the statement `node validation/executors/xyz`. The results will appear in the standard output.
+
+The `tests.js` executor makes use of the scripts in the `validation/tests/scripts` directory, and they, in turn, run tests on HTML files in the `validation/tests/targets` directory.
+
 ## Contribution
 
 You can define additional Testaro commands and functionality. Contributions are welcome.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "2.1.3",
+  "version": "2.2.1",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/tests/hover.js

@@ -8,188 +8,226 @@
   examined are the descendants of the grandparent of the element hovered over if that element
   has the tag name 'A' or 'BUTTON' or otherwise the descendants of the element. The only
   elements counted as being made visible by hovering are those with tag names 'A', 'BUTTON',
-  'INPUT', and 'SPAN', and those with 'role="menuitem"' attributes.
+  'INPUT', and 'SPAN', and those with 'role="menuitem"' attributes. The test waits 700 ms after
+  each hover in case of delayed effects. Despite this delay, the test makes the execution time
+  practical by randomly sampling targets instead of hovering over all of them. Therefore, the
+  results may vary from one execution to another.
 */
-exports.reporter = async (page, withItems) => {
-  // Initialize a counter.
-  let elementsChecked = 0;
-  // Identify the elements that are likely to trigger disclosures on hover.
-  const triggers = await page.$$(
-    'body a:visible, body button:visible, body li:visible, body [onmouseenter]:visible, body [onmouseover]:visible'
-  )
-  .catch(error => {
-    console.log(`ERROR getting hover triggers (${error.message})`);
-    return [];
-  });
-  // Identify the selectors of active elements likely to be disclosed by a hover.
-  const targetSelectors = ['a', 'button', 'input', '[role=menuitem]', 'span']
-  .map(selector => `${selector}:visible`)
-  .join(', ');
-  // Initialize the result.
-  const data = {
-    totals: {
-      triggers: 0,
-      madeVisible: 0,
-      opacityChanged: 0,
-      opacityAffected: 0,
-      unhoverables: 0
+
+// CONSTANTS
+
+// Selectors of active elements likely to be disclosed by a hover.
+const targetSelectors = ['a', 'button', 'input', '[role=menuitem]', 'span']
+.map(selector => `${selector}:visible`)
+.join(', ');
+// Initialize the result.
+const data = {
+  totals: {
+    triggers: 0,
+    madeVisible: 0,
+    opacityChanged: 0,
+    opacityAffected: 0,
+    unhoverables: 0
+  }
+};
+
+// VARIABLES
+
+// Counter.
+let elementsChecked = 0;
+
+// FUNCTIONS
+
+// Samples a population and returns the sample.
+const getSample = (population, sampleSize) => {
+  const popSize = population.length;
+  if (sampleSize > 0 && sampleSize < popSize) {
+    const sample = new Set();
+    while (sample.size < sampleSize) {
+      const index = Math.floor(popSize * Math.random());
+      sample.add(population[index]);
     }
-  };
+    return Array.from(sample);
+  }
+  else {
+    return [];
+  }
+};
+// Recursively finds and reports triggers and targets.
+const find = async (withItems, page, triggers) => {
   if (withItems) {
     data.items = {
       triggers: [],
       unhoverables: []
     };
   }
-  let triggerTag = '';
-  // FUNCTION DEFINITION START
-  // Recursively finds and reports triggers and targets.
-  const find = async triggers => {
-    // If any potential disclosure triggers remain:
-    if (triggers.length) {
-      // Identify the first of them.
-      const firstTrigger = triggers[0];
-      const tagNameJSHandle = await firstTrigger.getProperty('tagName')
-      .catch(error => {
-        console.log(`ERROR getting trigger tag name (${error.message})`);
-        return '';
-      });
-      if (tagNameJSHandle) {
-        const tagName = await tagNameJSHandle.jsonValue();
-        // Identify the root of a subtree likely to contain disclosed elements.
-        let root = firstTrigger;
-        if (['A', 'BUTTON'].includes(tagName)) {
-          const rootJSHandle = await page.evaluateHandle(
-            firstTrigger => {
-              const parent = firstTrigger.parentElement;
-              if (parent) {
-                return parent.parentElement || parent;
-              }
-              else {
-                return firstTrigger;
-              }
-            },
-            firstTrigger
-          );
-          root = rootJSHandle.asElement();
-        }
-        // Identify the visible active descendants of the root before the hover.
-        const preVisibles = await root.$$(targetSelectors);
-        // Identify all the descendants of the root.
-        const descendants = await root.$$('*');
-        // Identify their opacities before the hover.
-        const preOpacities = await page.evaluate(
-          elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-        );
-        try {
-          // Hover over the potential trigger.
-          await firstTrigger.hover({timeout: 700});
-          // Identify whether it controls other elements.
-          const isController = await page.evaluate(
-            element => element.ariaHasPopup || element.hasAttribute('aria-controls'), firstTrigger
-          );
-          // Wait for any delayed and/or slowed hover reaction if likely.
-          await page.waitForTimeout(
-            elementsChecked++ < 10 || tagName !== triggerTag || isController ? 1200 : 200
-          );
-          await root.waitForElementState('stable');
-          // Identify the visible active descendants of the root during the hover.
-          const postVisibles = await root.$$(targetSelectors);
-          // Identify the opacities of the descendants of the root during the hover.
-          const postOpacities = await page.evaluate(
-            elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
-          );
-          // Identify the elements with opacity changes.
-          const opacityTargets = descendants
-          .filter((descendant, index) => postOpacities[index] !== preOpacities[index]);
-          // Count them and their descendants.
-          const opacityAffected = opacityTargets.length
-            ? await page.evaluate(elements => elements.reduce(
-              (total, current) => total + 1 + current.querySelectorAll('*').length, 0
-            ), opacityTargets)
-            : 0;
-          // If hovering disclosed any element or changed any opacity:
-          if (postVisibles.length > preVisibles.length || opacityAffected) {
-            // Preserve the lengthened reaction wait, if any, for the next 5 tries.
-            if (elementsChecked < 11) {
-              elementsChecked = 5;
+  // If any potential disclosure triggers remain:
+  if (triggers.length) {
+    // Identify the first of them.
+    const firstTrigger = triggers[0];
+    const tagNameJSHandle = await firstTrigger.getProperty('tagName')
+    .catch(error => {
+      console.log(`ERROR getting trigger tag name (${error.message})`);
+      return '';
+    });
+    if (tagNameJSHandle) {
+      const tagName = await tagNameJSHandle.jsonValue();
+      // Identify the root of a subtree likely to contain disclosed elements.
+      let root = firstTrigger;
+      if (['A', 'BUTTON'].includes(tagName)) {
+        const rootJSHandle = await page.evaluateHandle(
+          firstTrigger => {
+            const parent = firstTrigger.parentElement;
+            if (parent) {
+              return parent.parentElement || parent;
             }
-            // Hover over the upper-left corner of the page, to undo any hover reactions.
-            await page.hover('body', {
-              position: {
-                x: 0,
-                y: 0
-              }
-            });
-            // Wait for any delayed and/or slowed hover reaction.
-            await page.waitForTimeout(200);
-            await root.waitForElementState('stable');
-            // Increment the counts of triggers and targets.
-            data.totals.triggers++;
-            const madeVisible = Math.max(0, postVisibles.length - preVisibles.length);
-            data.totals.madeVisible += madeVisible;
-            data.totals.opacityChanged += opacityTargets.length;
-            data.totals.opacityAffected += opacityAffected;
-            // If details are to be reported:
-            if (withItems) {
-              // Report them.
-              const triggerDataJSHandle = await page.evaluateHandle(args => {
-                // Returns the text of an element.
-                const textOf = (element, limit) => {
-                  const text = element.textContent.trim() || element.outerHTML.trim();
-                  return text.replace(/\s{2,}/sg, ' ').slice(0, limit);
-                };
-                const trigger = args[0];
-                const preVisibles = args[1];
-                const postVisibles = args[2];
-                const madeVisible = postVisibles
-                .filter(el => ! preVisibles.includes(el))
-                .map(el => ({
-                  tagName: el.tagName,
-                  text: textOf(el, 50)
-                }));
-                const opacityChanged = args[3].map(el => ({
-                  tagName: el.tagName,
-                  text: textOf(el, 50)
-                }));
-                return {
-                  tagName: trigger.tagName,
-                  id: trigger.id || '',
-                  text: textOf(trigger, 50),
-                  madeVisible,
-                  opacityChanged
-                };
-              }, [firstTrigger, preVisibles, postVisibles, opacityTargets]);
-              const triggerData = await triggerDataJSHandle.jsonValue();
-              data.items.triggers.push(triggerData);
+            else {
+              return firstTrigger;
             }
+          },
+          firstTrigger
+        );
+        root = rootJSHandle.asElement();
+      }
+      // Identify the visible active descendants of the root before the hover.
+      const preVisibles = await root.$$(targetSelectors);
+      // Identify all the descendants of the root.
+      const descendants = await root.$$('*');
+      // Identify their opacities before the hover.
+      const preOpacities = await page.evaluate(
+        elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
+      );
+      try {
+        // Hover over the potential trigger.
+        await firstTrigger.hover({timeout: 700});
+        // Identify whether it is coded as controlling other elements.
+        const isController = await page.evaluate(
+          element => element.ariaHasPopup || element.hasAttribute('aria-controls'), firstTrigger
+        );
+        // Wait for any delayed and/or slowed hover reaction, longer if coded as a controller.
+        await page.waitForTimeout(isController ? 1200 : 600);
+        await root.waitForElementState('stable');
+        // Identify the visible active descendants of the root during the hover.
+        const postVisibles = await root.$$(targetSelectors);
+        // Identify the opacities of the descendants of the root during the hover.
+        const postOpacities = await page.evaluate(
+          elements => elements.map(el => window.getComputedStyle(el).opacity), descendants
+        );
+        // Identify the elements with opacity changes.
+        const opacityTargets = descendants
+        .filter((descendant, index) => postOpacities[index] !== preOpacities[index]);
+        // Count them and their descendants.
+        const opacityAffected = opacityTargets.length
+          ? await page.evaluate(elements => elements.reduce(
+            (total, current) => total + 1 + current.querySelectorAll('*').length, 0
+          ), opacityTargets)
+          : 0;
+        // If hovering disclosed any element or changed any opacity:
+        if (postVisibles.length > preVisibles.length || opacityAffected) {
+          // Preserve the lengthened reaction wait, if any, for the next 5 tries.
+          if (elementsChecked < 11) {
+            elementsChecked = 5;
           }
-        }
-        catch (error) {
-          console.log('ERROR hovering');
-          // Returns the text of an element.
-          const textOf = async (element, limit) => {
-            let text = await element.textContent();
-            text = text.trim() || await element.innerHTML();
-            return text.trim().replace(/\s*/sg, '').slice(0, limit);
-          };
-          data.totals.unhoverables++;
+          // Hover over the upper-left corner of the page, to undo any hover reactions.
+          await page.hover('body', {
+            position: {
+              x: 0,
+              y: 0
+            }
+          });
+          // Wait for any delayed and/or slowed hover reaction.
+          await page.waitForTimeout(200);
+          await root.waitForElementState('stable');
+          // Increment the counts of triggers and targets.
+          data.totals.triggers++;
+          const madeVisible = Math.max(0, postVisibles.length - preVisibles.length);
+          data.totals.madeVisible += madeVisible;
+          data.totals.opacityChanged += opacityTargets.length;
+          data.totals.opacityAffected += opacityAffected;
+          // If details are to be reported:
           if (withItems) {
-            data.items.unhoverables.push({
-              tagName: tagName,
-              id: firstTrigger.id || '',
-              text: await textOf(firstTrigger, 50)
-            });
+            // Report them.
+            const triggerDataJSHandle = await page.evaluateHandle(args => {
+              // Returns the text of an element.
+              const textOf = (element, limit) => {
+                const text = element.textContent.trim() || element.outerHTML.trim();
+                return text.replace(/\s{2,}/sg, ' ').slice(0, limit);
+              };
+              const trigger = args[0];
+              const preVisibles = args[1];
+              const postVisibles = args[2];
+              const madeVisible = postVisibles
+              .filter(el => ! preVisibles.includes(el))
+              .map(el => ({
+                tagName: el.tagName,
+                text: textOf(el, 50)
+              }));
+              const opacityChanged = args[3].map(el => ({
+                tagName: el.tagName,
+                text: textOf(el, 50)
+              }));
+              return {
+                tagName: trigger.tagName,
+                id: trigger.id || '',
+                text: textOf(trigger, 50),
+                madeVisible,
+                opacityChanged
+              };
+            }, [firstTrigger, preVisibles, postVisibles, opacityTargets]);
+            const triggerData = await triggerDataJSHandle.jsonValue();
+            data.items.triggers.push(triggerData);
           }
         }
-        triggerTag = tagName;
       }
-      // Process the remaining potential triggers.
-      await find(triggers.slice(1));
+      catch (error) {
+        console.log('ERROR hovering');
+        // Returns the text of an element.
+        const textOf = async (element, limit) => {
+          let text = await element.textContent();
+          text = text.trim() || await element.innerHTML();
+          return text.trim().replace(/\s*/sg, '').slice(0, limit);
+        };
+        data.totals.unhoverables++;
+        if (withItems) {
+          data.items.unhoverables.push({
+            tagName: tagName,
+            id: firstTrigger.id || '',
+            text: await textOf(firstTrigger, 50)
+          });
+        }
+      }
     }
-  };
+    // Process the remaining potential triggers.
+    await find(withItems, page, triggers.slice(1));
+  }
+};
+exports.reporter = async (page, withItems) => {
+  // Identify the triggers.
+  const selectors = [
+    'body a:visible',
+    'body button:visible',
+    'body li:visible, body [onmouseenter]:visible',
+    'body [onmouseover]:visible'
+  ];
+  const triggers = await page.$$(selectors.join(', '))
+  .catch(error => {
+    console.log(`ERROR getting hover triggers (${error.message})`);
+    return [];
+  });
+  // If they number more than the sample size limit, sample them.
+  const triggerCount = triggers.length;
+  const sampleSize = 15;
+  const triggerSample = triggerCount > sampleSize ? getSample(triggers, 15) : triggers;
   // Find and document the hover-triggered disclosures.
-  await find(triggers);
+  await find(withItems, page, triggerSample);
+  // If the triggers were sampled:
+  if (triggerCount > sampleSize) {
+    // Change the totals to population estimates.
+    const multiplier = triggerCount / sampleSize;
+    Object.keys(data.totals).forEach(key => {
+      data.totals[key] = Math.round(multiplier * data.totals[key]);
+    });
+  }
   // Return the result.
   return {result: data};
 };

package/tests/radioSet.js

@@ -12,7 +12,7 @@ exports.reporter = async (page, withItems) => {
   // If itemization is required:
   if (withItems) {
     // Add the body of the textOf function as a string to the array.
-    const textOfBody = await fs.readFile(`${__dirname}../procs/test/textOf.txt`, 'utf8');
+    const textOfBody = await fs.readFile(`${__dirname}/../procs/test/textOf.txt`, 'utf8');
     args.push(textOfBody);
   }
   // Get the result data.

package/validation/executors/appBatch.js

@@ -0,0 +1,87 @@
+// app.js
+// Validator for Testaro application with a script and a batch.
+
+const options = {
+  script: {
+    what: 'Sample Testaro executor with 1 test',
+    strict: true,
+    commands: [
+      {
+        type: 'launch',
+        which: 'chromium',
+        what: 'Chromium browser'
+      },
+      {
+        type: 'url',
+        which: 'https://*',
+        what: 'URL to be replaced with URLs in the batch'
+      },
+      {
+        type: 'test',
+        which: 'bulk',
+        what: 'bulk'
+      },
+      {
+        type: 'test',
+        which: 'noSuchTest',
+        what: 'test that does not exist'
+      }
+    ]
+  },
+  batch: {
+    what: 'Two websites',
+    hosts: [
+      {
+        which: 'https://www.w3.org/',
+        what: 'W3C'
+      },
+      {
+        which: 'https://www.wikimedia.org/',
+        what: 'Wikimedia'
+      }
+    ]
+  },
+  log: [],
+  reports: []
+};
+const {handleRequest} = require(`${__dirname}/../../index`);
+const isValidReport = (reports, index) => {
+  const isValid = reports[index].acts
+  && reports[index].acts.length === 4
+  && reports[index].acts[2].result
+  && reports[index].acts[2].result.visibleElements
+  && typeof reports[index].acts[2].result.visibleElements === 'number'
+  && reports[index].acts[3].result
+  && typeof reports[index].acts[3].result === 'string'
+  && reports[index].acts[3].result.startsWith('ERROR');
+  return isValid;
+};
+handleRequest(options)
+.then(
+  () => {
+    const {log, reports} = options;
+    if (
+      log.length === 4
+      && log[1].event === 'timeStamp'
+      && /^[a-z0-9]+$/.test(log[1].value)
+      && log[2].event === 'batchSize'
+      && log[2].value === 2
+    ) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (reports.length === 2 && isValidReport(reports, 0) && isValidReport(reports, 1)) {
+      console.log('Success: Reports have been correctly populated');
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  },
+  rejection => {
+    console.log(`Failure: ${rejection}`);
+  }
+);

package/validation/executors/appNoBatch.js

@@ -0,0 +1,67 @@
+// app.js
+// Validator for Testaro application with a no-batch script.
+
+const options = {
+  script: {
+    what: 'Sample Testaro executor with 1 test',
+    strict: true,
+    commands: [
+      {
+        type: 'launch',
+        which: 'chromium',
+        what: 'Chromium browser'
+      },
+      {
+        type: 'url',
+        which: 'https://example.com/',
+        what: 'simple page, replaced if there is a batch'
+      },
+      {
+        type: 'test',
+        which: 'bulk',
+        what: 'bulk'
+      },
+      {
+        type: 'test',
+        which: 'noSuchTest',
+        what: 'test that does not exist'
+      }
+    ]
+  },
+  log: [],
+  reports: []
+};
+const {handleRequest} = require(`${__dirname}/../../index`);
+handleRequest(options)
+.then(
+  () => {
+    const {log, reports} = options;
+    if (log.length === 3 && log[1].event === 'timeStamp' && /^[a-z0-9]+$/.test(log[1].value)) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (
+      reports.length === 1
+      && reports[0].acts
+      && reports[0].acts.length === 4
+      && reports[0].acts[2].result
+      && reports[0].acts[2].result.visibleElements
+      && typeof reports[0].acts[2].result.visibleElements === 'number'
+      && reports[0].acts[3].result
+      && typeof reports[0].acts[3].result === 'string'
+      && reports[0].acts[3].result.startsWith('ERROR')
+    ) {
+      console.log('Success: Reports have been correctly populated');
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  },
+  rejection => {
+    console.log(`Failure: ${rejection}`);
+  }
+);

package/validation/executors/tests.js

@@ -0,0 +1,53 @@
+// app.js
+// Validator for Testaro tests.
+
+const fs = require('fs').promises;
+const {handleRequest} = require(`${__dirname}/../../index`);
+const validateTests = async () => {
+  const scriptFileNames = await fs.readdir(`${__dirname}/../tests/scripts`);
+  for (const scriptFileName of scriptFileNames) {
+    const rawScriptJSON = await fs
+    .readFile(`${__dirname}/../tests/scripts/${scriptFileName}`, 'utf8');
+    const scriptJSON = rawScriptJSON
+    .replace(/__targets__/g, `file://${__dirname}/../tests/targets`);
+    const script = JSON.parse(scriptJSON);
+    const options = {script};
+    options.log = [];
+    options.reports = [];
+    await handleRequest(options);
+    const {log, reports} = options;
+    if (log.length === 3 && log[1].event === 'timeStamp' && /^[a-z0-9]+$/.test(log[1].value)) {
+      console.log('Success: Log has been correctly populated');
+    }
+    else {
+      console.log('Failure: Log empty or invalid');
+      console.log(JSON.stringify(log, null, 2));
+    }
+    if (
+      reports.length === 1
+      && reports[0].acts
+      && reports[0].acts.length === script.commands.length
+      && reports[0].acts.every(
+        act => act.type && act.type === 'test'
+          ? act.result && act.result.failureCount !== undefined 
+          : true
+      )
+    ) {
+      console.log('Success: Reports have been correctly populated');
+      if (reports[0].acts.every(
+        act => act.type === 'test' ? act.result.failureCount === 0 : true
+      )) {
+        console.log('Success: No failures');
+      }
+      else {
+        console.log('Failure: At least one test has at least one failure');
+        console.log(JSON.stringify(reports, null, 2));
+      }
+    }
+    else {
+      console.log('Failure: Reports empty or invalid');
+      console.log(JSON.stringify(reports, null, 2));
+    }
+  }
+};
+validateTests();

package/validation/{scripts/test → tests/scripts}/bulk.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/bulk/good.html",
+      "which": "__targets__/bulk/good.html",
       "what": "small page"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/bulk/bad.html",
+      "which": "__targets__/bulk/bad.html",
       "what": "large page"
     },
     {

package/validation/{scripts/test → tests/scripts}/embAc.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/embAc/good.html",
+      "which": "__targets__/embAc/good.html",
       "what": "page without embedding in links or buttons"
     },
     {
@@ -26,7 +26,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/embAc/bad.html",
+      "which": "__targets__/embAc/bad.html",
       "what": "page with embeddings in links and buttons"
     },
     {

package/validation/{scripts/test → tests/scripts}/focAll.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/good.html",
+      "which": "__targets__/focAll/good.html",
       "what": "fully Tab-focusable page"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/less.html",
+      "which": "__targets__/focAll/less.html",
       "what": "partly Tab-focusable page"
     },
     {
@@ -41,7 +41,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focAll/more.html",
+      "which": "__targets__/focAll/more.html",
       "what": "page with added Tab-focusability"
     },
     {

package/validation/{scripts/test → tests/scripts}/focInd.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focInd/good.html",
+      "which": "__targets__/focInd/good.html",
       "what": "page with outline focus indication"
     },
     {
@@ -30,7 +30,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focInd/bad.html",
+      "which": "__targets__/focInd/bad.html",
       "what": "page with mixed focus indication"
     },
     {

package/validation/{scripts/test → tests/scripts}/focOp.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focOp/good.html",
+      "which": "__targets__/focOp/good.html",
       "what": "page with standard focusability and operability"
     },
     {
@@ -29,7 +29,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/focOp/bad.html",
+      "which": "__targets__/focOp/bad.html",
       "what": "page with deviant focusability and operability"
     },
     {

package/validation/{scripts/test → tests/scripts}/hover.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/hover/good.html",
+      "which": "__targets__/hover/good.html",
       "what": "page with standard hover behavior"
     },
     {
@@ -27,7 +27,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/hover/bad.html",
+      "which": "__targets__/hover/bad.html",
       "what": "page with deviant hover behavior"
     },
     {

package/validation/{scripts/test → tests/scripts}/labClash.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/labClash/good.html",
+      "which": "__targets__/labClash/good.html",
       "what": "page with standard labeling"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/labClash/bad.html",
+      "which": "__targets__/labClash/bad.html",
       "what": "page with deviant labeling"
     },
     {

package/validation/{scripts/test → tests/scripts}/linkUl.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/good.html",
+      "which": "__targets__/linkUl/good.html",
       "what": "page with underlined inline links"
     },
     {
@@ -26,7 +26,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/bad.html",
+      "which": "__targets__/linkUl/bad.html",
       "what": "page without underlined inline links"
     },
     {
@@ -43,7 +43,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/linkUl/na.html",
+      "which": "__targets__/linkUl/na.html",
       "what": "page without inline links"
     },
     {

package/validation/{scripts/test → tests/scripts}/menuNav.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/menuNav/good.html",
+      "which": "__targets__/menuNav/good.html",
       "what": "page with standard menu navigation"
     },
     {
@@ -52,7 +52,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/menuNav/bad.html",
+      "which": "__targets__/menuNav/bad.html",
       "what": "page with deviant menu navigation"
     },
     {

package/validation/{scripts/test → tests/scripts}/motion.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/motion/good.html",
+      "which": "__targets__/motion/good.html",
       "what": "page without motion"
     },
     {
@@ -30,7 +30,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/motion/bad.html",
+      "which": "__targets__/motion/bad.html",
       "what": "page with motion"
     },
     {

package/validation/{scripts/test → tests/scripts}/radioSet.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/radioSet/good.html",
+      "which": "__targets__/radioSet/good.html",
       "what": "page with standard radio-button grouping"
     },
     {
@@ -25,7 +25,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/radioSet/bad.html",
+      "which": "__targets__/radioSet/bad.html",
       "what": "page with deviant radio-button grouping"
     },
     {

package/validation/{scripts/test → tests/scripts}/role.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/role/good.html",
+      "which": "__targets__/role/good.html",
       "what": "page with standard roles"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/role/bad.html",
+      "which": "__targets__/role/bad.html",
       "what": "page with deviant roles"
     },
     {

package/validation/{scripts/test → tests/scripts}/styleDiff.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/styleDiff/good.html",
+      "which": "__targets__/styleDiff/good.html",
       "what": "page with consistent styles"
     },
     {
@@ -32,7 +32,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/styleDiff/bad.html",
+      "which": "__targets__/styleDiff/bad.html",
       "what": "page with consistent styles"
     },
     {

package/validation/{scripts/test → tests/scripts}/tabNav.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/tabNav/good.html",
+      "which": "__targets__/tabNav/good.html",
       "what": "page with standard menu navigation"
     },
     {
@@ -52,7 +52,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/tabNav/bad.html",
+      "which": "__targets__/tabNav/bad.html",
       "what": "page with deviant menu navigation"
     },
     {

package/validation/{scripts/test → tests/scripts}/zIndex.json

@@ -9,7 +9,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/zIndex/good.html",
+      "which": "__targets__/zIndex/good.html",
       "what": "page with no explicit z-index attributes"
     },
     {
@@ -23,7 +23,7 @@
     },
     {
       "type": "url",
-      "which": "file://__dirname/validation/targets/zIndex/bad.html",
+      "which": "__targets__/zIndex/bad.html",
       "what": "page with explicit z-index attributes roles"
     },
     {

package/validation/batches/sample.json

@@ -1,13 +0,0 @@
-{
-  "what": "Sample batch",
-  "hosts": [
-    {
-      "which": "https://www.w3.org/",
-      "what": "W3C"
-    },
-    {
-      "which": "https://www.wikimedia.org/",
-      "what": "Wikimedia"
-    }
-  ]
-}

package/validation/executors/sample.js

@@ -1,11 +0,0 @@
-// sample.js
-// Sample executor for Testaro.
-
-const options = {
-  reports: `${__dirname}/../../reports`,
-  // To use the sample batch, uncomment the following line.
-  batches: `${__dirname}/../../batches/sample.json`,
-  script: `${__dirname}/../../scripts/app/sample.json`
-};
-const {handleRequest} = require('../../index');
-handleRequest(options);

package/validation/scripts/app/sample.json

@@ -1,21 +0,0 @@
-{
-  "what": "Sample Testaro executor with 1 test",
-  "strict": true,
-  "commands": [
-    {
-      "type": "launch",
-      "which": "chromium",
-      "what": "Chromium browser"
-    },
-    {
-      "type": "url",
-      "which": "https://example.com/",
-      "what": "simple page, replaced if there is a batch"
-    },
-    {
-      "type": "test",
-      "which": "bulk",
-      "what": "bulk"
-    }
-  ]
-}