testaro 1.0.0

package/scoring/packageData.js

@@ -0,0 +1,171 @@
+/*
+  packageData
+  Compiles data on the per-URL distributions of issues in package tests.
+  Arguments:
+    0. Name of sibling repo containing reports/script directory.
+*/
+const fs = require('fs');
+const compilers = {
+  aatt: act => {
+    const {result} = act;
+    let data = null;
+    if (Array.isArray(result)) {
+      data = {};
+      result.forEach(issue => {
+        const {type, id} = issue;
+        if (type && id) {
+          const typeID = `${type[0]}:${id}`;
+          if (data[typeID]) {
+            data[typeID]++;
+          }
+          else {
+            data[typeID] = 1;
+          }
+        }
+      });
+    }
+    return data;
+  },
+  alfa: act => {
+    const {result} = act;
+    let data = null;
+    if (Array.isArray(result)) {
+      data = {};
+      result.forEach(issue => {
+        const {rule} = issue;
+        if (rule) {
+          const {ruleID} = rule;
+          if (data[ruleID]) {
+            data[ruleID]++;
+          }
+          else {
+            data[ruleID] = 1;
+          }
+        }
+      });
+    }
+    return data;
+  },
+  axe: act => {
+    const {result} = act;
+    const {items} = result;
+    let data = null;
+    if (items) {
+      data = {};
+      items.forEach(item => {
+        const {rule, elements} = item;
+        if (data[rule]) {
+          data[rule] += elements.length;
+        }
+        else {
+          data[rule] = elements.length;
+        }
+      });
+    }
+    return data;
+  },
+  ibm: act => {
+    const {result} = act;
+    const {content, url} = result;
+    const contentViolations = content && content.totals && content.totals.violation;
+    const urlViolations = url && url.totals && url.totals.violation;
+    let data = null;
+    if (contentViolations || urlViolations) {
+      let items;
+      if (contentViolations && urlViolations) {
+        items = contentViolations > urlViolations ? content.items : url.items;
+      }
+      else {
+        items = content.items || url.items;
+      }
+      if (items) {
+        data = {};
+        items.forEach(item => {
+          const {ruleId, level} = item;
+          const issueID = `${level[0]}:${ruleId}`;
+          if (data[issueID]) {
+            data[issueID]++;
+          }
+          else {
+            data[issueID] = 1;
+          }
+        });
+      }
+    }
+    return data;
+  },
+  wave: act => {
+    const {result} = act;
+    const {categories} = result;
+    let data = null;
+    if (categories) {
+      data = {};
+      const {error, contrast, alert} = categories;
+      [error, contrast, alert].forEach((category, index) => {
+        const {items} = category;
+        Object.keys(items).forEach(ruleName => {
+          const ruleID = `${'eca'[index]}:${ruleName}`;
+          const {count} = items[ruleName];
+          if (data[ruleID]) {
+            data[ruleID] += count;
+          }
+          else {
+            data[ruleID] = count;
+          }
+        });
+      });
+    }
+    return data;
+  }
+};
+const repo = process.argv[2];
+const compile = repo => {
+  const dirPath = `../${repo}/reports/script`;
+  const batchDirNames = fs
+  .readdirSync(dirPath, {withFileTypes: true})
+  .filter(dirEnt => dirEnt.isDirectory())
+  .map(dirEnt => dirEnt.name)
+  .filter(dirName => fs.readdirSync(`${dirPath}/${dirName}`).includes('jsonReports'));
+  const data = {};
+  batchDirNames.forEach(batchDirName => {
+    const reportNames = fs.readdirSync(`${dirPath}/${batchDirName}/jsonReports`);
+    reportNames.forEach(reportName => {
+      const reportJSON = fs.readFileSync(`${dirPath}/${batchDirName}/jsonReports/${reportName}`);
+      const report = JSON.parse(reportJSON);
+      const {acts} = report;
+      const urlAct = acts.find(act => act.type === 'url');
+      if (urlAct) {
+        const url = urlAct.result;
+        if (! data[url]) {
+          data[url] = {
+            aatt: {},
+            alfa: {},
+            axe: {},
+            ibm: {},
+            wave: {}
+          };
+        }
+        const testActs = acts.filter(act => act.type === 'test');
+        testActs.forEach(testAct => {
+          if (testAct.which === 'aatt') {
+            data[url].aatt = compilers.aatt(testAct);
+          }
+          else if (testAct.which === 'alfa') {
+            data[url].alfa = compilers.alfa(testAct);
+          }
+          else if (testAct.which === 'axe') {
+            data[url].axe = compilers.axe(testAct);
+          }
+          else if (testAct.which === 'ibm') {
+            data[url].ibm = compilers.ibm(testAct);
+          }
+          else if (testAct.which === 'wave') {
+            data[url].wave = compilers.wave(testAct);
+          }
+        });
+      }
+    });
+  });
+  return data;
+};
+fs.writeFileSync('scoring/package/data.json', JSON.stringify(compile(repo), null, 2));

package/scoring/packageIssues.js

@@ -0,0 +1,34 @@
+/*
+  packageIssues
+  Compiles a list of all issue types appearing in packageData.json.
+*/
+const fs = require('fs');
+const compile = () => {
+  const dataJSON = fs.readFileSync('scoring/package/data.json', 'utf8');
+  const reportData = JSON.parse(dataJSON);
+  const reports = Object.values(reportData);
+  const data = {
+    aatt: new Set(),
+    alfa: new Set(),
+    axe: new Set(),
+    ibm: new Set(),
+    wave: new Set()
+  };
+  reports.forEach(entry => {
+    Object.keys(entry).forEach(key => {
+      if (entry[key] !== null) {
+        Object.keys(entry[key]).forEach(issueName => {
+          data[key].add(issueName);
+        });
+      }
+    });
+  });
+  return {
+    aatt: Array.from(data.aatt).sort(),
+    alfa: Array.from(data.alfa).sort(),
+    axe: Array.from(data.axe).sort(),
+    ibm: Array.from(data.ibm).sort(),
+    wave: Array.from(data.wave).sort()
+  };
+};
+fs.writeFileSync('scoring/package/issues.json', JSON.stringify(compile(), null, 2));

package/scoring/rulesetData.json

@@ -0,0 +1,15 @@
+{
+  "aatt": {
+    "public": true,
+    "url": ""
+  },
+  "axe": {
+    "public": true
+  },
+  "ibm": {
+    "public": true
+  },
+  "wave": {
+    "public": false
+  }
+}

package/tests/aatt.js

@@ -0,0 +1,64 @@
+/*
+  aatt
+  This test implements the HTML CodeSniffer ruleset for accessibility via AATT.
+*/
+
+// IMPORTS
+const {evaluate} = require('aatt');
+
+// FUNCTIONS
+exports.reporter = async page => {
+  const timeLimit = 30;
+  // Get the HTML of the document body.
+  const source = await page.content();
+  // Return the result of a test with the HTML CodeSniffer WCAG 2.1 AA ruleset as a string.
+  const report = evaluate({
+    source,
+    output: 'json',
+    engine: 'htmlcs',
+    level: 'WCAG2AA'
+  });
+  // Wait for it until the time limit expires.
+  let timeoutID;
+  const wait = new Promise(resolve => {
+    timeoutID = setTimeout(() => {
+      resolve('');
+    }, 1000 * timeLimit);
+  });
+  const resultIfFast = await Promise.race([report, wait]);
+  // If it arrived within the time limit:
+  if (resultIfFast) {
+    clearTimeout(timeoutID);
+    // Remove the non-JSON prefix and (if any) suffix from the string.
+    const reportJSON = resultIfFast.replace(/^.+?Object]\s+|\s+done\s*$/sg, '');
+    try {
+      // Convert the JSON string to an array.
+      const issueArray = JSON.parse(reportJSON);
+      // Remove the notices from the array.
+      const nonNotices = issueArray.filter(issue => issue.type !== 'notice');
+      // Convert the technique property from a string to an array of strings.
+      nonNotices.forEach(issue => {
+        if (issue.type) {
+          const longTech = issue.techniques;
+          issue.techniques = longTech.replace(/a><a/g, 'a>%<a').split('%');
+          issue.id = issue
+          .techniques
+          .map(technique => technique.replace(/^.+?>|<\/a>$/g, ''))
+          .sort()
+          .join('+');
+        }
+      });
+      return {result: nonNotices};
+    }
+    catch (error) {
+      console.log(`ERROR processing AATT report (${error.message})`);
+      return {result: 'ERROR processing AATT report'};
+    }
+  }
+  // Otherwise, i.e. if the result did not arrive within the time limit:
+  else {
+    // Report the failure.
+    console.log('ERROR: getting report took too long');
+    return {result: 'ERROR: getting AATT report took too long'};
+  }
+};

package/tests/alfa.js

@@ -0,0 +1,107 @@
+/*
+  alfa
+  This test implements the alfa ruleset for accessibility.
+*/
+
+// IMPORTS
+const {Audit} = require('@siteimprove/alfa-act');
+const {Scraper} = require('@siteimprove/alfa-scraper');
+const alfaRules = require('@siteimprove/alfa-rules');
+
+// FUNCTIONS
+// Conducts and reports an alfa test.
+exports.reporter = async page => {
+  // Get the document containing the summaries of the alfa rules.
+  const context = page.context();
+  const rulePage = await context.newPage();
+  rulePage.on('console', msg => {
+    const msgText = msg.text();
+    console.log(msgText);
+  });
+  const response = await rulePage.goto('https://alfa.siteimprove.com/rules', {timeout: 10000})
+  .catch(error => {
+    console.log(`ERROR: navigation to URL timed out (${error})`);
+    return {result: {error: 'ERROR: navigation to URL timed out'}};
+  });
+  let ruleData = {};
+  if (response.status() === 200) {
+    // Compile data on the rule IDs and summaries.
+    ruleData = await rulePage.evaluate(() => {
+      const rulePs = Array.from(document.querySelectorAll('p.h5'));
+      const ruleData = {};
+      rulePs.forEach(ruleP => {
+        const childNodes = Array.from(ruleP.childNodes);
+        const ruleID = childNodes[0].textContent.slice(4).toLowerCase();
+        const ruleText = childNodes
+        .slice(1)
+        .map(node => node.textContent)
+        .join(' ')
+        .trim()
+        .replace(/"/g, '\'')
+        .replace(/\s+/g, ' ');
+        ruleData[ruleID] = ruleText;
+      });
+      return ruleData;
+    });
+    await rulePage.close();
+  }
+  const data = [];
+  await Scraper.with(async scraper => {
+    for (const input of await scraper.scrape(page.url())) {
+      const audit = Audit.of(input, alfaRules.default);
+      const outcomes = Array.from(await audit.evaluate());
+      outcomes.forEach((outcome, index) => {
+        const {target} = outcome;
+        if (target && ! target._members) {
+          const outcomeJ = outcome.toJSON();
+          const verdict = outcomeJ.outcome;
+          if (verdict !== 'passed') {
+            const {rule} = outcomeJ;
+            const {tags, uri, requirements} = rule;
+            const ruleID = uri.replace(/^.+-/, '');
+            const ruleSummary = ruleData[ruleID] || '';
+            const targetJ = outcomeJ.target;
+            const codeLines = target.toString().split('\n');
+            if (codeLines[0] === '#document') {
+              codeLines.splice(2, codeLines.length - 3, '...');
+            }
+            else if (codeLines[0].startsWith('<html')) {
+              codeLines.splice(1, codeLines.length - 2, '...');
+            }
+            const outcomeData = {
+              index,
+              verdict,
+              rule: {
+                ruleID,
+                ruleSummary,
+                scope: '',
+                uri,
+                requirements
+              },
+              target: {
+                type: targetJ.type,
+                tagName: targetJ.name || '',
+                path: target.path(),
+                codeLines: codeLines.map(line => line.length > 99 ? `${line.slice(0, 99)}...` : line)
+              }
+            };
+            const etcTags = [];
+            tags.forEach(tag => {
+              if (tag.type === 'scope') {
+                outcomeData.rule.scope = tag.scope;
+              }
+              else {
+                etcTags.push(tag);
+              }
+            });
+            if (etcTags.length) {
+              outcomeData.etcTags = etcTags;
+            }
+            data.push(outcomeData);
+          }
+        }
+      });
+    }
+  });
+  return {result: data};
+};

package/tests/axe.js

@@ -0,0 +1,109 @@
+/*
+  axe
+  This test implements the axe-core ruleset for accessibility.
+
+  The rules argument defaults to all rules; otherwise, specify an array of rule names.
+  Experimental, needs-review, and best-practice rules are ignored.
+*/
+// IMPORTS
+const {injectAxe, getViolations} = require('axe-playwright');
+// FUNCTIONS
+// Conducts and reports an Axe test.
+exports.reporter = async (page, withItems, rules = []) => {
+  // Initialize the report.
+  const data = {};
+  // Inject axe-core into the page.
+  await injectAxe(page)
+  .catch(error => {
+    console.log(`ERROR: Axe injection failed (${error.message})`);
+    data.result = 'ERROR: axe injection failed';
+  });
+  // If the injection succeeded:
+  if (! data.result) {
+    // Get the data on the elements violating the specified axe-core rules.
+    const axeOptions = {};
+    if (rules.length) {
+      axeOptions.runOnly = rules;
+    }
+    const axeReport = await getViolations(page, null, axeOptions)
+    .catch(error => {
+      console.log(`ERROR: Axe failed (${error.message}'`);
+      return '';
+    });
+    // If the test succeeded:
+    if (Array.isArray(axeReport)) {
+      // Initialize a report.
+      data.warnings = 0;
+      data.violations = {
+        minor: 0,
+        moderate: 0,
+        serious: 0,
+        critical: 0
+      };
+      if (withItems) {
+        data.items = [];
+      }
+      // If there were any violations:
+      if (axeReport.length) {
+        // FUNCTION DEFINITIONS START
+        // Compacts a check violation.
+        const compactCheck = checkObj => {
+          return {
+            check: checkObj.id,
+            description: checkObj.message,
+            impact: checkObj.impact
+          };
+        };
+        // Compacts a violating element.
+        const compactViolator = elObj => {
+          const out = {
+            selector: elObj.target[0],
+            impact: elObj.impact
+          };
+          if (elObj.any && elObj.any.length) {
+            out['must pass any of'] = elObj.any.map(checkObj => compactCheck(checkObj));
+          }
+          if (elObj.none && elObj.none.length) {
+            out['must pass all of'] = elObj.none.map(checkObj => compactCheck(checkObj));
+          }
+          return out;
+        };
+        // Compacts a violated rule.
+        const compactRule = rule => {
+          const out = {
+            rule: rule.id,
+            description: rule.description,
+            impact: rule.impact,
+            elements: {}
+          };
+          if (rule.nodes && rule.nodes.length) {
+            out.elements = rule.nodes.map(el => compactViolator(el));
+          }
+          return out;
+        };
+        // FUNCTION DEFINITIONS END
+        // For each rule violated:
+        axeReport.forEach(rule => {
+          // For each element violating the rule:
+          rule.nodes.forEach(element => {
+            // Increment the element count of the impact of its violation.
+            data.violations[element.impact]++;
+          });
+          // If details are required:
+          if (withItems) {
+            // Add it to the report.
+            data.items.push(compactRule(rule));
+          }
+        });
+      }
+    }
+    // Otherwise, i.e. if the test failed:
+    else {
+      // Report this.
+      data.error = 'ERROR: axe failed';
+      console.log('ERROR: axe failed');
+    }
+  }
+  // Return the result.
+  return {result: data};
+};

package/tests/bulk.js

@@ -0,0 +1,21 @@
+/*
+  bulk
+  This test reports the count of visible elements.
+
+  The test assumes that simplicity and compactness, with one page having one purpose,
+  is an accessibility virtue. Users with visual, motor, and cognitive disabilities
+  often have trouble finding what they want or understanding the purpose of a page
+  if the page is cluttered with content.
+*/
+exports.reporter = async page => {
+  const data = {};
+  await page.waitForSelector('body', {timeout: 10000})
+  .catch(error => {
+    console.log(`ERROR (${error.message})`);
+    data.error = 'ERROR: bulk timed out';
+    return {result: data};
+  });
+  const visibleElements = await page.$$('body :visible');
+  data.visibleElements = visibleElements.length;
+  return {result: data};
+};

package/tests/embAc.js

@@ -0,0 +1,36 @@
+/*
+  embAc
+  This test reports interactive elements (links, buttons, inputs, and select lists)
+  contained by links or buttons. Such embedding not only violates the HTML standard,
+  but also complicates user interaction and creates risks of error. It becomes
+  non-obvious what a user will activate with a click.
+*/
+exports.reporter = async (page, withItems) => await page.$$eval(
+  'a a, a button, a input, a select, button a, button button, button input, button select',
+  (bads, withItems) => {
+    // FUNCTION DEFINITION START
+    // Returns a space-minimized copy of a string.
+    const compact = string => string.replace(/[\t\n]/g, '').replace(/\s{2,}/g, ' ').trim();
+    // FUNCTION DEFINITION END
+    const totals = {
+      links: 0,
+      buttons: 0,
+      inputs: 0,
+      selects: 0
+    };
+    const items = [];
+    // Total and, if requested, itemize the faulty elements.
+    bads.forEach(bad => {
+      totals[Object.keys(totals)[['A', 'BUTTON', 'INPUT', 'SELECT'].indexOf(bad.tagName)]]++;
+      if (withItems) {
+        items.push(compact(bad.outerHTML));
+      }
+    });
+    // Return the result.
+    const data = {totals};
+    if (withItems) {
+      data.items = items;
+    }
+    return {result: data};
+  }, withItems
+);

package/tests/focAll.js

@@ -0,0 +1,62 @@
+/*
+  focAll
+  This test reports discrepancies between focusable and Tab-focused element counts.
+  The test first counts all the visible focusable (i.e. with tabIndex 0) elements
+  (except counting each group of radio buttons as only one focusable element). Then
+  it repeatedly presses the Tab (or Option-Tab in webkit) key until it has reached
+  all the elements it can and counts those elements. If the latter are more than the
+  former, Tab-key navigation made more focusable elements visible. If the latter are
+  fewer than the former, the page manages focus so as to prevent Tab-key navigation
+  from reaching all focusable elements. Either of these can complicate navigation for
+  users. It may disappoint the expectation that the content will remain stable as they
+  move with the Tab key, or the expectation that they can reach every focusable element
+  (or widget, such as one radio button or tab in each group) merely by pressing the Tab
+  key.
+*/
+exports.reporter = async page => {
+  // Identify the count of visible focusable elements.
+  const tabFocusables = await page.$$eval(
+    'body *:visible',
+    visibles => {
+      const focusables = visibles.filter(visible => visible.tabIndex === 0);
+      // Count as focusable only 1 radio button per group.
+      const radios = focusables.filter(el => el.tagName === 'INPUT' && el.type === 'radio');
+      const radioNames = new Set(radios.map(radio => radio.name));
+      return focusables.length - radios.length + radioNames.size;
+    }
+  );
+  /*
+    Repeatedly perform a Tab or (in webkit) Opt-Tab keypress and count the focused elements.
+    Asumptions:
+      0. No page has more than 2000 focusable elements.
+      1. No shadow root that reports itself as the active element has more than 100 focusable
+        descendants.
+  */
+  let tabFocused = 0;
+  let refocused = 0;
+  while (refocused < 100 && tabFocused < 2000) {
+    await page.keyboard.press(page.browserTypeName === 'webkit' ? 'Alt+Tab' : 'Tab');
+    const isNewFocus = await page.evaluate(() => {
+      const focus = document.activeElement;
+      if (focus === null || focus.tagName === 'BODY' || focus.dataset.autotestfocused) {
+        return false;
+      }
+      else {
+        focus.dataset.autotestfocused = 'true';
+        return true;
+      }
+    });
+    if (isNewFocus) {
+      tabFocused++;
+    }
+    else {
+      refocused++;
+    }
+  }
+  // Return the result.
+  return {result: {
+    tabFocusables,
+    tabFocused,
+    discrepancy: tabFocused - tabFocusables
+  }};
+};