testaro 62.0.3 → 62.1.0

package/aceconfig.js

@@ -22,5 +22,6 @@ module.exports = {
     'recommendation'
   ],
   cacheFolder: tmpDir,
-  outputFolder: tmpDir
+  outputFolder: tmpDir,
+  puppeteerArgs: ['--no-sandbox', '--disable-setuid-sandbox']
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "62.0.3",
+  "version": "62.1.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/testaro/hovInd.js

@@ -123,7 +123,7 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
   // Get a sample of the triggers.
   const sampleIndexes = getSample(locsAll, sampleSize);
   const sample = locsAll.filter((loc, index) => sampleIndexes.includes(index));
-  // For each trigger in the sample:
+  // For each trigger:
   for (const loc of sample) {
     try {
       // Get its style properties.
@@ -135,7 +135,7 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
       // Blur it.
       await loc.blur({timeout: 500});
       // If blurring succeeds, try to hover over it.
-      await loc.hover({timeout: 500});
+      await loc.hover({timeout: 600});
       // If hovering succeeds, get its style properties.
       const hovStyles = await getHoverStyles(loc);
       // If all 3 style declarations belong to the same element:

package/testaro/hover.js

@@ -17,6 +17,8 @@
 
 // Module to perform common operations.
 const {getBasicResult, getVisibleCountChange} = require('../procs/testaro');
+// Module to perform Playwright operations.
+const playwright = require('playwright');
 
 // FUNCTIONS
 
@@ -47,7 +49,9 @@ exports.reporter = async (page, withItems) => {
   ].join(', '));
   const allLocs = await candidateLocs.all();
   const violations = [];
-  const data = {};
+  const data = {
+    hoverableCount: allLocs.length
+  };
   // For each locator:
   for (const loc of allLocs) {
     // Get the XPath of the element referenced by the locator.
@@ -63,7 +67,7 @@ exports.reporter = async (page, withItems) => {
     // Get a locator for the observation root.
     const rootLoc = page.locator(`xpath=${xPath}`);
     const loc0 = await rootLoc.locator('*:visible');
-    // Get a count of the visible elements in the observation tree.
+    // Get a pre-hover count of the visible elements in the observation tree.
     const elementCount0 = await loc0.count();
     try {
       // Hover over the element.
@@ -86,9 +90,14 @@ exports.reporter = async (page, withItems) => {
     }
     // If hovering throws an error:
     catch(error) {
-      // Report that the test was prevented.
+      // If the error is a timeout:
+      if (error instanceof playwright.errors.TimeoutError) {
+        // Skip the locator.
+        continue;
+      }
+      // Otherwise, i.e. if the error is not a timeout, report this and quit.
       data.prevented = true;
-      data.error = `ERROR hovering over an element (${error.message})`;
+      data.error = `ERROR hovering over an element (${error.message.slice(0, 200)})`;
       break;
     }
   }

package/tests/ibm.js

@@ -22,7 +22,6 @@
 
 // IMPORTS
 
-const fs = require('fs').promises;
 // Scanner. Importing and executing 'close' crashed the Node process.
 const accessibilityChecker = require('accessibility-checker');
 const {getCompliance} = accessibilityChecker;
@@ -30,7 +29,7 @@ const {getCompliance} = accessibilityChecker;
 // FUNCTIONS
 
 // Runs the IBM test and returns the result.
-const run = async (content, timeLimit) => {
+const run = async content => {
   const nowLabel = (new Date()).toISOString().slice(0, 19);
   try {
     const ibmReport = await getCompliance(content, nowLabel);
@@ -66,13 +65,13 @@ const limitRuleTotals = (actReport, rules) => {
   }
 };
 // Trims an IBM report.
-const trimActReport = (data, actReport, withItems, rules) => {
+const trimActReport = (actReport, withItems, rules) => {
   // If the act report includes a summary:
   if (actReport && actReport.summary) {
     // Remove excluded rules from the act report.
     limitRuleTotals(actReport, rules);
-    // If the act report includes totals:
     const totals = actReport.summary.counts;
+    // If the act report includes totals:
     if (totals) {
       // If itemization is required:
       if (withItems) {
@@ -101,115 +100,67 @@ const trimActReport = (data, actReport, withItems, rules) => {
     }
     // Otherwise, i.e. if it excludes totals:
     else {
-      // Report this.
-      const error = 'ERROR: No totals reported';
-      console.log(error);
-      data.prevented = true;
-      data.error = error;
-      // Return an empty act report.
+      // Return an act report with this error.
       return {
-        totals: {},
-        items: []
+        totals: null,
+        items: [],
+        error: 'No totals reported'
       };
     }
   }
   // Otherwise, i.e. if it excludes a summary:
   else {
-    // Report this.
-    const error = 'ERROR: No summary reported';
-    console.log(error);
-    data.prevented = true;
-    data.error = error;
-    // Return an empty act report.
+    // Return an act report with this error.
     return {
-      totals: {},
-      items: []
-    };
-  }
-};
-// Performs the IBM tests and returns an act report.
-const doTest = async (content, withItems, timeLimit, rules) => {
-  // Conduct the tests.
-  const data = {};
-  try {
-    const runReport = await run(content, timeLimit);
-    // If there were results:
-    if (runReport.report) {
-      // Return a trimmed act report.
-      const {report} = runReport;
-      const trimmedReport = trimActReport(data, report, withItems, rules);
-      return {
-        data,
-        result: trimmedReport
-      };
-    }
-    // Otherwise, i.e. if there were no results:
-    else {
-      // Return this.
-      return {
-        data: runReport,
-        result: {}
-      }
-    }
-  }
-  catch(error) {
-    const message = `Act failed (${error.message.slice(0, 200)})`;
-    console.log(message);
-    data.prevented = true;
-    data.error = message;
-    return {
-      data,
-      result: {}
+      totals: null,
+      items: [],
+      error: 'No summary reported'
     };
   }
 };
 // Conducts and reports the IBM Equal Access tests.
-exports.reporter = async (page, report, actIndex, timeLimit) => {
+exports.reporter = async (page, report, actIndex) => {
   const act = report.acts[actIndex];
   const {withItems, withNewContent, rules} = act;
   const contentType = withNewContent ? 'new' : 'existing';
   try {
     const typeContent = contentType === 'existing' ? await page.content() : page.url();
-    // Perform the tests.
-    const actReport = await doTest(typeContent, withItems, timeLimit, rules);
-    // If the testing was finished on time:
-    if (actReport !== 'timedOut') {
-      const {data, result} = actReport;
-      // If the act was prevented:
-      if (data && data.prevented) {
-        // Report this.
-        const message = 'ERROR: Act was prevented';
-        data.error ??= message;
-        // Return the failure.
+    // Conduct the tests.
+    const runReport = await run(typeContent);
+    const {report} = runReport;
+    // If there were results:
+    if (report) {
+      // Trim them.
+      const trimmedReport = trimActReport(report, withItems, rules);
+      const {error} = trimmedReport;
+      // If the report was not trimmable:
+      if (error) {
+        // Return an act report with this error.
         return {
-          data,
+          data: {
+            prevented: true,
+            error
+          },
           result: {}
-        };
-      }
-      // Otherwise, i.e. if the act was not prevented:
-      else {
-        // Return the result.
-        return {
-          data,
-          result
-        };
+        }
       }
-    }
-    // Otherwise, i.e. if the testing timed out:
-    else {
-      // Report this.
+      // Otherwise, i.e. if the report was trimmable, return it.
       return {
-        data: {
-          prevented: true,
-          error: message
-        },
-        result: {}
+        data: {},
+        result: trimmedReport
       };
     }
+    // Otherwise, i.e. if there was only an error, return it in an act report.
+    return {
+      data: runReport,
+      result: {}
+    }
   }
+  // If an error occurred:
   catch(error) {
-    const message = `Act crashed (${error.message.slice(0, 200)})`;
-    console.log(`ERROR: ${message}`);
+    const message = `Act failed (${error.message.slice(0, 200)})`;
+    console.log(message);
+    // Return it in an act report.
     return {
       data: {
         prevented: true,

package/tests/qualWeb.js

@@ -42,9 +42,24 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
     timeout: timeLimit * 1000,
     monitor: false
   };
+  try {
     // Start the QualWeb core engine.
-  await qualWeb.start(clusterOptions, {headless: true});
-  // Specify the invariant test options.
+    await qualWeb.start(clusterOptions, {
+      headless: true,
+      args: ['--no-sandbox', '--disable-setuid-sandbox']
+    });
+  }
+  // If the start fails:
+  catch(error) {
+    return {
+      data: {
+        prevented: true,
+        error: `Core engine start failed (${error.message})`
+      },
+      result
+    };
+  }
+  // Otherwise, i.e. if the start succeeds, specify the invariant test options.
   const qualWebOptions = {
     log: {
       console: false,
@@ -128,9 +143,22 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
       qualWebOptions.modules.push(bpModule);
       qualWebOptions.execute.bp = true;
     }
-    // Get the report.
-    let actReports = await qualWeb.evaluate(qualWebOptions);
-    result = actReports[withNewContent ? qualWebOptions.url : 'customHtml'];
+    let qwReport;
+    try {
+      // Get the report.
+      qwReport = await qualWeb.evaluate(qualWebOptions);
+    }
+    catch(error) {
+      return {
+        data: {
+          prevented: true,
+          error: `qualWeb evaluation failed (${error.message})`
+        },
+        result
+      };
+    }
+    // Otherwise, i.e. if the evaluation succeeded, get the report.
+    result = qwReport[withNewContent ? qualWebOptions.url : 'customHtml'];
     // If it contains a copy of the DOM:
     if (result && result.system && result.system.page && result.system.page.dom) {
       // Delete the copy.
@@ -184,24 +212,24 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
               }
               else {
                 data.prevented = true;
-                data.error = 'ERROR: No assertions';
+                data.error = 'No assertions';
               }
             }
             else {
               data.prevented = true;
-              data.error = `ERROR: No ${section} section`;
+              data.error = `No ${section} section`;
             }
           }
         }
       }
       else {
         data.prevented = true;
-        data.error = 'ERROR: No modules';
+        data.error = 'No modules';
       }
     }
     else {
       data.prevented = true;
-      data.error = 'ERROR: No DOM';
+      data.error = 'No DOM';
     }
     // Stop the QualWeb core engine.
     await qualWeb.stop();
@@ -210,20 +238,14 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
       JSON.stringify(result);
     }
     catch(error) {
-      const message = `ERROR: qualWeb result cannot be made JSON (${error.message})`;
       data.prevented = true;
-      data.error = message;
-    };
+      data.error = `qualWeb result cannot be made JSON (${error.message})`;
+    }
   }
   catch(error) {
-    const message = error.message.slice(0, 200);
     data.prevented = true;
-    data.error = message;
-    result = {
-      prevented: true,
-      error: message
-    };
-  };
+    data.error = `qualWeb evaluation failed (${error.message})`;
+  }
   return {
     data,
     result