testaro 62.0.4 → 62.1.0

package/aceconfig.js

@@ -22,5 +22,6 @@ module.exports = {
     'recommendation'
   ],
   cacheFolder: tmpDir,
-  outputFolder: tmpDir
+  outputFolder: tmpDir,
+  puppeteerArgs: ['--no-sandbox', '--disable-setuid-sandbox']
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "62.0.4",
+  "version": "62.1.0",
   "description": "Run 1000 web accessibility tests from 11 tools and get a standardized report",
   "main": "index.js",
   "scripts": {

package/testaro/hovInd.js

@@ -123,7 +123,7 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
   // Get a sample of the triggers.
   const sampleIndexes = getSample(locsAll, sampleSize);
   const sample = locsAll.filter((loc, index) => sampleIndexes.includes(index));
-  // For each trigger in the sample:
+  // For each trigger:
   for (const loc of sample) {
     try {
       // Get its style properties.
@@ -135,7 +135,7 @@ exports.reporter = async (page, withItems, sampleSize = 20) => {
       // Blur it.
       await loc.blur({timeout: 500});
       // If blurring succeeds, try to hover over it.
-      await loc.hover({timeout: 500});
+      await loc.hover({timeout: 600});
       // If hovering succeeds, get its style properties.
       const hovStyles = await getHoverStyles(loc);
       // If all 3 style declarations belong to the same element:

package/testaro/hover.js

@@ -17,6 +17,8 @@
 
 // Module to perform common operations.
 const {getBasicResult, getVisibleCountChange} = require('../procs/testaro');
+// Module to perform Playwright operations.
+const playwright = require('playwright');
 
 // FUNCTIONS
 
@@ -47,7 +49,9 @@ exports.reporter = async (page, withItems) => {
   ].join(', '));
   const allLocs = await candidateLocs.all();
   const violations = [];
-  const data = {};
+  const data = {
+    hoverableCount: allLocs.length
+  };
   // For each locator:
   for (const loc of allLocs) {
     // Get the XPath of the element referenced by the locator.
@@ -63,7 +67,7 @@ exports.reporter = async (page, withItems) => {
     // Get a locator for the observation root.
     const rootLoc = page.locator(`xpath=${xPath}`);
     const loc0 = await rootLoc.locator('*:visible');
-    // Get a count of the visible elements in the observation tree.
+    // Get a pre-hover count of the visible elements in the observation tree.
     const elementCount0 = await loc0.count();
     try {
       // Hover over the element.
@@ -86,9 +90,14 @@ exports.reporter = async (page, withItems) => {
     }
     // If hovering throws an error:
     catch(error) {
-      // Report that the test was prevented.
+      // If the error is a timeout:
+      if (error instanceof playwright.errors.TimeoutError) {
+        // Skip the locator.
+        continue;
+      }
+      // Otherwise, i.e. if the error is not a timeout, report this and quit.
       data.prevented = true;
-      data.error = `ERROR hovering over an element (${error.message})`;
+      data.error = `ERROR hovering over an element (${error.message.slice(0, 200)})`;
       break;
     }
   }

package/tests/qualWeb.js

@@ -42,9 +42,24 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
     timeout: timeLimit * 1000,
     monitor: false
   };
+  try {
     // Start the QualWeb core engine.
-  await qualWeb.start(clusterOptions, {headless: true});
-  // Specify the invariant test options.
+    await qualWeb.start(clusterOptions, {
+      headless: true,
+      args: ['--no-sandbox', '--disable-setuid-sandbox']
+    });
+  }
+  // If the start fails:
+  catch(error) {
+    return {
+      data: {
+        prevented: true,
+        error: `Core engine start failed (${error.message})`
+      },
+      result
+    };
+  }
+  // Otherwise, i.e. if the start succeeds, specify the invariant test options.
   const qualWebOptions = {
     log: {
       console: false,
@@ -128,9 +143,22 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
       qualWebOptions.modules.push(bpModule);
       qualWebOptions.execute.bp = true;
     }
-    // Get the report.
-    let actReports = await qualWeb.evaluate(qualWebOptions);
-    result = actReports[withNewContent ? qualWebOptions.url : 'customHtml'];
+    let qwReport;
+    try {
+      // Get the report.
+      qwReport = await qualWeb.evaluate(qualWebOptions);
+    }
+    catch(error) {
+      return {
+        data: {
+          prevented: true,
+          error: `qualWeb evaluation failed (${error.message})`
+        },
+        result
+      };
+    }
+    // Otherwise, i.e. if the evaluation succeeded, get the report.
+    result = qwReport[withNewContent ? qualWebOptions.url : 'customHtml'];
     // If it contains a copy of the DOM:
     if (result && result.system && result.system.page && result.system.page.dom) {
       // Delete the copy.
@@ -184,24 +212,24 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
               }
               else {
                 data.prevented = true;
-                data.error = 'ERROR: No assertions';
+                data.error = 'No assertions';
               }
             }
             else {
               data.prevented = true;
-              data.error = `ERROR: No ${section} section`;
+              data.error = `No ${section} section`;
             }
           }
         }
       }
       else {
         data.prevented = true;
-        data.error = 'ERROR: No modules';
+        data.error = 'No modules';
       }
     }
     else {
       data.prevented = true;
-      data.error = 'ERROR: No DOM';
+      data.error = 'No DOM';
     }
     // Stop the QualWeb core engine.
     await qualWeb.stop();
@@ -210,20 +238,14 @@ exports.reporter = async (page, report, actIndex, timeLimit) => {
       JSON.stringify(result);
     }
     catch(error) {
-      const message = `ERROR: qualWeb result cannot be made JSON (${error.message})`;
       data.prevented = true;
-      data.error = message;
-    };
+      data.error = `qualWeb result cannot be made JSON (${error.message})`;
+    }
   }
   catch(error) {
-    const message = error.message.slice(0, 200);
     data.prevented = true;
-    data.error = message;
-    result = {
-      prevented: true,
-      error: message
-    };
-  };
+    data.error = `qualWeb evaluation failed (${error.message})`;
+  }
   return {
     data,
     result