testaro 78.4.0 → 78.6.0

package/tests/qualWeb.ts

@@ -15,27 +15,13 @@ import type {Page} from 'playwright';
 import {getAttributeXPath, getXPathCatalogIndex} from '../procs/xPath';
 import {getStandardResult, addInstance} from '../procs/standard';
 import type {Act, Report, StandardInstance, StandardResult} from '../types';
-/*
-  The @qualweb packages declare their types only in package-exports maps, which
-  this project's node10 module resolution cannot read, so the imports stay
-  requires and are untyped.
-*/
-const {QualWeb} = require('@qualweb/core');
-const {ACTRules} = require('@qualweb/act-rules');
-const {WCAGTechniques} = require('@qualweb/wcag-techniques');
-const {BestPractices} = require('@qualweb/best-practices');
-const {PlaywrightDriver} = require('@qualweb/playwright-driver');
-import {qualWebDefaults} from '../procs/config';
+import {readFileSync} from 'fs';
 
 // TYPES
 
 // The qualWeb-act properties this reporter consumes.
 interface QualWebAct extends Act {
   rules?: string[];
-  // Whether the tool's browser runs stealth evasions and blocks ads and trackers (both default
-  // to true, or to QUALWEB_STEALTH and QUALWEB_ADBLOCK when set).
-  stealth?: boolean;
-  adBlock?: boolean;
 }
 // The section names of a QualWeb report.
 type QwSection = 'act-rules' | 'wcag-techniques' | 'best-practices';
@@ -61,87 +47,39 @@ interface QwRuleAssertions {
 interface QwModuleReport {
   assertions?: Record<string, QwRuleAssertions>;
 }
-// The native result: the customHtml QualWeb report.
+// The native result: the module reports, keyed by section, as in QualWeb's own report.
 interface QwNativeResult {
-  system?: {
-    page?: {
-      dom?: unknown;
-    };
-  };
+  system?: {url?: string; evaluation?: string};
   modules?: Record<string, QwModuleReport>;
 }
-// The options of a QualWeb evaluation.
-interface QualWebOptions {
-  log: {console: boolean; file: boolean};
-  crawlOptions: {
-    maxDepth: number;
-    maxUrls: number;
-    timeout: number;
-    maxParallelCrawls: number;
-    logging: boolean;
-  };
-  execute: {counter: boolean; act?: boolean; wcag?: boolean; bp?: boolean};
-  modules: unknown[];
-  html?: string;
-  'act-rules'?: {rules?: string[]; levels?: string[]; principles?: string[]};
-  'wcag-techniques'?: {techniques?: string[]; levels?: string[]; principles?: string[]};
-  'best-practices'?: {bestPractices?: string[]};
+// The options of one section's runner (QualWeb's EvaluationModule options).
+interface QwSectionOptions {
+  include?: string[];
+  exclude?: string[];
+  levels?: string[];
+  principles?: string[];
 }
 
 // CONSTANTS
 
 /*
-  QualWeb core engine with Playwright as driver, one instance per combination of options,
-  created on first use. The driver's constructor options are {launcher, launchOptions,
-  browser}, so its former {adBlock, stealth} options were ignored; stealth is provided by
-  launching through playwright-extra with the stealth plugin, and ad and tracker blocking by
-  the Ghostery blocker enabled on each page before it loads (QualWeb's beforePageLoad hook).
-  The blocker's filter lists are cached in the temporary directory.
+  QualWeb runs in the page under test, as @qualweb/core runs it in its own browser: the
+  page-side bundles first, then one bundle per section. It used to be given page.content() and
+  run by @qualweb/core, which loads that HTML with setContent() into a blank page — so relative
+  stylesheet links resolved nowhere, the page it tested was unstyled, and every CSS-dependent
+  rule (contrast, visibility, text spacing…) judged a page the user never sees. The bundles are
+  evaluated (not added as script tags), so the page's Content Security Policy does not apply.
 */
-interface QualWebEngine {
-  start: (options: unknown) => Promise<void>;
-  evaluate: (options: unknown) => Promise<{customHtml: QwNativeResult}>;
-  stop: () => Promise<void>;
-  use: (plugin: unknown) => void;
-}
-const qualWebInstances: Record<string, QualWebEngine> = {};
-const getQualWeb = (options: {stealth: boolean; adBlock: boolean}): QualWebEngine => {
-  const key = `${options.stealth}:${options.adBlock}`;
-  if (! qualWebInstances[key]) {
-    let launcher;
-    if (options.stealth) {
-      const {chromium} = require('playwright-extra');
-      chromium.use(require('puppeteer-extra-plugin-stealth')());
-      launcher = chromium;
-    }
-    const qualWeb: QualWebEngine = new QualWeb(undefined, new PlaywrightDriver(launcher ? {launcher} : {}));
-    if (options.adBlock) {
-      const {PlaywrightBlocker} = require('@ghostery/adblocker-playwright');
-      let blockerPromise: Promise<unknown> | null = null;
-      qualWeb.use({
-        async beforePageLoad(page: {nativePage: unknown}) {
-          blockerPromise ??= PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch, {
-            path: require('path').join(require('os').tmpdir(), 'qualweb-adblocker-engine.bin'),
-            read: require('fs').promises.readFile,
-            write: require('fs').promises.writeFile
-          }).catch((error: Error) => {
-            console.log(`qualWeb: ad blocker unavailable, continuing without it (${error.message})`);
-            return null;
-          });
-          const blocker = await blockerPromise as {enableBlockingInPage: (page: unknown) => Promise<void>} | null;
-          if (blocker) {
-            await blocker.enableBlockingInPage(page.nativePage);
-          }
-        }
-      });
-    }
-    qualWebInstances[key] = qualWeb;
-  }
-  return qualWebInstances[key];
+const readBundle = (name: string): string => readFileSync(require.resolve(name), 'utf8');
+let pageBundles: string[] | null = null;
+const sectionBundles: Partial<Record<QwSection, string>> = {};
+const sectionPackages: Record<QwSection, string> = {
+  'act-rules': '@qualweb/act-rules',
+  'wcag-techniques': '@qualweb/wcag-techniques',
+  'best-practices': '@qualweb/best-practices'
 };
-const actRulesModule = new ACTRules({});
-const wcagModule = new WCAGTechniques({});
-const bpModule = new BestPractices({});
+// Rules a section may exclude after they crash before the section is given up on.
+const MAX_EXCLUDED = 5;
 // Mapping of QualWeb module violation types to ordinal severities.
 const ordinalSeverities: Record<QwSection, Record<string, StandardInstance['ordinalSeverity']>> = {
   'act-rules': {
@@ -168,20 +106,10 @@ const ordinalSeverities: Record<QwSection, Record<string, StandardInstance['ordi
 
 // Conducts and reports the QualWeb tests.
 export const reporter = async (page: Page, report: Report, actIndex: number, timeLimit: number) => {
-  const qualWebAct = report.acts[actIndex] as QualWebAct;
-  const qualWeb = getQualWeb({
-    stealth: qualWebAct.stealth ?? qualWebDefaults.stealth,
-    adBlock: qualWebAct.adBlock ?? qualWebDefaults.adBlock
-  });
   const act = report.acts[actIndex] as QualWebAct;
   const {rules} = act;
-  const clusterOptions = {
-    maxConcurrency: 1,
-    timeout: timeLimit * 1000,
-    monitor: false
-  };
   // Initialize the act report.
-  const data: {prevented?: boolean; error?: string} = {};
+  const data: {prevented?: boolean; error?: string; rulePreventions?: Record<string, string>} = {};
   const result: {nativeResult: QwNativeResult; standardResult: StandardResult} = {
     nativeResult: {},
     standardResult: {}
@@ -192,209 +120,210 @@ export const reporter = async (page: Page, report: Report, actIndex: number, tim
     // Initialize the standard result.
     result.standardResult = getStandardResult();
   }
-  try {
-    // Start the QualWeb core engine, which launches a Playwright browser.
-    await qualWeb.start(clusterOptions);
-  }
-  // If the start fails:
-  catch(error) {
+  /*
+    QualWeb tests HTML documents only: on anything else (e.g. an SVG image) its rules report
+    false results such as a missing lang on the html element.
+  */
+  const rootNamespace = await page.evaluate(() => document.documentElement?.namespaceURI ?? null)
+  .catch(() => null);
+  // If the document is not HTML:
+  if (rootNamespace !== 'http://www.w3.org/1999/xhtml') {
+    // Report this.
     return {
       data: {
         prevented: true,
-        error: `Core engine start failed (${(error as Error).message})`
+        error: `qualWeb tests HTML documents only (root namespace ${rootNamespace})`
       },
       result
     };
   }
-  // Otherwise, i.e. if the start succeeds, specify the invariant test options.
-  const qualWebOptions: QualWebOptions = {
-    log: {
-      console: false,
-      file: false
-    },
-    crawlOptions: {
-      maxDepth: 0,
-      maxUrls: 1,
-      timeout: timeLimit * 1000,
-      maxParallelCrawls: 1,
-      logging: true
-    },
-    execute: {
-      counter: true
-    },
-    modules: []
+  // Specify, per section, whether and what to test (a rules spec of 'act:', 'wcag:' or 'best:'
+  // alone skips that section; with numbers, it tests only those rules).
+  const allAAA = {
+    levels: ['A', 'AA', 'AAA'],
+    principles: ['Perceivable', 'Operable', 'Understandable', 'Robust']
   };
-  try {
-    // Provide the page content, including the data-xpath attributes.
-    qualWebOptions.html = await page.content();
-    // Specify which rules to test for, adding a custom execute property for report processing.
-    const actSpec = rules ? rules.find(typeRules => typeRules.startsWith('act:')) : null;
-    const wcagSpec = rules ? rules.find(typeRules => typeRules.startsWith('wcag:')) : null;
-    const bestSpec = rules ? rules.find(typeRules => typeRules.startsWith('best:')) : null;
-    if (actSpec) {
-      if (actSpec === 'act:') {
-        qualWebOptions.execute.act = false;
-      }
-      else {
-        const actRules = actSpec.slice(4).split(',').map(num => `QW-ACT-R${num}`);
-        qualWebOptions['act-rules'] = {rules: actRules};
-        qualWebOptions.modules.push(actRulesModule);
-        qualWebOptions.execute.act = true;
-      }
+  const sectionSpec = (prefix: string, codePrefix: string, all: QwSectionOptions): QwSectionOptions | null => {
+    const spec = rules?.find(typeRules => typeRules.startsWith(prefix));
+    if (spec === undefined) {
+      return all;
     }
-    else {
-      qualWebOptions['act-rules'] = {
-        levels: ['A', 'AA', 'AAA'],
-        principles: ['Perceivable', 'Operable', 'Understandable', 'Robust']
-      };
-      qualWebOptions.modules.push(actRulesModule);
-      qualWebOptions.execute.act = true;
+    if (spec === prefix) {
+      return null;
     }
-    if (wcagSpec) {
-      if (wcagSpec === 'wcag:') {
-        qualWebOptions.execute.wcag = false;
-      }
-      else {
-        const wcagTechniques = wcagSpec.slice(5).split(',').map(num => `QW-WCAG-T${num}`);
-        qualWebOptions['wcag-techniques'] = {techniques: wcagTechniques};
-        qualWebOptions.modules.push(wcagModule);
-        qualWebOptions.execute.wcag = true;
-      }
+    return {include: spec.slice(prefix.length).split(',').map(num => `${codePrefix}${num}`)};
+  };
+  const sections: Partial<Record<QwSection, QwSectionOptions>> = {};
+  for (const [section, options] of [
+    ['act-rules', sectionSpec('act:', 'QW-ACT-R', allAAA)],
+    ['wcag-techniques', sectionSpec('wcag:', 'QW-WCAG-T', allAAA)],
+    ['best-practices', sectionSpec('best:', 'QW-BP', {})]
+  ] as [QwSection, QwSectionOptions | null][]) {
+    if (options) {
+      sections[section] = options;
     }
-    else {
-      qualWebOptions['wcag-techniques'] = {
-        levels: ['A', 'AA', 'AAA'],
-        principles: ['Perceivable', 'Operable', 'Understandable', 'Robust']
-      };
-      qualWebOptions.modules.push(wcagModule);
-      qualWebOptions.execute.wcag = true;
+  }
+  const deadline = Date.now() + timeLimit * 1000;
+  try {
+    // Load QualWeb into the page.
+    pageBundles ??= ['@qualweb/qw-page', '@qualweb/util', '@qualweb/locale'].map(readBundle);
+    for (const source of pageBundles) {
+      await page.evaluate(source);
     }
-    if (bestSpec) {
-      if (bestSpec === 'best:') {
-        qualWebOptions.execute.bp = false;
+    // The page's own source, which QualWeb's meta-element rules read.
+    const sourceHtml = await page.content();
+    const modules: Record<string, QwModuleReport> = {};
+    const rulePreventions: Record<string, string> = {};
+    // For each section to be tested:
+    for (const [section, options] of Object.entries(sections) as [QwSection, QwSectionOptions][]) {
+      if (Date.now() > deadline) {
+        throw new Error(`time limit of ${timeLimit}s reached before ${section}`);
       }
-      else {
-        const bestPractices = bestSpec.slice(5).split(',').map(num => `QW-BP${num}`);
-        qualWebOptions['best-practices'] = {bestPractices};
-        qualWebOptions.modules.push(bpModule);
-        qualWebOptions.execute.bp = true;
+      sectionBundles[section] ??= readBundle(sectionPackages[section]);
+      await page.evaluate(sectionBundles[section]!);
+      /*
+        One rule that throws takes its whole section down with it (e.g. QW-ACT-R76 in 0.8.5 on
+        lab(), oklab(), color() or color-mix() colours, qualweb/qualweb#352), so a rule named in
+        the stack trace of a crash is excluded and the section rerun; a crash that names no rule
+        prevents only its section.
+      */
+      const outcome = await page.evaluate(({section, options, sourceHtml, maxExcluded}) => {
+        // Only this section's bundle has been loaded, so name only its runner.
+        const Runner = section === 'act-rules'
+          // @ts-expect-error: defined by the section bundle evaluated above.
+          ? ACTRulesRunner
+          : section === 'wcag-techniques'
+            // @ts-expect-error: defined by the section bundle evaluated above.
+            ? WCAGTechniquesRunner
+            // @ts-expect-error: defined by the section bundle evaluated above.
+            : BestPracticesRunner;
+        const preventions: Record<string, string> = {};
+        const exclude: string[] = [];
+        for (;;) {
+          try {
+            const opts = exclude.length ? {...options, exclude: [...(options.exclude ?? []), ...exclude]} : options;
+            const runner = new Runner(opts, 'en').configure(opts).test({sourceHtml, newTabWasOpen: false});
+            (window as any).__testaroQwRunners = {...(window as any).__testaroQwRunners, [section]: runner};
+            // QualWeb core keeps the act-rules runner here for its special-case pass.
+            if (section === 'act-rules') {
+              (window as any).act = runner;
+            }
+            return {ok: true, preventions};
+          }
+          catch(error) {
+            const message = String((error as Error)?.message ?? error).slice(0, 200);
+            // Rule classes are named QW_ACT_R76, QW_WCAG_T23, QW_BP1.
+            const rule = String((error as Error)?.stack ?? '').match(/QW_(?:ACT_R|WCAG_T|BP)\d+/)?.[0].replace(/_/g, '-');
+            if (! rule || exclude.includes(rule) || exclude.length >= maxExcluded) {
+              preventions[section] = `section failed: ${message}`;
+              return {ok: false, preventions};
+            }
+            preventions[rule] = message;
+            exclude.push(rule);
+          }
+        }
+      }, {section, options, sourceHtml, maxExcluded: MAX_EXCLUDED});
+      Object.assign(rulePreventions, outcome.preventions);
+      if (! outcome.ok) {
+        continue;
       }
+      /*
+        QualWeb core's special case for act-rules: QW-ACT-R40 (zoomed text not clipped) is
+        re-tested at a 640 × 512 viewport, then the viewport is restored.
+      */
+      const include = options.include;
+      if (section === 'act-rules' && (! include || include.includes('QW-ACT-R40'))) {
+        const viewport = page.viewportSize();
+        if (viewport) {
+          try {
+            await page.setViewportSize({width: 640, height: 512});
+            await page.evaluate(() => (window as any).act?.testSpecial?.());
+          }
+          catch(error) {
+            rulePreventions['QW-ACT-R40'] = `special-case pass failed: ${String((error as Error).message).slice(0, 200)}`;
+          }
+          finally {
+            await page.setViewportSize(viewport);
+          }
+        }
+      }
+      modules[section] = await page.evaluate(
+        section => JSON.parse(JSON.stringify((window as any).__testaroQwRunners[section].getReport())),
+        section
+      );
     }
-    else {
-      qualWebOptions['best-practices'] = {};
-      qualWebOptions.modules.push(bpModule);
-      qualWebOptions.execute.bp = true;
-    }
-    let qwReport;
-    try {
-      // Get the report.
-      qwReport = await qualWeb.evaluate(qualWebOptions);
-    }
-    catch(error) {
-      return {
-        data: {
-          prevented: true,
-          error: `qualWeb evaluation failed (${(error as Error).message})`
-        },
-        result
-      };
+    if (Object.keys(rulePreventions).length) {
+      data.rulePreventions = rulePreventions;
     }
-    // Add the report to the result.
-    result.nativeResult = qwReport.customHtml;
-    const {nativeResult, standardResult} = result;
-    // If the report contains, as it should, a copy of the DOM:
-    if (nativeResult?.system?.page?.dom) {
-      // Delete the copy for parsimony.
-      delete nativeResult.system.page.dom;
-      const {modules} = nativeResult;
-      // If the report contains, as it should, a modules property:
-      if (modules) {
-        // For each test section in it:
-        for (const section of ['act-rules', 'wcag-techniques', 'best-practices'] as QwSection[]) {
-          // If testing in the section was specified:
-          if (qualWebOptions[section]) {
-            // If the section exists:
-            if (modules[section]) {
-              const {assertions} = modules[section];
-              // If it contains assertions (test results):
-              if (assertions) {
-                const ruleIDs = Object.keys(assertions);
-                // For each rule:
-                for (const ruleID of ruleIDs) {
-                  const ruleAssertions = assertions[ruleID];
-                  const {metadata} = ruleAssertions;
-                  // If there were any warnings or failures:
-                  if (metadata?.warning || metadata?.failed) {
-                    // Delete nonviolations from the results.
-                    ruleAssertions.results = ruleAssertions.results.filter(
-                      raResult => raResult.verdict !== 'passed'
-                    );
-                    // For each test result:
-                    for (const raResult of ruleAssertions.results) {
-                      const {elements, verdict} = raResult;
-                      // If any violations are reported:
-                      if (elements?.length) {
-                        // For each violating element:
-                        for (const element of elements) {
-                          // Limit the size of its reported excerpt.
-                          if ((element.htmlCode?.length as number) > 2000) {
-                            element.htmlCode = `${element.htmlCode!.slice(0, 2000)} …`;
-                          }
-                          // If standard results are to be reported:
-                          if (standard) {
-                            const what = `[${verdict}] ${raResult.description}`;
-                            const xPath = getAttributeXPath(element.htmlCode);
-                            // Add an instance to the standard result.
-                            addInstance(standardResult, {
-                              ruleID,
-                              what,
-                              ordinalSeverity: ordinalSeverities[section][verdict],
-                              outcome: verdict === 'warning' ? 'cantTell' : 'failed',
-                              catalogIndex: getXPathCatalogIndex(report, xPath)
-                            });
-                          }
-                        };
-                      }
-                    };
+    // Add the module reports to the result.
+    result.nativeResult = {system: {url: page.url(), evaluation: 'in-page'}, modules};
+    const {standardResult} = result;
+    // For each section that was to be tested:
+    for (const section of Object.keys(sections) as QwSection[]) {
+      const moduleReport = modules[section];
+      /*
+        A section that failed as a whole is recorded in rulePreventions; the act is prevented
+        only if no section ran.
+      */
+      if (! moduleReport) {
+        continue;
+      }
+      const {assertions} = moduleReport;
+      // If it contains assertions (test results):
+      if (assertions) {
+        const ruleIDs = Object.keys(assertions);
+        // For each rule:
+        for (const ruleID of ruleIDs) {
+          const ruleAssertions = assertions[ruleID];
+          const {metadata} = ruleAssertions;
+          // If there were any warnings or failures:
+          if (metadata?.warning || metadata?.failed) {
+            // Delete nonviolations from the results.
+            ruleAssertions.results = ruleAssertions.results.filter(
+              raResult => raResult.verdict !== 'passed'
+            );
+            // For each test result:
+            for (const raResult of ruleAssertions.results) {
+              const {elements, verdict} = raResult;
+              // If any violations are reported:
+              if (elements?.length) {
+                // For each violating element:
+                for (const element of elements) {
+                  // Limit the size of its reported excerpt.
+                  if ((element.htmlCode?.length as number) > 2000) {
+                    element.htmlCode = `${element.htmlCode!.slice(0, 2000)} …`;
                   }
-                  // Otherwise, i.e. if there were no warnings or failures:
-                  else {
-                    // Delete the rule.
-                    delete assertions[ruleID];
+                  // If standard results are to be reported:
+                  if (standard) {
+                    const what = `[${verdict}] ${raResult.description}`;
+                    const xPath = getAttributeXPath(element.htmlCode);
+                    // Add an instance to the standard result.
+                    addInstance(standardResult, {
+                      ruleID,
+                      what,
+                      ordinalSeverity: ordinalSeverities[section][verdict],
+                      outcome: verdict === 'warning' ? 'cantTell' : 'failed',
+                      catalogIndex: getXPathCatalogIndex(report, xPath)
+                    });
                   }
                 };
               }
-              // Otherwise, i.e. if it contains no assertions:
-              else {
-                // Report this.
-                data.prevented = true;
-                data.error = 'No assertions';
-              }
-            }
-            // Otherwise, i.e. if the section is missing:
-            else {
-              // Report this.
-              data.prevented = true;
-              data.error = `No ${section} section`;
-            }
+            };
           }
-        }
-      }
-      // Otherwise, i.e. if the report does not contain a modules property:
-      else {
-        // Report this.
-        data.prevented = true;
-        data.error = 'No modules';
+          // Otherwise, i.e. if there were no warnings or failures:
+          else {
+            // Delete the rule.
+            delete assertions[ruleID];
+          }
+        };
       }
     }
-    // Otherwise, i.e. if the report does not contain a copy of the DOM:
-    else {
+    // If no section ran:
+    if (! Object.keys(modules).length) {
       // Report this.
       data.prevented = true;
-      data.error = 'No DOM';
+      data.error = `No qualWeb section ran (${JSON.stringify(rulePreventions).slice(0, 300)})`;
     }
-    // Stop the QualWeb core engine.
-    await qualWeb.stop();
     // Test whether the result is an object.
     try {
       JSON.stringify(result);