testaro 12.3.2 → 12.4.1

package/README.md

@@ -31,7 +31,9 @@ Testaro uses:
 - [Playwright](https://playwright.dev/) to launch browsers, perform user actions in them, and perform tests
 - [pixelmatch](https://www.npmjs.com/package/pixelmatch) to measure motion
 
-Testaro includes some of its own accessibility tests. In addition, it performs the tests of these tools:
+Testaro includes some of its own accessibility tests. Some of them are derived from tests performed by the [BBC Accessibility Standards Checker](https://github.com/bbc/bbc-a11y).
+
+In addition, Testaro performs tests of these tools:
 - [accessibility-checker](https://www.npmjs.com/package/accessibility-checker) (the IBM Equal Access Accessibility Checker)
 - [alfa](https://alfa.siteimprove.com/) (Siteimprove alfa)
 - [axe-playwright](https://www.npmjs.com/package/axe-playwright) (Deque Axe-core)
@@ -42,8 +44,6 @@ Testaro includes some of its own accessibility tests. In addition, it performs t
 - [Tenon](https://tenon.io/documentation/what-tenon-tests.php) (Level Access)
 - [WAVE API](https://wave.webaim.org/api/) (WebAIM WAVE)
 
-Some of the Testaro tests are derived from tests performed by the [BBC Accessibility Standards Checker](https://github.com/bbc/bbc-a11y).
-
 As of this version, the counts of tests of the tools referenced above were:
 - Alfa: 103
 - Axe-core: 138
@@ -54,9 +54,8 @@ As of this version, the counts of tests of the tools referenced above were:
 - QualWeb core: 121
 - Tenon: 180
 - WAVE: 110
-- subtotal: 1327
-- Testaro tests: 24
-- grand total: 1351
+- Testaro: 29
+- total: 1356
 
 ## Quasi-tests
 
@@ -537,6 +536,8 @@ The third item in each array, if there are 3 items in the array, is the criterio
 
 A typical use for an `expect` property is checking the correctness of a Testaro test. Thus, the validation jobs in the `validation/tests/jobs` directory all contain `test` acts with `expect` properties. See the “Validation” section below.
 
+When a `test` act has an `expect` property, the result for that act has an `expectations` property reporting whether the expectations were satisfied. The value of `expectations` is an array of objects, one object per expectation. Each object includes a `property` property identifying the expectation, and a `passed` property with `true` or `false` value reporting whether the expectation was satisfied. If applicable, it also has other properties identifying what was expected and what was actually reported.
+
 ## Execution
 
 ### Introduction

package/actSpecs.js

@@ -187,6 +187,12 @@ exports.actSpecs = {
         rules: [false, 'array', 'areNumbers', 'rule numbers (e.g., 25), if not all']
       }
     ],
+    dupAtt: [
+      'Perform a dupAtt test',
+      {
+        withItems: [true, 'boolean', '', 'itemize']
+      }
+    ],
     elements: [
       'Perform an elements test',
       {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "12.3.2",
+  "version": "12.4.1",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {
@@ -35,6 +35,7 @@
     "accessibility-checker": "*",
     "axe-playwright": "*",
     "dotenv": "*",
+    "node-fetch": "*",
     "pixelmatch": "*",
     "playwright": "*"
   },

package/run.js

@@ -39,6 +39,7 @@ const tests = {
   bulk: 'count of visible elements',
   continuum: 'Level Access Continuum, community edition',
   docType: 'document without a doctype property',
+  dupAtt: 'elements with duplicate attributes',
   elements: 'data on specified elements',
   embAc: 'active elements embedded in links or buttons',
   filter: 'filter styles on elements',
@@ -498,7 +499,7 @@ const isTrue = (object, specs) => {
   }
   // Otherwise, i.e. if the expectation is of a property value:
   else if (specs.length === 3) {
-    // Determine whether the expectation was fulfilled.
+    // Return whether the expectation was fulfilled.
     const relation = specs[1];
     const criterion = specs[2];
     let satisfied;
@@ -518,16 +519,17 @@ const isTrue = (object, specs) => {
       satisfied = actual !== criterion;
     }
     else if (relation === 'i') {
-      satisfied = actual.includes(criterion);
+      satisfied = typeof actual === 'string' && actual.includes(criterion);
     }
     else if (relation === '!i') {
-      satisfied = ! actual.includes(criterion);
+      satisfied = typeof actual === 'string' && ! actual.includes(criterion);
     }
     return [actual, satisfied];
   }
   // Otherwise, i.e. if the specifications are invalid:
   else {
-    //
+    // Return this.
+    return [null, false];
   }
 };
 // Adds a wait error result to an act.

package/tests/attVal.js

@@ -5,13 +5,13 @@
 exports.reporter = async (page, attributeName, areLicit, values, withItems) => {
   // Identify the elements that have the specified attribute with illicit values.
   const badAttributeData = await page.evaluate(
-    args => {
+    async args => {
       const attributeName = args[0];
       // Whether the values are the licit or the illicit ones.
       const areLicit = args[1];
       const values = args[2];
       // Returns the text of an element.
-      const textOf = async (element, limit) => {
+      const textOf = (element, limit) => {
         let text = element.textContent;
         text = text.trim() || element.innerHTML;
         return text.replace(/\s+/sg, ' ').replace(/<>&/g, '').slice(0, limit);

package/tests/dupAtt.js

@@ -0,0 +1,54 @@
+/*
+  dupAtt.js
+  This test reports duplicate attributes in the source of a document.
+*/
+
+// ########## IMPORTS
+
+// Module to make HTTP requests.
+const fetch = require('node-fetch');
+// Module to process files.
+const fs = require('fs/promises');
+
+// ########## FUNCTIONS
+
+// Reports failures.
+exports.reporter = async (page, withItems) => {
+  // Initialize the data.
+  const data = {total: 0};
+  if (withItems) {
+    data.items = [];
+  }
+  // Get the page.
+  const url = page.url();
+  const scheme = url.replace(/:.+/, '');
+  let rawPage;
+  if (scheme === 'file') {
+    const filePath = url.slice(7);
+    rawPage = await fs.readFile(filePath, 'utf8');
+  }
+  else {
+    rawPage = await fetch(url);
+  }
+  // Extract its elements, in a uniform format.
+  const elements = rawPage
+  .match(/<[^/<>]+>/g)
+  .map(element => element.slice(1, -1).trim().replace(/\s*=\s*/g, '='))
+  .map(element => element.replace(/\s+/g, ' '));
+  // For each element:
+  elements.forEach(element => {
+    // Identify its attributes.
+    const attributes = element.split(' ').slice(1).map(attVal => attVal.replace(/=.+/, ''));
+    // If any is duplicated:
+    const attSet = new Set(attributes);
+    if (attSet.size < attributes.length) {
+      // Add this to the data.
+      data.total++;
+      if (withItems) {
+        data.items.push(element);
+      }
+    }
+  });
+  // Return the data.
+  return {result: data};
+};

package/validation/tests/jobs/allHidden.json

@@ -1,5 +1,5 @@
 {
-  "id": "allHiddenVal",
+  "id": "allHidden",
   "what": "validation of allHidden test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/attVal.json

@@ -0,0 +1,60 @@
+{
+  "id": "attVal",
+  "what": "validation of attVal test",
+  "strict": true,
+  "timeLimit": 20,
+  "acts": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "usual browser"
+    },
+    {
+      "type": "url",
+      "which": "__targets__/attVal/good.html",
+      "what": "page with permitted attribute values"
+    },
+    {
+      "type": "test",
+      "which": "attVal",
+      "attributeName": "lang",
+      "areLicit": true,
+      "values": ["en-US", "de-CH"],
+      "what": "attribute values",
+      "withItems": true,
+      "expect": [
+        ["total", "=", 0],
+        ["items.1"]
+      ]
+    },
+    {
+      "type": "url",
+      "which": "__targets__/attVal/bad.html",
+      "what": "page with permitted attribute values"
+    },
+    {
+      "type": "test",
+      "which": "attVal",
+      "attributeName": "lang",
+      "areLicit": false,
+      "values": ["en", "de"],
+      "what": "attribute values",
+      "withItems": true,
+      "expect": [
+        ["total", "=", 2],
+        ["items.0.tagName", "=", "HTML"],
+        ["items.0.attributeValue", "=", "en"],
+        ["items.1.tagName", "=", "SPAN"],
+        ["items.1.attributeValue", "=", "de"],
+        ["items.1.textStart", "i", "Veloparkieren"]
+      ]
+    }
+  ],
+  "sources": {
+    "script": "",
+    "host": {},
+    "requester": ""
+  },
+  "creationTime": "2023-04-19T12:34:00",
+  "timeStamp": "00000"
+}

package/validation/tests/jobs/autocomplete.json

@@ -1,5 +1,5 @@
 {
-  "id": "autocompleteVal",
+  "id": "autocomplete",
   "what": "validation of autocomplete test",
   "strict": true,
   "timeLimit": 20,
@@ -20,7 +20,7 @@
       "what": "autocomplete attributes for name and email",
       "withItems": true,
       "expect": [
-        ["totals", "=", 0],
+        ["total", "=", 0],
         ["items.1"]
       ]
     },
@@ -35,7 +35,7 @@
       "what": "autocomplete attributes for name and email",
       "withItems": true,
       "expect": [
-        ["totals", "=", 4],
+        ["total", "=", 4],
         ["items.1.0", "=", "family-name"],
         ["items.2.1", "=", "Your email address"]
       ]
@@ -46,6 +46,6 @@
     "host": {},
     "requester": ""
   },
-  "creationTime": "2003-04-16T21:06:00",
+  "creationTime": "2023-04-16T21:06:00",
   "timeStamp": "00000"
 }

package/validation/tests/jobs/bulk.json

@@ -1,5 +1,5 @@
 {
-  "id": "bulkVal",
+  "id": "bulk",
   "what": "validation of bulk test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/docType.json

@@ -1,5 +1,5 @@
 {
-  "id": "docTypeVal",
+  "id": "docType",
   "what": "validation of docType test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/dupAtt.json

@@ -0,0 +1,51 @@
+{
+  "id": "dupAtt",
+  "what": "validation of dupAtt test",
+  "strict": true,
+  "timeLimit": 20,
+  "acts": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "usual browser"
+    },
+    {
+      "type": "url",
+      "which": "__targets__/dupAtt/good.html",
+      "what": "page without duplicate attributes"
+    },
+    {
+      "type": "test",
+      "which": "dupAtt",
+      "what": "elements with duplicate attributes",
+      "withItems": true,
+      "expect": [
+        ["total", "=", 0],
+        ["items.1"]
+      ]
+    },
+    {
+      "type": "url",
+      "which": "__targets__/dupAtt/bad.html",
+      "what": "page with duplicate attributes"
+    },
+    {
+      "type": "test",
+      "which": "dupAtt",
+      "what": "elements with duplicate attributes",
+      "withItems": true,
+      "expect": [
+        ["total", "=", 2],
+        ["items.0", "=", "p class=\"narrow\" id=\"daParagraph\" class=\"wide\""],
+        ["items.1", "i", "large"]
+      ]
+    }
+  ],
+  "sources": {
+    "script": "",
+    "host": {},
+    "requester": ""
+  },
+  "creationTime": "2023-04-18T11:02:00",
+  "timeStamp": "00000"
+}

package/validation/tests/jobs/elements.json

@@ -1,5 +1,5 @@
 {
-  "id": "elementsVal",
+  "id": "elements",
   "what": "validation of elements test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/embAc.json

@@ -1,5 +1,5 @@
 {
-  "id": "embAcVal",
+  "id": "embAc",
   "what": "validation of embAc test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/filter.json

@@ -1,5 +1,5 @@
 {
-  "id": "filterVal",
+  "id": "filter",
   "what": "validation of filter test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/focAll.json

@@ -1,5 +1,5 @@
 {
-  "id": "focAllVal",
+  "id": "focAll",
   "what": "validation of focAll test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/focInd.json

@@ -1,5 +1,5 @@
 {
-  "id": "focIndVal",
+  "id": "focInd",
   "what": "validation of focInd test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/focOp.json

@@ -1,5 +1,5 @@
 {
-  "id": "focOpVal",
+  "id": "focOp",
   "what": "validation of focOp test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/focVis.json

@@ -1,5 +1,5 @@
 {
-  "id": "focVisVal",
+  "id": "focVis",
   "what": "validation of focVis test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/hover.json

@@ -1,5 +1,5 @@
 {
-  "id": "hoverVal",
+  "id": "hover",
   "what": "validation of hover test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/labClash.json

@@ -1,5 +1,5 @@
 {
-  "id": "labClashVal",
+  "id": "labClash",
   "what": "validation of labClash test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/linkTo.json

@@ -1,5 +1,5 @@
 {
-  "id": "linkToVal",
+  "id": "linkTo",
   "what": "validation of linkTo test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/linkUl.json

@@ -1,5 +1,5 @@
 {
-  "id": "linkUlVal",
+  "id": "linkUl",
   "what": "validation of linkUl test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/menuNav.json

@@ -1,5 +1,5 @@
 {
-  "id": "menuNavVal",
+  "id": "menuNav",
   "what": "validation of menuNav test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/miniText.json

@@ -1,5 +1,5 @@
 {
-  "id": "miniTextVal",
+  "id": "miniText",
   "what": "validation of miniText test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/motion.json

@@ -1,5 +1,5 @@
 {
-  "id": "motionVal",
+  "id": "motion",
   "what": "validation of motion test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/nonTable.json

@@ -1,5 +1,5 @@
 {
-  "id": "nonTableVal",
+  "id": "nonTable",
   "what": "validation of nonTable test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/radioSet.json

@@ -1,5 +1,5 @@
 {
-  "id": "radioSetVal",
+  "id": "radioSet",
   "what": "validation of radioSet test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/role.json

@@ -1,5 +1,5 @@
 {
-  "id": "roleVal",
+  "id": "role",
   "what": "validation of role test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/styleDiff.json

@@ -1,5 +1,5 @@
 {
-  "id": "styleDiffVal",
+  "id": "styleDiff",
   "what": "validation of styleDiff test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/tabNav.json

@@ -1,5 +1,5 @@
 {
-  "id": "tabNavVal",
+  "id": "tabNav",
   "what": "validation of tabNav test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/textNodes.json

@@ -1,5 +1,5 @@
 {
-  "id": "elementsVal",
+  "id": "elements",
   "what": "validation of elements test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/title.json

@@ -1,5 +1,5 @@
 {
-  "id": "titleVal",
+  "id": "title",
   "what": "validation of title test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/titledEl.json

@@ -1,5 +1,5 @@
 {
-  "id": "titledElVal",
+  "id": "titledEl",
   "what": "validation of titledEl test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/jobs/zIndex.json

@@ -1,5 +1,5 @@
 {
-  "id": "zIndexVal",
+  "id": "zIndex",
   "what": "validation of zIndex test",
   "strict": true,
   "timeLimit": 20,

package/validation/tests/targets/attVal/bad.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Page with illicit attribute values</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page with illicit attribute values</h1>
+      <p><q><span lang="de">Veloparkieren nicht gestattet</span></q> is Swiss German, so it should be marked up with the <code>de-ch</code> language tag.</p>
+    </main>
+  </body>
+</html>

package/validation/tests/targets/attVal/good.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page with licit attribute values</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page with illicit attribute values</h1>
+      <p><q><span>lang="de-CH">Veloparkieren nicht gestattet</span></q> is Swiss German, so it is marked up with the <code>de-CH</code> language tag.</p>
+    </main>
+  </body>
+</html>

package/validation/tests/targets/dupAtt/bad.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page with duplicate attributes</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page with duplicate attributes</h1>
+      <p class="narrow" id="daParagraph" class="wide">Submit</p>
+      <p><button
+        aria-label="large"
+        aria-label="small"
+      >A paragraph</button></p>
+    </main>
+  </body>
+</html>

package/validation/tests/targets/dupAtt/good.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page without duplicate attributes</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page without duplicate attributes</h1>
+      <p class="narrow" id="okParagraph" lang="es-US">Buscar</p>
+      <p><button id="p" aria-label="A body of text">A paragraph</button></p>
+    </main>
+  </body>
+</html>

package/watch.js

@@ -24,6 +24,8 @@ const reportDir = process.env.REPORTDIR;
 
 // ########## FUNCTIONS
 
+// Returns a string representing the date and time.
+const nowString = () => (new Date()).toISOString().slice(0, 19);
 // Checks for a directory job.
 const checkDirJob = async watchee => {
   try {
@@ -44,12 +46,12 @@ const checkDirJob = async watchee => {
       }
     }
     else {
-      console.log('No job to do');
+      console.log(`No job to do (${nowString()})`);
       return {};
     }
   }
   catch(error) {
-    console.log(`Directory watching failed (${error.message})`);
+    console.log('ERROR: Directory watching failed');
     return {};
   }
 };
@@ -86,13 +88,13 @@ const checkNetJob = async watchee => {
     request.end();
   });
   if (job.id) {
-    console.log(`Network job ${job.id} received`);
+    console.log(`Network job ${job.id} received (${nowString()})`);
   }
   else if (job.message) {
     console.log(job.message);
   }
   else {
-    console.log('No network job to do');
+    console.log(`No network job to do (${nowString()})`);
   }
   return job;
 };
@@ -155,7 +157,7 @@ const writeNetReport = async report => {
       const reportJSON = JSON.stringify(report, null, 2);
       request.write(reportJSON);
       request.end();
-      console.log(`Report ${report.id} submitted`);
+      console.log(`Report ${report.id} submitted (${nowString()})`);
     }
     else {
       console.log('ERROR: Report has no sources property');
@@ -217,7 +219,9 @@ exports.cycle = async (isDirWatch, isForever, interval, watchee = null) => {
   let statusOK = true;
   // Prevent a wait before the first iteration.
   let empty = false;
-  console.log(`Watching started with intervals of ${interval} seconds when idle`);
+  console.log(
+    `Watching started with intervals of ${interval} seconds when idle (${nowString()})`
+  );
   while (statusOK) {
     if (empty) {
       await wait(intervalMS);
@@ -233,14 +237,14 @@ exports.cycle = async (isDirWatch, isForever, interval, watchee = null) => {
     // If there was one:
     if (job.id) {
       // Run it, save a report, and if applicable send the report to the job source.
-      console.log(`Running job ${job.id}`);
+      console.log(`Running job ${job.id} (${nowString()})`);
       await runJob(JSON.parse(JSON.stringify(job)), isDirWatch);
-      console.log(`Job ${job.id} finished`);
+      console.log(`Job ${job.id} finished (${nowString()})`);
       // If a directory was watched:
       if (isDirWatch) {
         // Archive the job.
         await archiveJob(job, watchee);
-        console.log(`Job ${job.id} archived`);
+        console.log(`Job ${job.id} archived (${nowString()})`);
       }
       // If watching was specified for only 1 job, quit.
       statusOK = isForever;
@@ -253,5 +257,5 @@ exports.cycle = async (isDirWatch, isForever, interval, watchee = null) => {
       empty = true;
     }
   }
-  console.log('Watching ended');
+  console.log(`Watching ended (${nowString()})`);
 };