testaro 5.13.1 → 5.14.0

package/README.md

@@ -442,11 +442,16 @@ For example, a `test` command might have this `expect` property:
 
 That would state the expectation that the `result` property of the `acts` item for that test in the report will have a `total.links` property with the value 5, a `total.links.underlined` property with a value less than 6, **no** `total.links.outlined` property, and a `docLang` property with a value different from `es-ES`.
 
+The first item in each array is an identifier of a property within the `result` property. The item has the format of a string with `.` delimiters. Each `.`-delimited segment its the name of the next property in the hierarchy. If the current object is an array, the next segment must be a non-negative integer, representing the index of an element of the array. For example, `items.1.attributes.0` references the first element of the array that is the `attributes` property of the object that is the second element of the array that is the `items` property of `result`. (In JavaScript, this would be written `items[1].attributes[0]`, but in the `expect` property all property names are `.`-delimited.)
+
 The second item in each array, if there are 3 items in the array, is an operator, drawn from:
 - `<`: less than
 - `=`: equal to
 - `>`: greater than
 - `!`: unequal to
+- `i`: includes
+
+The third item in each array, if there are 3 items in the array, is the criterion with which the value of the first property is compared.
 
 A typical use for an `expect` property is checking the correctness of a Testaro test. Thus, the validation scripts in the `validation/tests/scripts` directory all contain `test` commands with `expect` properties. See the “Validation” section below.
 
@@ -593,9 +598,9 @@ In order to make network watching possible, you must define these environment va
 
 As mentioned above, using the high-level method to run Testaro jobs requires `SCRIPTDIR`, `BATCHDIR`, and `REPORTDIR` environment variables.
 
-If a `tenon` test is included in the script, environment variables named `TENON_USER` and `TENON_PASSWORD` must exist, with your Tenon username and password, respectively, as their values.
+If a `tenon` test is included in the script, environment variables named `TENON_USER` and `TENON_PASSWORD` must exist, with your Tenon username and password, respectively, as their values. You can obtain those from [Tenon](https://tenon.io/documentation/overview).
 
-If a `wave` test is included in the script, an environment variable named `WAVE_KEY` must exist, with your WAVE API key as its value.
+If a `wave` test is included in the script, an environment variable named `WAVE_KEY` must exist, with your WAVE API key as its value. You can get it from [WebAIM](https://wave.webaim.org/api/).
 
 The `text` command can interpolate the value of an environment variable into text that it enters on a page, as documented in the `commands.js` file.
 

package/high.js

@@ -1,7 +1,7 @@
 /*
   high.js
   Invokes Testaro with the high-level method.
-  Usage example: node high tp12 weborgs
+  Usage example: node high tp15 weborgs
 */
 
 const {runJob} = require('./create');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "5.13.1",
+  "version": "5.14.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/run.js

@@ -304,24 +304,30 @@ const launch = async typeName => {
       browserContext.on('page', async page => {
         // Make the page current.
         currentPage = page;
-        // Make abbreviations of its console messages get reported in the Playwright console.
+        // Make its console messages get reported in the Playwright console.
         page.on('console', msg => {
           const msgText = msg.text();
-          const parts = [msgText.slice(0, 75)];
-          if (msgText.length > 75) {
-            parts.push(msgText.slice(75, 150));
-            if (msgText.length > 150) {
-              const tail = msgText.slice(150).slice(-150);
-              if (msgText.length > 300) {
-                parts.push('...');
-              }
-              parts.push(tail.slice(0, 75));
-              if (tail.length > 75) {
-                parts.push(tail.slice(75));
+          let indentedMsg = '';
+          if (debug) {
+            indentedMsg = `    | ${msg.text()}`;
+          }
+          else {
+            const parts = [msgText.slice(0, 75)];
+            if (msgText.length > 75) {
+              parts.push(msgText.slice(75, 150));
+              if (msgText.length > 150) {
+                const tail = msgText.slice(150).slice(-150);
+                if (msgText.length > 300) {
+                  parts.push('...');
+                }
+                parts.push(tail.slice(0, 75));
+                if (tail.length > 75) {
+                  parts.push(tail.slice(75));
+                }
               }
             }
+            indentedMsg = parts.map(part => `    | ${part}`).join('\n');
           }
-          const indentedMsg = parts.map(part => `    | ${part}`).join('\n');
           console.log(`\n${indentedMsg}`);
           const msgTextLC = msgText.toLowerCase();
           const msgLength = msgText.length;
@@ -562,23 +568,34 @@ const isTrue = (object, specs) => {
     propertyTree.shift();
     actual = actual[propertyTree[0]];
   }
-  // Determine whether the expectation was fulfilled.
-  if (relation === '=') {
-    satisfied = actual === criterion;
-  }
-  else if (relation === '<') {
-    satisfied = actual < criterion;
-  }
-  else if (relation === '>') {
-    satisfied = actual > criterion;
-  }
-  else if (relation === '!') {
-    satisfied = actual !== criterion;
+  if (actual === undefined) {
+    return [null, false];
   }
-  else if (! relation) {
-    satisfied = actual === undefined;
+  else {
+    // Determine whether the expectation was fulfilled.
+    if (relation === '=') {
+      satisfied = actual === criterion;
+    }
+    else if (relation === '<') {
+      satisfied = actual < criterion;
+    }
+    else if (relation === '>') {
+      satisfied = actual > criterion;
+    }
+    else if (relation === '!') {
+      satisfied = actual !== criterion;
+    }
+    else if (relation === 'i') {
+      satisfied = actual.includes(criterion);
+    }
+    else if (relation === '!i') {
+      satisfied = ! actual.includes(criterion);
+    }
+    else if (! relation) {
+      satisfied = actual === undefined;
+    }
+    return [actual, satisfied];
   }
-  return [actual, satisfied];
 };
 // Adds a wait error result to an act.
 const waitError = (page, act, error, what) => {
@@ -682,7 +699,7 @@ const doActs = async (report, actIndex, page) => {
           const result = act.result = {};
           // If the text is to be the URL:
           if (what === 'url') {
-            // Wait for it and quit on failure.
+            // Wait for the URL to be the exact text and quit on failure.
             try {
               await page.waitForURL(which, {timeout: 15000});
               result.found = true;
@@ -695,7 +712,7 @@ const doActs = async (report, actIndex, page) => {
           }
           // Otherwise, if the text is to be a substring of the page title:
           else if (what === 'title') {
-            // Wait for it and quit on failure.
+            // Wait for the page title to include the text, case-insensitively, and quit on failure.
             try {
               await page.waitForFunction(
                 text => document
@@ -717,7 +734,7 @@ const doActs = async (report, actIndex, page) => {
           }
           // Otherwise, if the text is to be a substring of the text of the page body:
           else if (what === 'body') {
-            // Wait for it and quit on failure.
+            // Wait for the body to include the text, case-insensitively, and quit on failure.
             try {
               await page.waitForFunction(
                 text => document
@@ -949,13 +966,12 @@ const doActs = async (report, actIndex, page) => {
             // Otherwise, if the act is a move:
             else if (moves[act.type]) {
               const selector = typeof moves[act.type] === 'string' ? moves[act.type] : act.what;
-              // Try to identify the element to perform the move on.
+              // Try up to 5 times to:
               act.result = {found: false};
               let selection = {};
               let tries = 0;
               const slimText = act.which ? debloat(act.which) : '';
               while (tries++ < 5 && ! act.result.found) {
-                // If the page still exists:
                 if (page) {
                   // Identify the elements of the specified type.
                   const selections = await page.$$(selector);
@@ -967,16 +983,17 @@ const doActs = async (report, actIndex, page) => {
                       let matchCount = 0;
                       const selectionTexts = [];
                       for (selection of selections) {
-                        // Add its text or an empty string to the list of texts of such elements.
+                        // Add its lower-case text or an empty string to the list of element texts.
                         const selectionText = slimText ? await textOf(page, selection) : '';
                         selectionTexts.push(selectionText);
-                        // If its text includes any specified text:
+                        // If its text includes any specified text, case-insensitively:
                         if (selectionText.includes(slimText)) {
                           // If the element has the specified index among such elements:
                           if (matchCount++ === (act.index || 0)) {
                             // Report it as the matching element and stop checking.
                             act.result.found = true;
-                            act.result.text = slimText;
+                            act.result.textSpec = slimText;
+                            act.result.textContent = selectionText;
                             break;
                           }
                         }

package/tests/allHidden.js

@@ -0,0 +1,48 @@
+/*
+  allHidden
+  This test reports a page that is entirely or mainly hidden.
+*/
+exports.reporter = async page => {
+  const data = await page.evaluate(() => {
+    const {body} = document;
+    const main = body ? body.querySelector('main') : null;
+    const styles = {
+      doc: window.getComputedStyle(document.documentElement),
+      body: body ? window.getComputedStyle(body) : null,
+      main: main ? window.getComputedStyle(main) : null
+    };
+    const data = {
+      hidden: {
+        document: document.documentElement.hidden,
+        body: body && body.hidden,
+        main: main && main.hidden
+      },
+      reallyHidden: {
+        document: false,
+        body: false,
+        main: false
+      },
+      display: {
+        document: styles.doc && styles.doc.display,
+        body: styles.body && styles.body.display,
+        main: styles.main && styles.main.display
+      },
+      visHidden: {
+        document: styles.doc && styles.doc.visibility === 'hidden',
+        body: styles.body && styles.body.visibility === 'hidden',
+        main: styles.main && styles.main.visibility === 'hidden'
+      },
+      ariaHidden: {
+        document: document.documentElement.ariaHidden === 'true',
+        body: body && body.ariaHidden === 'true',
+        main: main && main.ariaHidden === 'true'
+      }
+    };
+    ['document', 'body', 'main'].forEach(element => {
+      if (data.hidden[element] && ! ['block', 'flex'].includes(data.display[element])) {
+        data.reallyHidden[element] = true;
+      }
+    });
+  });
+  return {result: data};
+};

package/tests/elements.js

@@ -8,8 +8,8 @@
     3. Data on each specified element also include data on its sibling nodes.
 */
 exports.reporter = async (page, detailLevel, tagName, onlyVisible, attribute) => {
-  // Determine a selector of the specified elements.
-  let selector = tagName || '*';
+  // Determine a selector of the specified elements, including any descendants of open shadow roots.
+  let selector = `body ${tagName ? tagName.toLowerCase() : '*'}`;
   if (attribute) {
     selector += `[${attribute}]`;
   }
@@ -19,7 +19,8 @@ exports.reporter = async (page, detailLevel, tagName, onlyVisible, attribute) =>
   let data = {};
   // Get the data on the elements.
   try {
-    data = await page.$$eval(selector, (elements, detailLevel) => {
+    const locator = page.locator(selector);
+    data = await locator.evaluateAll((elements, detailLevel) => {
       // FUNCTION DEFINITIONS START
       // Compacts a string.
       const compact = string => string.replace(/\s+/g, ' ').trim();
@@ -28,10 +29,11 @@ exports.reporter = async (page, detailLevel, tagName, onlyVisible, attribute) =>
         const sibInfo = {
           type: nodeType
         };
+        // If the sibling node is an element:
         if (nodeType === 1) {
           sibInfo.tagName = node.tagName;
           sibInfo.attributes = [];
-          node.attributes.forEach(attribute => {
+          Array.from(node.attributes).forEach(attribute => {
             sibInfo.attributes.push({
               name: attribute.name,
               value: attribute.value
@@ -106,12 +108,12 @@ exports.reporter = async (page, detailLevel, tagName, onlyVisible, attribute) =>
           }
           // If the parental text content is required:
           if (detailLevel > 1) {
-            // Add it to the element data.
-            datum.parentTextContent = parent ? parent.textContent : '';
+            // Add it (excluding any shadow-root descendants) to the element data.
+            datum.parentTextContent = parent ? compact(parent.textContent) : '';
           }
           // If sibling itemization is required:
           if (detailLevel === 3) {
-            // Add the sibling data to the element data.
+            // Add data on the siblings, excluding any descendants of shadow roots, to the data.
             datum.siblings = {
               before: [],
               after: []

package/validation/executors/{styleDiff.js → test.js}

@@ -1,15 +1,17 @@
-// app.js
-// Validator for Testaro tests.
+// test.js
+// Validator for one Testaro test.
+// Execution example: node validation/executors/test focOp
 
 const fs = require('fs').promises;
 const {handleRequest} = require(`${__dirname}/../../run`);
+const test = process.argv[2];
 const validateTests = async () => {
   const totals = {
     attempts: 0,
     successes: 0
   };
   const scriptFileNames = await fs.readdir(`${__dirname}/../tests/scripts`);
-  for (const scriptFileName of scriptFileNames.filter(fileName => fileName === 'styleDiff.json')) {
+  for (const scriptFileName of scriptFileNames.filter(fileName => fileName === `${test}.json`)) {
     const rawScriptJSON = await fs
     .readFile(`${__dirname}/../tests/scripts/${scriptFileName}`, 'utf8');
     const scriptJSON = rawScriptJSON

package/validation/executors/tests.js

@@ -1,4 +1,4 @@
-// app.js
+// tests.js
 // Validator for Testaro tests.
 
 const fs = require('fs').promises;

package/validation/executors/watchNet.js

@@ -1,4 +1,4 @@
-// watchDir.js
+// watchNet.js
 // Validator for network watching.
 
 const fs = require('fs/promises');

package/validation/tests/scripts/docType.json

@@ -0,0 +1,37 @@
+{
+  "what": "validation of docType test",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "usual browser"
+    },
+    {
+      "type": "url",
+      "which": "__targets__/docType/good.html",
+      "what": "page with doctype"
+    },
+    {
+      "type": "test",
+      "which": "docType",
+      "what": "doctype",
+      "expect": [
+        ["docHasType", "=", true]
+      ]
+    },
+    {
+      "type": "url",
+      "which": "__targets__/docType/bad.html",
+      "what": "page without doctype"
+    },
+    {
+      "type": "test",
+      "which": "docType",
+      "what": "doctype",
+      "expect": [
+        ["docHasType", "=", false]
+      ]
+    }
+  ]
+}

package/validation/tests/scripts/elements.json

@@ -0,0 +1,131 @@
+{
+  "what": "validation of elements test",
+  "strict": true,
+  "commands": [
+    {
+      "type": "launch",
+      "which": "chromium",
+      "what": "usual browser"
+    },
+    {
+      "type": "url",
+      "which": "__targets__/elements/index.html",
+      "what": "page with a shadow root"
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 0,
+      "expect": [
+        ["total", "=", 21]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 0,
+      "tagName": "P",
+      "expect": [
+        ["total", "=", 5]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 0,
+      "tagName": "LI",
+      "onlyVisible": true,
+      "expect": [
+        ["total", "=", 2]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 0,
+      "tagName": "P",
+      "attribute": "class",
+      "expect": [
+        ["total", "=", 2]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 1,
+      "tagName": "LI",
+      "attribute": "class=first",
+      "expect": [
+        ["total", "=", 1],
+        ["items.0.tagName", "=", "LI"],
+        ["items.0.parentTagName", "=", "UL"],
+        ["items.0.code", "i", "first"],
+        ["items.0.attributes.0.name", "=", "class"],
+        ["items.0.attributes.0.value", "=", "first"],
+        ["items.0.textContent", "=", "Io"]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 1,
+      "tagName": "INPUT",
+      "expect": [
+        ["total", "=", 2],
+        ["items.0.labels.1", "i", "anything"],
+        ["items.1.labelers.0", "i", "something else"]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 2,
+      "tagName": "LI",
+      "attribute": "class=first",
+      "expect": [
+        ["items.0.textContent", "=", "Io"],
+        ["items.0.parentTextContent", "i", "tute"]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 2,
+      "tagName": "LI",
+      "onlyVisible": true,
+      "attribute": "class",
+      "expect": [
+        ["total", "=", 1],
+        ["items.0.parentTextContent", "i", "alia"],
+        ["items.0.parentTextContent", "i", "tute"]
+      ]
+    },
+    {
+      "type": "test",
+      "which": "elements",
+      "what": "elements",
+      "detailLevel": 3,
+      "tagName": "P",
+      "attribute": "class=first",
+      "expect": [
+        ["items.0.textContent", "=", "This is a paragraph."],
+        ["items.0.parentTagName", "=", "MAIN"],
+        ["items.0.parentTextContent", "i", "something."],
+        ["items.0.siblings.before.0.type", "=", 3],
+        ["items.0.siblings.before.1.type", "=", 1],
+        ["items.0.siblings.before.1.tagName", "=", "H1"],
+        ["items.0.siblings.after.12.type", "=", 1],
+        ["items.0.siblings.after.12.tagName", "=", "LABEL"],
+        ["items.0.siblings.after.12.attributes.0.name", "=", "for"]
+      ]
+    }
+  ]
+}

package/validation/tests/targets/docType/bad.html

@@ -0,0 +1,14 @@
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page without doctype</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page without doctype</h1>
+      <p>This page has no doctype.</p>
+    </main>
+  </body>
+</html>

package/validation/tests/targets/docType/good.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page with doctype</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page with doctype</h1>
+      <p>This page has a doctype.</p>
+    </main>
+  </body>
+</html>

package/validation/tests/targets/elements/index.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <title>Page</title>
+    <meta name="description" content="tester">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <main>
+      <h1>Page</h1>
+      <h2>Main part</h2>
+      <p class="first">This is a paragraph.</p>
+      <p>This is a list:</p>
+      <ul lang="eo">
+        <li class="first">Io</li>
+        <li>Io alia</li>
+        <li class="last" hidden>Io tute alia</li>
+      </ul>
+      <h2>Auxiliary part</h2>
+      <p>This is a summary.</p>
+      <div id="host"></div>
+      <div id="guest">
+        <h3>Details</h3>
+        <p class="sole">Here are some details.</p>
+      </div>
+      <label for="input0">Input something.</label>
+      <input id="input0">
+      <label for="input0">Did you input anything?</label>
+      <p id="input1Label">Input something else.</p>
+      <input aria-labelledby="input1Label">
+    </main>
+    <script>
+      document
+      .getElementById('host')
+      .attachShadow({mode: 'open'})
+      .appendChild(document.getElementById('guest'));
+    </script>
+  </body>
+</html>