testaro 16.4.0 → 17.0.0

package/README.md

@@ -603,10 +603,10 @@ If you do not specify rules when using the `testaro` tool, Testaro will test for
 
 The `rules` property for `testaro` is an array whose first item is either `'y'` or `'n'` and whose remaining items are rule IDs. If `'y'`, then only the specified rules’ tests are performed. If `'n'`, then all the evaluative tests are performed, except for the specified rules.
 
-It has been found that the `focOp` and `motion` tests of the `testaro` tool work properly only when the `webkit` browser type is in use. If you want to use `testaro` with different browser types for different tests, you can include 2 or 3 `testaro` test acts in your job, specifying different browser types and different rules.
-
 The `testaro` tool (like the `ibm` tool) has a `withItems` property. If you set it to `false`, the `standardResult` object of `testaro` will contain an `instances` property with summaries that identify issues and instance counts. If you set it to `true`, some of the instances will be itemized.
 
+Unlike any other tool, the `testaro` tools requires a `stopOnFail` property, which specifies whether a failure to conform to any rule (i.e. any value of `totals` other than `[0, 0, 0, 0]`) should terminate the execution of tests for the remaining rules.
+
 ###### WAVE
 
 If a `wave` test act is included in the job, an environment variable named `WAVE_KEY` must exist, with your WAVE API key as its value. You can get it from [WebAIM](https://wave.webaim.org/api/).

package/actSpecs.js

@@ -184,6 +184,7 @@ exports.actSpecs = {
       'Perform Testaro tests',
       {
         withItems: [true, 'boolean', '', 'itemize'],
+        stopOnFail: [true, 'boolean', '', 'whether testing is to stop after first failure'],
         args: [false, 'object', 'areArrays', 'extra args (object; property names are rule IDs and values are arrays of additional argument values ({autocomplete: [["first name", "forename", "given name"], ["last name", "surname", "family name"], ["email"]], buttonMenu: [], focInd: [false, 250], hover: [20]}) by default'],
       }
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testaro",
-  "version": "16.4.0",
+  "version": "17.0.0",
   "description": "Automation of accessibility testing",
   "main": "index.js",
   "scripts": {

package/testaro/focOp.js

@@ -2,10 +2,6 @@
   focOp
   Related to Tenon rule 190.
 
-  WARNING: The chromium and firefox browsers in Playwright fail to accept are stricter on this test than the
-  chromium browser is.
-  misclassifying the tabIndex values of elements. Launch the webkit browser to run this test.
-
   This test reports discrepancies between Tab-focusability and operability. The standard
   practice is to make focusable elements operable and vice versa. If focusable elements are not
   operable, users are likely to be surprised that nothing happens when they try to operate such

package/tests/testaro.js

@@ -54,7 +54,7 @@ const etcRules = {
 
 // Conducts and reports a Testaro test.
 exports.reporter = async (page, options) => {
-  const {withItems, args} = options;
+  const {withItems, stopOnFail, args} = options;
   const argRules = args ? Object.keys(args) : null;
   const rules = options.rules || ['y', ... Object.keys(evalRules)];
   // Initialize the data.
@@ -90,6 +90,11 @@ exports.reporter = async (page, options) => {
       Object.keys(report).forEach(key => {
         data.rules[rule][key] = report[key];
       });
+      // If testing is to stop after a failure and the page failed the test:
+      if (stopOnFail && report.totals.some(total => total)) {
+        // Stop testing.
+        break;
+      }
     }
   }
   // Otherwise, i.e. if the rule specification is invalid:

package/validation/tests/jobs/allCaps.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -89,6 +90,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/allHidden.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -60,6 +61,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -97,6 +99,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -134,6 +137,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -176,6 +180,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -213,6 +218,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -245,6 +251,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",
@@ -277,6 +284,7 @@
       "type": "test",
       "which": "testaro",
       "what": "entirely or mainly hidden page",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/allSlanted.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -94,6 +95,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/attVal.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "attribute values",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -44,6 +45,7 @@
       "which": "testaro",
       "what": "attribute values",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -73,6 +75,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -148,6 +151,7 @@
       "which": "testaro",
       "what": "attribute values",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",

package/validation/tests/jobs/autocomplete.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -39,6 +40,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -65,6 +67,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -117,6 +120,7 @@
       "which": "testaro",
       "what": "autocomplete attributes for name and email",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/bulk.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "visible element count",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -45,6 +46,7 @@
       "type": "test",
       "which": "testaro",
       "what": "visible element count",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/buttonMenu.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -52,6 +53,7 @@
       "args": {
         "buttonMenu": [["End", "+", "-", "-", "Home"]]
       },
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -156,6 +158,7 @@
       "args":{
         "buttonMenu": [["End", "+", "-", "-", "Home"]]
       },
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/distortion.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -84,6 +85,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",

package/validation/tests/jobs/docType.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "doctype",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",
@@ -45,6 +46,7 @@
       "type": "test",
       "which": "testaro",
       "what": "doctype",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",

package/validation/tests/jobs/dupAtt.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "elements with duplicate attributes",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "elements with duplicate attributes",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -108,6 +110,7 @@
       "which": "testaro",
       "what": "elements with duplicate attributes",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/elements.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -38,6 +39,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -58,6 +60,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -78,6 +81,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -98,6 +102,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -148,6 +153,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -178,6 +184,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.items.0.textContent",
@@ -204,6 +211,7 @@
       "which": "testaro",
       "what": "elements",
       "attribute": "class",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.total",
@@ -234,6 +242,7 @@
       "type": "test",
       "which": "testaro",
       "what": "elements",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.elements.data.items.0.textContent",

package/validation/tests/jobs/embAc.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "embedded active elements",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "embedded active elements",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -108,6 +110,7 @@
       "which": "testaro",
       "what": "embedded active elements",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/filter.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "filter",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "filter",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -118,6 +120,7 @@
       "which": "testaro",
       "what": "filter",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/focAll.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "Tab-focusability",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "Tab-focusability",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -93,6 +95,7 @@
       "which": "testaro",
       "what": "Tab-focusability",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/focInd.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "withItems": true,
       "what": "focus indication",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "withItems": true,
       "what": "focus indication",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -133,6 +135,7 @@
       "which": "testaro",
       "withItems": false,
       "what": "focus indication",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",

package/validation/tests/jobs/focOp.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "focusability and operability",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "what": "focusability and operability",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -158,6 +160,7 @@
       "which": "testaro",
       "what": "focusability and operability",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/focVis.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "off-display focused links",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/hovInd.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/hover.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -52,6 +53,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",
@@ -191,6 +193,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",
@@ -235,6 +238,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",

package/validation/tests/jobs/labClash.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "Labeling",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "what": "Labeling",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/lineHeight.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",
@@ -84,6 +85,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.3",

package/validation/tests/jobs/linkAmb.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -89,6 +90,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",

package/validation/tests/jobs/linkExt.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -89,6 +90,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/linkTitle.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -84,6 +85,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/linkTo.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "links without destinations",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -81,6 +82,7 @@
       "which": "testaro",
       "what": "links without destinations",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/linkUl.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "adjacent-link underlining",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "adjacent-link underlining",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -78,6 +80,7 @@
       "which": "testaro",
       "what": "inline-link underlining",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",

package/validation/tests/jobs/miniText.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "small text",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -96,6 +97,7 @@
       "which": "testaro",
       "what": "small text",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/motion.json

@@ -17,6 +17,7 @@
     {
       "type": "test",
       "which": "testaro",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -59,6 +60,7 @@
       "type": "test",
       "which": "testaro",
       "what": "spontaneous change of content",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.instances.length",

package/validation/tests/jobs/nonTable.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -105,6 +106,7 @@
       "which": "testaro",
       "what": "tables used for layout",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/pseudoP.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -84,6 +85,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/radioSet.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "radioSet",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -46,6 +47,7 @@
       "which": "testaro",
       "what": "radioSet",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -148,6 +150,7 @@
       "which": "testaro",
       "what": "radioSet",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/role.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "role",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",
@@ -50,6 +51,7 @@
       "type": "test",
       "which": "testaro",
       "what": "role",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.1",

package/validation/tests/jobs/styleDiff.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "withItems": true,
       "what": "styleDiff",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "withItems": true,
       "what": "styleDiff",
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/tabNav.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "tab-list navigation",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "what": "tab-list navigation",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -138,6 +140,7 @@
       "which": "testaro",
       "what": "tab-list navigation",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/targetSize.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -114,6 +115,7 @@
       "type": "test",
       "which": "testaro",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",

package/validation/tests/jobs/textNodes.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "text nodes",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.textNodes.data.nodeCount",
@@ -38,6 +39,7 @@
       "type": "test",
       "which": "testaro",
       "what": "text nodes",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.textNodes.data.nodeCount",
@@ -58,6 +60,7 @@
       "type": "test",
       "which": "testaro",
       "what": "text nodes",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.textNodes.data.nodeCount",
@@ -83,6 +86,7 @@
       "type": "test",
       "which": "testaro",
       "what": "text nodes",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.textNodes.data.nodeCount",
@@ -167,6 +171,7 @@
       "type": "test",
       "which": "testaro",
       "what": "text nodes",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.textNodes.data.nodeCount",

package/validation/tests/jobs/title.json

@@ -18,6 +18,7 @@
       "type": "test",
       "which": "testaro",
       "what": "document title",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.title.data.title",
@@ -40,6 +41,7 @@
       "type": "test",
       "which": "testaro",
       "what": "document title",
+      "stopOnFail": true,
       "expect": [
         [
           "result.rules.title.data.title",

package/validation/tests/jobs/titledEl.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "title attributes on inappropriate elements",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",
@@ -86,6 +87,7 @@
       "which": "testaro",
       "what": "title attributes on inappropriate elements",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.2",

package/validation/tests/jobs/zIndex.json

@@ -19,6 +19,7 @@
       "which": "testaro",
       "what": "zIndex",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -51,6 +52,7 @@
       "which": "testaro",
       "what": "zIndex",
       "withItems": true,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",
@@ -108,6 +110,7 @@
       "which": "testaro",
       "what": "zIndex",
       "withItems": false,
+      "stopOnFail": true,
       "expect": [
         [
           "standardResult.totals.0",