testaro 78.0.6 → 78.4.0

package/validation/validateTest.js

@@ -1,6 +1,7 @@
 /*
   © 2022–2025 CVS Health and/or one of its affiliates. All rights reserved.
-  © 2025 Jonathan Robert Pool.
+  © 2026 Jeff Witt.
+  © 2025–2026 Jonathan Robert Pool.
 
   Licensed under the MIT License. See LICENSE file at the project root or
   https://opensource.org/license/mit/ for details.
@@ -15,13 +16,13 @@
 
 // IMPORTS
 
-// Module to process files.
-const fs = require('fs').promises;
 // Module to run Testaro jobs.
 const {doJob} = require('../run');
+// Function to evaluate an expectation against an object.
+const {isTrue} = require('../procs/doActs');
 
 // CONSTANTS
-const job = {
+const jobTemplate = {
   id: '250101T0000-aaa-00',
   what: '',
   strict: true,
@@ -52,18 +53,45 @@ const job = {
 
 // FUNCTIONS
 
-// Validates a Testaro test.
+// Validates a Testaro test and returns a result summary.
 exports.validateTest = async testID => {
   // Get data for a job for the test.
   const jobProperties = require(`./tests/jobProperties/${testID}.json`);
-  // Use the data to complete the job.
+  // Use the data to complete a copy of the job template, so jobs cannot contaminate each other.
+  const job = structuredClone(jobTemplate);
+  // Give the job an ID unique to this process, so concurrent validations do not share a
+  // temporary directory (run.js keeps and deletes tmp/<jobID>).
+  job.id = `${jobTemplate.id}-${process.pid}-${Date.now().toString(36)}`;
   if (jobProperties.standard) {
     job.standard = jobProperties.standard;
   }
+  // Test-act isolation (process, browser, or page), if the validator specifies it.
+  if (jobProperties.isolation) {
+    job.isolation = jobProperties.isolation;
+  }
+  // Rules that compare a page with its catalog image (motion) need one to be made.
+  if (jobProperties.imageColor !== undefined) {
+    job.imageColor = jobProperties.imageColor;
+  }
   job.what = `validate Testaro test ${jobProperties.rule}`;
   job.acts = jobProperties.acts;
-  // Perform it.
-  report = await doJob(job);
+  /*
+    Make the job target the page that the job launches. The catalog, and the page
+    image that the motion rule compares its own screenshot with, are made on the
+    job target before the first act, so leaving the target at its default made
+    every validation catalog a catalog of an unrelated page.
+  */
+  const launchAct = jobProperties.acts.find(act => act.type === 'launch');
+  if (launchAct && launchAct.target && launchAct.target.url) {
+    job.target = {
+      what: launchAct.target.what || 'page for test validation',
+      url: launchAct.target.url
+    };
+  }
+  // Initialize the failure descriptions.
+  const failures = [];
+  // Perform the job.
+  const report = await doJob(job);
   // Report whether the end time was reported.
   const {acts, jobData} = report;
   if (jobData.endTime && /^(?:\d{2}-){2}\d{2}T\d{2}:\d{2}$/.test(jobData.endTime)) {
@@ -71,6 +99,22 @@ exports.validateTest = async testID => {
   }
   else {
     console.log('Failure: End time empty or invalid');
+    failures.push('End time empty or invalid');
+  }
+  // If the validator has report-level expectations, evaluate them against the report.
+  const reportExpectations = jobProperties.expectReport;
+  if (Array.isArray(reportExpectations)) {
+    reportExpectations.forEach(spec => {
+      const truth = isTrue(report, spec, report.catalog);
+      if (! truth[1]) {
+        const message = `Report expectation ${JSON.stringify(spec)} not met (actual ${JSON.stringify(truth[0])})`;
+        console.log(`Failure: ${message}`);
+        failures.push(message);
+      }
+    });
+    if (! failures.length) {
+      console.log('Success: Report expectations met');
+    }
   }
   // If the test acts were correctly reported:
   const testActs = acts.filter(act => act.type && act.type === 'test');
@@ -82,8 +126,8 @@ exports.validateTest = async testID => {
   ) {
     // Report this.
     console.log('Success: Reports have been correctly populated');
-    // If all expectations were satisfied:
-    if (testActs.every(testAct => testAct.expectationFailures === 0)) {
+    // If all expectations, including any report-level ones, were satisfied:
+    if (testActs.every(testAct => testAct.expectationFailures === 0) && ! failures.length) {
       // Report this.
       console.log('######## Success: No failures\n');
     }
@@ -93,6 +137,7 @@ exports.validateTest = async testID => {
       console.log(
         '######## Failure: The test has at least one failure (see “"passed": false” below)\n'
       );
+      failures.push('At least one expectation was not satisfied');
       // Output the acts that had failures.
       console.log(
         JSON.stringify(
@@ -103,6 +148,13 @@ exports.validateTest = async testID => {
   }
   else {
     console.log('Failure: Act results empty or invalid');
+    failures.push('Act results empty or invalid');
     console.log(JSON.stringify(acts, null, 2));
   }
+  // Return the result summary.
+  return {
+    testID,
+    success: ! failures.length,
+    failures
+  };
 };

package/.claude/settings.local.json

@@ -1,11 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(git branch *)",
-      "Bash(git add *)",
-      "Bash(git commit -m ' *)",
-      "Bash(git push *)"
-    ]
-  },
-  "outputStyle": "Explanatory"
-}

package/.eslintrc.json

@@ -1,41 +0,0 @@
-{
-  "env": {
-    "browser": true,
-    "commonjs": true,
-    "es2021": true,
-    "node": true
-  },
-  "extends": "eslint:recommended",
-  "parserOptions": {
-    "ecmaVersion": 2021
-  },
-  "rules": {
-    "indent": [
-      "error",
-      2,
-      {
-        "MemberExpression": 0,
-        "ObjectExpression": "first"
-      }
-    ],
-    "linebreak-style": [
-      "error",
-      "unix"
-    ],
-    "quotes": [
-      "error",
-      "single"
-    ],
-    "semi": [
-      "error",
-      "always"
-    ],
-    "no-use-before-define": [
-      "error"
-    ],
-    "brace-style": [
-      "error",
-      "stroustrup"
-    ]
-  }
-}

package/htmlcs/.eslintrc.json

@@ -1,67 +0,0 @@
-{
-  "root": true,
-  "env": {
-    "browser": true,
-    "commonjs": true,
-    "es2021": true,
-    "node": true
-  },
-  "extends": "eslint:recommended",
-  "parserOptions": {
-    "ecmaVersion": 5
-  },
-  "rules": {
-    "indent": [
-      "error",
-      2,
-      {
-        "MemberExpression": 0,
-        "ObjectExpression": "first"
-      }
-    ],
-    "linebreak-style": [
-      "error",
-      "unix"
-    ],
-    "quotes": [
-      "error",
-      "single"
-    ],
-    "semi": [
-      "error",
-      "always"
-    ],
-    "brace-style": [
-      "error",
-      "stroustrup"
-    ],
-    "no-undef": "off",
-    "no-redeclare": "off",
-    "no-unused-vars": "off",
-    "no-empty": "off",
-    "no-prototype-builtins": "off",
-    "no-cond-assign": "off",
-    "no-fallthrough": "off",
-    "no-constant-condition": "off",
-    "block-spacing": [
-      "error",
-      "never"
-    ],
-    "array-bracket-spacing": [
-      "error",
-      "never"
-    ],
-    "space-before-function-paren": [
-      "error",
-      "always"
-    ],
-    "space-in-parens": [
-      "error",
-      "never"
-    ],
-    "object-curly-spacing": [
-      "error",
-      "never"
-    ]
-  }
-}

package/memory/MEMORY.md

@@ -1,3 +0,0 @@
-# Memory Index
-
-- [Validation system paused](project_validation_pause.md) — new validation tests on hold pending redesign of broken validation architecture

package/memory/project_validation_pause.md

@@ -1,10 +0,0 @@
----
-name: Validation system paused
-description: All new validation test work is on hold pending a redesign of the currently broken validation architecture
-type: project
----
-
-New validation tests and updates to existing validation tests are paused. The validation system is currently broken and a redesign is anticipated. Do not create or modify files in validation/tests/ until the user indicates the redesign is underway.
-
-**Why:** The validation architecture is broken and being reconsidered.
-**How to apply:** Skip validation file creation/updates when adding or modifying Testaro rules until further notice.

package/validation/tests/jobProperties/linkTitle.json

@@ -1,127 +0,0 @@
-{
-  "rule": "linkTitle",
-  "timeLimit": 20,
-  "acts": [
-    {
-      "type": "launch",
-      "target": {
-        "url": "file://validation/tests/targets/linkTitle/index.html",
-        "what": "page with normal and rodundantly titled links"
-      }
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": true,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.0",
-          "=",
-          1
-        ],
-        [
-          "standardResult.totals.3",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.length",
-          "=",
-          1
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "linkTitle"
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "Link has"
-        ],
-        [
-          "standardResult.instances.0.ordinalSeverity",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.tagName",
-          "=",
-          "A"
-        ],
-        [
-          "standardResult.instances.0.id",
-          "=",
-          "basqueInfo"
-        ],
-        [
-          "standardResult.instances.0.location.type",
-          "=",
-          "selector"
-        ],
-        [
-          "standardResult.instances.0.location.spec",
-          "=",
-          "#basqueInfo"
-        ],
-        [
-          "standardResult.instances.0.excerpt",
-          "i",
-          "Basque information"
-        ]
-      ],
-      "rules": [
-        "y",
-        "linkTitle"
-      ]
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": false,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.0",
-          "=",
-          1
-        ],
-        [
-          "standardResult.totals.2",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.length",
-          "=",
-          1
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "linkTitle"
-        ],
-        [
-          "standardResult.instances.0.count",
-          "=",
-          1
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "Links have"
-        ],
-        [
-          "standardResult.instances.0.tagName",
-          "=",
-          "A"
-        ]
-      ],
-      "rules": [
-        "y",
-        "linkTitle"
-      ]
-    }
-  ]
-}

package/validation/tests/jobProperties/opFoc.json

@@ -1,164 +0,0 @@
-{
-  "rule": "opFoc",
-  "timeLimit": 20,
-  "acts": [
-    {
-      "type": "launch",
-      "target": {
-        "url": "file://validation/tests/targets/opFoc/good.html",
-        "what": "page with standard focusability and operability"
-      }
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": true,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.2",
-          "=",
-          0
-        ],
-        [
-          "standardResult.totals.3",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.length",
-          "=",
-          0
-        ]
-      ],
-      "rules": [
-        "y",
-        "opFoc"
-      ]
-    },
-    {
-      "type": "url",
-      "which": "file://validation/tests/targets/opFoc/bad.html",
-      "what": "page with deviant focusability and operability"
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": true,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.2",
-          "=",
-          0
-        ],
-        [
-          "standardResult.totals.3",
-          "=",
-          3
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "opFoc"
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "Element is operable (tag name BUTTON)"
-        ],
-        [
-          "standardResult.instances.0.ordinalSeverity",
-          "=",
-          3
-        ],
-        [
-          "standardResult.instances.0.tagName",
-          "=",
-          "BUTTON"
-        ],
-        [
-          "standardResult.instances.0.location.doc",
-          "=",
-          "dom"
-        ],
-        [
-          "standardResult.instances.0.location.type",
-          "=",
-          "box"
-        ],
-        [
-          "standardResult.instances.0.location.spec.width",
-          ">",
-          0
-        ],
-        [
-          "standardResult.instances.0.excerpt",
-          "=",
-          "button"
-        ],
-        [
-          "standardResult.instances.1.tagName",
-          "=",
-          "SPAN"
-        ],
-        [
-          "standardResult.instances.2.what",
-          "i",
-          "Element is operable (pointer cursor)"
-        ]
-      ],
-      "rules": [
-        "y",
-        "opFoc"
-      ]
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": false,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.2",
-          "=",
-          0
-        ],
-        [
-          "standardResult.totals.3",
-          "=",
-          3
-        ],
-        [
-          "standardResult.totals.1",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "opFoc"
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "Elements are operable but"
-        ],
-        [
-          "standardResult.instances.0.ordinalSeverity",
-          "=",
-          3
-        ],
-        [
-          "standardResult.instances.0.count",
-          "=",
-          3
-        ]
-      ],
-      "rules": [
-        "y",
-        "opFoc"
-      ]
-    }
-  ]
-}

package/validation/tests/jobProperties/targetSmall.json

@@ -1,152 +0,0 @@
-{
-  "rule": "targetsNear",
-  "timeLimit": 20,
-  "acts": [
-    {
-      "type": "launch",
-      "target": {
-        "url": "file://validation/tests/targetsNear/index.html",
-        "what": "page with variously sized interaction targets"
-      }
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": true,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.0",
-          "=",
-          6
-        ],
-        [
-          "standardResult.totals.1",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "targetsNear"
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "is less than 44"
-        ],
-        [
-          "standardResult.instances.0.ordinalSeverity",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.tagName",
-          "=",
-          "BUTTON"
-        ],
-        [
-          "standardResult.instances.0.id",
-          "=",
-          "defaultbutton"
-        ],
-        [
-          "standardResult.instances.0.location.doc",
-          "=",
-          "dom"
-        ],
-        [
-          "standardResult.instances.0.location.type",
-          "=",
-          "selector"
-        ],
-        [
-          "standardResult.instances.0.location.spec",
-          "=",
-          "#defaultbutton"
-        ],
-        [
-          "standardResult.instances.0.excerpt",
-          "i",
-          "Default (small)"
-        ],
-        [
-          "standardResult.instances.2.what",
-          "i",
-          "is less than"
-        ],
-        [
-          "standardResult.instances.2.tagName",
-          "=",
-          "INPUT"
-        ],
-        [
-          "standardResult.instances.2.excerpt",
-          "i",
-          "Small box:"
-        ],
-        [
-          "standardResult.instances.2.location.spec.y",
-          ">",
-          0
-        ],
-        [
-          "standardResult.instances.4.tagName",
-          "=",
-          "A"
-        ],
-        [
-          "standardResult.instances.4.excerpt",
-          "=",
-          "Wikimedia Foundation (small)"
-        ]
-      ],
-      "rules": [
-        "y",
-        "targetsNear"
-      ]
-    },
-    {
-      "type": "test",
-      "which": "testaro",
-      "withItems": false,
-      "stopOnFail": true,
-      "expect": [
-        [
-          "standardResult.totals.0",
-          "=",
-          6
-        ],
-        [
-          "standardResult.totals.3",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.ruleID",
-          "=",
-          "targetsNear"
-        ],
-        [
-          "standardResult.instances.0.what",
-          "i",
-          "are smaller"
-        ],
-        [
-          "standardResult.instances.0.ordinalSeverity",
-          "=",
-          0
-        ],
-        [
-          "standardResult.instances.0.count",
-          "=",
-          6
-        ]
-      ],
-      "rules": [
-        "y",
-        "targetsNear"
-      ]
-    }
-  ]
-}