testaro 16.0.0 → 16.2.0

package/testaro/motion.js

@@ -4,131 +4,46 @@
   brief, or else stoppable by the user. But stopping motion can be difficult or impossible, and,
   by the time a user manages to stop motion, the motion may have caused annoyance or harm. For
   superior accessibility, a page contains no motion until and unless the user authorizes it. The
-  test compares screen shots of the part of the page within the visible viewport. You can specify
-  how many milliseconds to wait before the first screen shot (delay), how many milliseconds to wait
-  between screen shots (interval), and how many screen shots to make (count). The test compares the
-  screen shots and reports 9 statistics:
-    0. bytes: an array of the sizes of the screen shots, in bytes
-    1. localRatios: an array of the ratios of bytes of the larger to the smaller of adjacent pairs
-      of screen shots
-    2. meanLocalRatio: the mean of the ratios in the localRatios array
-    3. maxLocalRatio: the greatest of the ratios in the localRatios array
-    4. globalRatio: the ratio of bytes of the largest to the smallest screen shot
-    5. pixelChanges: an array of counts of differing pixels between adjacent pairs of screen shots
-    6. meanPixelChange: the mean of the counts in the pixelChanges array
-    7. maxPixelChange: the greatest of the counts in the pixelChanges array
-    8. changeFrequency: what fraction of the adjacent pairs of screen shots has pixel differences
+  test compares two screen shots of the viewport 2 seconds and 6 seconds after page load. It
+  reports a rule violation if any pixels change. The larger the change fraction, the greater the
+  ordinal severity.
   
-  WARNING: This test uses the Playwright page.screenshot method, which produces incorrect results
-  when the browser type is chromium and is not implemented for the firefox browser type. The only
-  browser type usable with this test is webkit.
+  WARNING: This test uses the Playwright page.screenshot method, which is not implemented for the
+  firefox browser type.
 */
-const pixelmatch = require('pixelmatch');
-const {PNG} = require('pngjs');
-// Creates and returns a screenshot.
-const shoot = async page => {
-  // Make a screenshot as a buffer.
-  return await page.screenshot({
-    fullPage: false,
-    omitBackground: true,
-    timeout: 3000
-  })
-  .catch(error => {
-    console.log(`ERROR: Screenshot failed(${error.message})`);
-    return '';
-  });
-};
-// Recursively creates and returns screenshots.
-const shootAll = async (page, delay, interval, count, toDo, buffers) => {
-  // Wait.
-  await page.waitForTimeout(toDo === count ? delay : interval);
-  // Make a screenshot.
-  const buffer = await shoot(
-    page, `${page.url().replace(/^.+\/\/|\/$/g, '').replace(/\//g, '+')}-${count - toDo}`
-  );
-  // Get its dimensions.
-  if (buffer.length) {
-    buffers.push(buffer);
-    if (toDo > 1) {
-      return shootAll(page, delay, interval, count, toDo - 1, buffers);
-    }
-    else {
-      return buffers;
-    }
-  }
-  else {
-    return '';
-  }
-};
-// Returns a number rounded to 2 decimal digits.
-const round = (num, precision) => Number.parseFloat(num.toPrecision(precision));
+
+// IMPORTS
+
+// Module to get pixel changes between two times.
+const {visChange} = require('../procs/visChange');
+
+// FUNCTIONS
+
 // Reports motion in a page.
-exports.reporter = async (page, withItems, delay = 2500, interval = 2500, count = 5) => {
-  // Make screenshots and get their image buffers.
-  const shots = await shootAll(page, delay, interval, count, count, []);
-  // If the shooting succeeded:
-  if (shots.length === count) {
-    // Get the sizes of the shots in bytes of code.
-    const bytes = shots.map(shot => shot.length);
-    // Get their ratios between adjacent pairs of shots.
-    const localRatios = bytes.slice(1).map((size, index) => round(
-      (size > bytes[index] ? size / bytes[index] : bytes[index] / size), 4
-    ));
-    // Get the mean and maximum of those ratios.
-    const meanLocalRatio = round(
-      localRatios.reduce((sum, currentRatio) => sum + currentRatio) / localRatios.length, 4
-    );
-    const maxLocalRatio = Math.max(...localRatios);
-    // Get the ratio between the largest and smallest shot.
-    const globalRatio = round((Math.max(...bytes) / Math.min(...bytes)), 4);
-    // Get the shots as PNG images.
-    const pngs = shots.map(shot => PNG.sync.read(shot));
-    // Get their dimensions.
-    const {width, height} = pngs[0];
-    // Get the counts of differing pixels between adjacent pairs of shots.
-    const pixelChanges = pngs
-    .slice(1)
-    .map((png, index) => pixelmatch(pngs[index].data, png.data, null, width, height));
-    // Get the mean and maximum of those counts.
-    const meanPixelChange = Math.floor(
-      pixelChanges.reduce((sum, currentChange) => sum + currentChange) / pixelChanges.length
-    );
-    const maxPixelChange = Math.max(...pixelChanges);
-    const changeFrequency = round(
-      pixelChanges.reduce((count, change) => count + (change ? 1 : 0), 0) / pixelChanges.length, 2
-    );
-    // Return the result.
-    const rawCount = 2 * (meanLocalRatio - 1)
-    + maxLocalRatio - 1
-    + globalRatio - 1
-    + meanPixelChange / 10000
-    + maxPixelChange / 25000
-    + 3 * changeFrequency;
-    const count = rawCount ? Math.round(rawCount) : 0;
-    return {
-      data: {
-        bytes,
-        localRatios,
-        meanLocalRatio,
-        maxLocalRatio,
-        globalRatio,
-        pixelChanges,
-        meanPixelChange,
-        maxPixelChange,
-        changeFrequency
-      },
-      totals: [
-        0,
-        0,
-        count,
-        0
-      ],
-      standardInstances: count ? [{
+exports.reporter = async page => {
+  // Initialize the totals and standard instances.
+  const totals = [0, 0, 0, 0];
+  const standardInstances = [];
+  // Make screenshots and get the result.
+  const data = await visChange(page, {
+    delayBefore: 2000,
+    delayBetween: 3000
+  });
+  // If the screenshots succeeded:
+  if (data.success) {
+    // If any pixels were changed:
+    if (data.pixelChanges) {
+      // Get the ordinal severity from the fractional pixel change.
+      const ordinalSeverity = Math.floor(Math.min(3, 0.4 * Math.sqrt(data.changePercent)));
+      // Add to the totals.
+      totals[ordinalSeverity] = 1;
+      // Get a summary standard instance.
+      standardInstances.push({
         ruleID: 'motion',
         what: 'Content moves or changes without user request',
-        count,
-        ordinalSeverity: 2,
-        tagName: '',
+        count: 1,
+        ordinalSeverity,
+        tagName: 'HTML',
         id: '',
         location: {
           doc: '',
@@ -136,17 +51,13 @@ exports.reporter = async (page, withItems, delay = 2500, interval = 2500, count
           spec: ''
         },
         excerpt: ''
-      }] : []
-    };
-  }
-  // Otherwise, i.e. if the shooting failed:
-  else {
-    // Return failure.
-    return {
-      data: {
-        prevented: true,
-        error: 'ERROR: screenshots failed'
-      }
-    };
+      });
+    }
   }
+  // Return the result.
+  return {
+    data,
+    totals,
+    standardInstances
+  };
 };

package/validation/tests/jobs/hovInd.json

@@ -0,0 +1,320 @@
+{
+  "id": "hover",
+  "what": "validation of hover test",
+  "strict": true,
+  "timeLimit": 40,
+  "acts": [
+    {
+      "type": "launch",
+      "which": "webkit",
+      "what": "only compatible browser"
+    },
+    {
+      "type": "url",
+      "which": "__targets__/hover/good.html",
+      "what": "page with standard hover behavior"
+    },
+    {
+      "type": "test",
+      "which": "testaro",
+      "withItems": true,
+      "expect": [
+        [
+          "standardResult.totals.0",
+          "=",
+          0
+        ],
+        [
+          "standardResult.totals.2",
+          "=",
+          0
+        ],
+        [
+          "standardResult.instances.length",
+          "=",
+          0
+        ]
+      ],
+      "rules": [
+        "y",
+        "hover"
+      ],
+      "args": {
+        "hover": [5]
+      }
+    },
+    {
+      "type": "url",
+      "which": "__targets__/hover/bad.html",
+      "what": "page with deviant hover behavior"
+    },
+    {
+      "type": "test",
+      "which": "testaro",
+      "withItems": true,
+      "expect": [
+        [
+          "standardResult.totals.3",
+          ">",
+          0
+        ],
+        [
+          "standardResult.instances.0.ruleID",
+          "=",
+          "hover"
+        ],
+        [
+          "standardResult.instances.0.what",
+          "i",
+          "over the element"
+        ],
+        [
+          "standardResult.instances.0.tagName",
+          "=",
+          "A"
+        ],
+        [
+          "standardResult.instances.0.location.doc",
+          "=",
+          "dom"
+        ],
+        [
+          "standardResult.instances.0.location.type",
+          "=",
+          "box"
+        ],
+        [
+          "standardResult.instances.0.location.spec.height",
+          "<",
+          "20"
+        ],
+        [
+          "standardResult.instances.0.excerpt",
+          "i",
+          "Trigger 1"
+        ],
+        [
+          "standardResult.instances.2.what",
+          "i",
+          "is not hoverable"
+        ],
+        [
+          "standardResult.instances.2.ordinalSeverity",
+          "=",
+          3
+        ],
+        [
+          "standardResult.instances.2.tagName",
+          "=",
+          "BUTTON"
+        ],
+        [
+          "standardResult.instances.2.id",
+          "=",
+          "smallButton"
+        ],
+        [
+          "standardResult.instances.0.location.doc",
+          "=",
+          "dom"
+        ],
+        [
+          "standardResult.instances.0.location.type",
+          "=",
+          "selector"
+        ],
+        [
+          "standardResult.instances.0.location.spec",
+          "=",
+          "#smallButton"
+        ],
+        [
+          "standardResult.instances.2.excerpt",
+          "i",
+          "Trigger 3"
+        ]
+      ],
+      "rules": [
+        "y",
+        "hover"
+      ],
+      "args": {
+        "hover": [6]
+      }
+    },
+    {
+      "type": "test",
+      "which": "testaro",
+      "withItems": false,
+      "expect": [
+        [
+          "standardResult.totals.3",
+          ">",
+          0
+        ],
+        [
+          "standardResult.instances.length",
+          ">",
+          0
+        ],
+        [
+          "standardResult.instances.0.ruleID",
+          "=",
+          "hover"
+        ],
+        [
+          "standardResult.instances.0.what",
+          "i",
+          "over elements"
+        ],
+        [
+          "standardResult.instances.0.ordinalSeverity",
+          ">",
+          -1
+        ],
+        [
+          "standardResult.instances.0.count",
+          ">",
+          0
+        ]
+      ],
+      "rules": [
+        "y",
+        "hover"
+      ],
+      "args": {
+        "hover": [6]
+      }
+    },
+    {
+      "type": "test",
+      "which": "testaro",
+      "withItems": false,
+      "expect": [
+        [
+          "standardResult.totals.3",
+          "<",
+          7
+        ],
+        [
+          "standardResult.totals.2",
+          "<",
+          4
+        ],
+        [
+          "standardResult.totals.1",
+          "<",
+          13
+        ],
+        [
+          "standardResult.totals.0",
+          "<",
+          4
+        ]
+      ],
+      "rules": [
+        "y",
+        "hover"
+      ],
+      "args": {
+        "hover": [2]
+      }
+    },
+    {
+      "type": "url",
+      "which": "__targets__/hover/styleBad.html",
+      "what": "page with deviant trigger styles"
+    },
+    {
+      "type": "test",
+      "which": "testaro",
+      "withItems": true,
+      "expect": [
+        [
+          "standardResult.totals.1",
+          "=",
+          0
+        ],
+        [
+          "standardResult.totals.2",
+          "=",
+          2
+        ],
+        [
+          "standardResult.totals.3",
+          "=",
+          1
+        ],
+        [
+          "standardResult.instances.0.ruleID",
+          "=",
+          "hover"
+        ],
+        [
+          "standardResult.instances.0.what",
+          "i",
+          "hides"
+        ],
+        [
+          "standardResult.instances.0.tagName",
+          "=",
+          "LI"
+        ],
+        [
+          "standardResult.instances.0.ordinalSeverity",
+          "=",
+          3
+        ],
+        [
+          "standardResult.instances.0.excerpt",
+          "i",
+          "loses its cursor"
+        ],
+        [
+          "standardResult.instances.1.ruleID",
+          "=",
+          "hover"
+        ],
+        [
+          "standardResult.instances.1.what",
+          "i",
+          "cursor nonstandard"
+        ],
+        [
+          "standardResult.instances.1.tagName",
+          "=",
+          "A"
+        ],
+        [
+          "standardResult.instances.1.id",
+          "=",
+          "trigger1"
+        ],
+        [
+          "standardResult.instances.1.ordinalSeverity",
+          "=",
+          2
+        ],
+        [
+          "standardResult.instances.1.excerpt",
+          "i",
+          "Trigger 1"
+        ]
+      ],
+      "rules": [
+        "y",
+        "hover"
+      ],
+      "args": {
+        "hover": [4]
+      }
+    }
+  ],
+  "sources": {
+    "script": "",
+    "host": {},
+    "requester": ""
+  },
+  "creationTime": "2013-05-28T12:00:00",
+  "timeStamp": "00000"
+}