testaro 78.0.8 → 78.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitattributes +94 -0
- package/.github/workflows/ci.yml +58 -0
- package/.github/workflows/publish.yml +49 -0
- package/.github/workflows/typescript.yml +35 -0
- package/AGENTS.md +2 -2
- package/CLAUDE.md +24 -18
- package/CONTAINERS.md +9 -6
- package/CONTRIBUTING.md +29 -3
- package/Dockerfile +1 -1
- package/README.md +73 -19
- package/UPGRADES.md +4 -0
- package/actSpecs-doc.md +13 -1
- package/actSpecs.js +29 -5
- package/call.js +6 -5
- package/docker-compose.yml +8 -2
- package/docs/checkpoint-scanning.md +199 -0
- package/docs/standard-result-outcome.md +165 -0
- package/env.example +44 -10
- package/eslint.config.mjs +101 -0
- package/netWatch.js +372 -222
- package/package.json +46 -31
- package/pour/README.md +47 -0
- package/pour/pour.min.js +8 -0
- package/procs/actDo.js +808 -0
- package/procs/catalog.d.ts +17 -0
- package/procs/catalog.js +312 -221
- package/procs/catalog.ts +398 -0
- package/procs/checkpoint.js +110 -0
- package/procs/config.d.ts +35 -0
- package/procs/config.js +62 -0
- package/procs/dateTime.js +2 -1
- package/procs/doActs.js +442 -849
- package/procs/doTestAct.js +27 -121
- package/procs/flow.js +221 -0
- package/procs/generateRuleRegistry.js +94 -0
- package/procs/getSource.d.ts +26 -0
- package/procs/job.js +115 -1
- package/procs/launch.d.ts +40 -0
- package/procs/launch.js +239 -108
- package/procs/nu.d.ts +48 -0
- package/procs/scope.js +196 -0
- package/procs/shoot.d.ts +33 -0
- package/procs/standard.d.ts +18 -0
- package/procs/standard.js +83 -0
- package/procs/standard.ts +126 -0
- package/procs/testAct.js +145 -0
- package/procs/testaro.d.ts +20 -0
- package/procs/testaro.js +236 -189
- package/procs/testaro.ts +321 -0
- package/procs/userPath.js +126 -0
- package/procs/xPath.d.ts +4 -0
- package/procs/xPath.js +81 -69
- package/procs/xPath.ts +106 -0
- package/procs/xPathScript.d.ts +6 -0
- package/procs/xPathScript.js +70 -0
- package/run.js +3 -2
- package/surea11y/README.md +44 -0
- package/surea11y/surea11y.browser.js +14 -0
- package/testaro/adbID.d.ts +3 -0
- package/testaro/adbID.js +38 -41
- package/testaro/adbID.ts +66 -0
- package/testaro/allCapStyle.d.ts +3 -0
- package/testaro/allCapStyle.js +31 -34
- package/testaro/allCapStyle.ts +55 -0
- package/testaro/allCaps.d.ts +16 -0
- package/testaro/allCaps.js +179 -151
- package/testaro/allCaps.ts +228 -0
- package/testaro/allHidden.d.ts +13 -0
- package/testaro/allHidden.js +29 -30
- package/testaro/allHidden.ts +50 -0
- package/testaro/allSlanted.d.ts +3 -0
- package/testaro/allSlanted.js +30 -33
- package/testaro/allSlanted.ts +54 -0
- package/testaro/altScheme.d.ts +3 -0
- package/testaro/altScheme.js +27 -30
- package/testaro/altScheme.ts +50 -0
- package/testaro/attVal.d.ts +3 -0
- package/testaro/attVal.js +20 -21
- package/testaro/attVal.ts +52 -0
- package/testaro/autocomplete.d.ts +3 -0
- package/testaro/autocomplete.js +59 -75
- package/testaro/autocomplete.ts +101 -0
- package/testaro/bulk.d.ts +13 -0
- package/testaro/bulk.js +32 -33
- package/testaro/bulk.ts +55 -0
- package/testaro/buttonMenu.d.ts +9 -0
- package/testaro/buttonMenu.js +317 -319
- package/testaro/buttonMenu.ts +391 -0
- package/testaro/captionLoc.d.ts +3 -0
- package/testaro/captionLoc.js +17 -20
- package/testaro/captionLoc.ts +40 -0
- package/testaro/datalistRef.d.ts +3 -0
- package/testaro/datalistRef.js +33 -36
- package/testaro/datalistRef.ts +55 -0
- package/testaro/distortion.d.ts +3 -0
- package/testaro/distortion.js +57 -26
- package/testaro/distortion.ts +81 -0
- package/testaro/docType.d.ts +15 -0
- package/testaro/docType.js +25 -25
- package/testaro/docType.ts +45 -0
- package/testaro/dupAtt.d.ts +16 -0
- package/testaro/dupAtt.js +113 -104
- package/testaro/dupAtt.ts +144 -0
- package/testaro/elements.d.ts +6 -0
- package/testaro/elements.js +153 -153
- package/testaro/elements.ts +215 -0
- package/testaro/embAc.d.ts +3 -0
- package/testaro/embAc.js +19 -20
- package/testaro/embAc.ts +40 -0
- package/testaro/focAll.d.ts +13 -0
- package/testaro/focAll.js +185 -191
- package/testaro/focAll.ts +217 -0
- package/testaro/focAndOp.d.ts +3 -0
- package/testaro/focAndOp.js +100 -104
- package/testaro/focAndOp.ts +128 -0
- package/testaro/focInd.d.ts +3 -0
- package/testaro/focInd.js +64 -66
- package/testaro/focInd.ts +96 -0
- package/testaro/focVis.d.ts +3 -0
- package/testaro/focVis.js +28 -30
- package/testaro/focVis.ts +52 -0
- package/testaro/headEl.d.ts +10 -0
- package/testaro/headEl.js +61 -62
- package/testaro/headEl.ts +82 -0
- package/testaro/headingAmb.d.ts +3 -0
- package/testaro/headingAmb.js +54 -63
- package/testaro/headingAmb.ts +72 -0
- package/testaro/hovInd.d.ts +12 -0
- package/testaro/hovInd.js +163 -130
- package/testaro/hovInd.ts +199 -0
- package/testaro/hover.d.ts +3 -0
- package/testaro/hover.js +155 -126
- package/testaro/hover.ts +154 -0
- package/testaro/hr.d.ts +3 -0
- package/testaro/hr.js +14 -17
- package/testaro/hr.ts +36 -0
- package/testaro/imageLink.d.ts +3 -0
- package/testaro/imageLink.js +17 -20
- package/testaro/imageLink.ts +42 -0
- package/testaro/labClash.d.ts +3 -0
- package/testaro/labClash.js +31 -33
- package/testaro/labClash.ts +54 -0
- package/testaro/legendLoc.d.ts +3 -0
- package/testaro/legendLoc.js +17 -20
- package/testaro/legendLoc.ts +42 -0
- package/testaro/lineHeight.d.ts +3 -0
- package/testaro/lineHeight.js +43 -48
- package/testaro/lineHeight.ts +70 -0
- package/testaro/linkAmb.d.ts +7 -0
- package/testaro/linkAmb.js +79 -80
- package/testaro/linkAmb.ts +105 -0
- package/testaro/linkExt.d.ts +3 -0
- package/testaro/linkExt.js +13 -16
- package/testaro/linkExt.ts +35 -0
- package/testaro/linkOldAtt.d.ts +3 -0
- package/testaro/linkOldAtt.js +25 -28
- package/testaro/linkOldAtt.ts +48 -0
- package/testaro/linkTo.d.ts +3 -0
- package/testaro/linkTo.js +22 -22
- package/testaro/linkTo.ts +43 -0
- package/testaro/linkUl.d.ts +3 -0
- package/testaro/linkUl.js +30 -34
- package/testaro/linkUl.ts +54 -0
- package/testaro/miniText.d.ts +3 -0
- package/testaro/miniText.js +41 -44
- package/testaro/miniText.ts +68 -0
- package/testaro/motion.d.ts +10 -0
- package/testaro/motion.js +92 -96
- package/testaro/motion.ts +125 -0
- package/testaro/nonTable.d.ts +3 -0
- package/testaro/nonTable.js +39 -45
- package/testaro/nonTable.ts +66 -0
- package/testaro/optRoleSel.d.ts +3 -0
- package/testaro/optRoleSel.js +16 -19
- package/testaro/optRoleSel.ts +41 -0
- package/testaro/phOnly.d.ts +3 -0
- package/testaro/phOnly.js +18 -21
- package/testaro/phOnly.ts +43 -0
- package/testaro/pseudoP.d.ts +3 -0
- package/testaro/pseudoP.js +36 -38
- package/testaro/pseudoP.ts +59 -0
- package/testaro/radioSet.d.ts +3 -0
- package/testaro/radioSet.js +57 -59
- package/testaro/radioSet.ts +79 -0
- package/testaro/registry.d.ts +63 -0
- package/testaro/registry.js +67 -0
- package/testaro/registry.ts +141 -0
- package/testaro/role.d.ts +3 -0
- package/testaro/role.js +27 -29
- package/testaro/role.ts +53 -0
- package/testaro/secHeading.d.ts +3 -0
- package/testaro/secHeading.js +30 -33
- package/testaro/secHeading.ts +53 -0
- package/testaro/styleDiff.d.ts +25 -0
- package/testaro/styleDiff.js +248 -252
- package/testaro/styleDiff.ts +303 -0
- package/testaro/tabNav.d.ts +33 -0
- package/testaro/tabNav.js +272 -342
- package/testaro/tabNav.ts +454 -0
- package/testaro/targetsNear.d.ts +9 -0
- package/testaro/targetsNear.js +130 -132
- package/testaro/targetsNear.ts +160 -0
- package/testaro/textNodes.d.ts +6 -0
- package/testaro/textNodes.js +139 -135
- package/testaro/textNodes.ts +185 -0
- package/testaro/textSem.d.ts +3 -0
- package/testaro/textSem.js +25 -28
- package/testaro/textSem.ts +47 -0
- package/testaro/title.d.ts +9 -0
- package/testaro/title.js +16 -13
- package/testaro/title.ts +31 -0
- package/testaro/titledEl.d.ts +3 -0
- package/testaro/titledEl.js +15 -18
- package/testaro/titledEl.ts +38 -0
- package/testaro/zIndex.d.ts +3 -0
- package/testaro/zIndex.js +19 -22
- package/testaro/zIndex.ts +42 -0
- package/tests/alfa.d.ts +45 -0
- package/tests/alfa.js +138 -141
- package/tests/alfa.ts +214 -0
- package/tests/aslint.d.ts +33 -0
- package/tests/aslint.js +272 -249
- package/tests/aslint.ts +301 -0
- package/tests/axe.d.ts +27 -0
- package/tests/axe.js +199 -200
- package/tests/axe.ts +277 -0
- package/tests/ed11y.d.ts +28 -0
- package/tests/ed11y.js +141 -99
- package/tests/ed11y.ts +178 -0
- package/tests/htmlcs.d.ts +21 -0
- package/tests/htmlcs.js +174 -140
- package/tests/htmlcs.ts +181 -0
- package/tests/ibm.d.ts +52 -0
- package/tests/ibm.js +164 -166
- package/tests/ibm.ts +251 -0
- package/tests/nuVal.d.ts +13 -0
- package/tests/nuVal.js +107 -112
- package/tests/nuVal.ts +145 -0
- package/tests/nuVnu.d.ts +15 -0
- package/tests/nuVnu.js +141 -111
- package/tests/nuVnu.ts +144 -0
- package/tests/pour.d.ts +31 -0
- package/tests/pour.js +242 -0
- package/tests/pour.ts +273 -0
- package/tests/qualWeb.d.ts +39 -0
- package/tests/qualWeb.js +302 -272
- package/tests/qualWeb.ts +415 -0
- package/tests/surea11y.d.ts +33 -0
- package/tests/surea11y.js +288 -0
- package/tests/surea11y.ts +334 -0
- package/tests/testaro.d.ts +25 -0
- package/tests/testaro.js +745 -652
- package/tests/testaro.ts +862 -0
- package/tests/wave.d.ts +46 -0
- package/tests/wave.js +166 -177
- package/tests/wave.ts +252 -0
- package/tsconfig.json +17 -0
- package/types.d.ts +243 -0
- package/types.js +10 -0
- package/types.ts +376 -0
- package/validation/act/README.md +46 -0
- package/validation/act/capture.js +424 -0
- package/validation/act/chromium-issue-draft.md +66 -0
- package/validation/act/fp-triage-2026-08-22.md +86 -0
- package/validation/act/isolation-notes.md +159 -0
- package/validation/act/playwright-issue-draft.md +73 -0
- package/validation/act/propose-mappings.js +0 -0
- package/validation/act/repro-cdp-raw.js +105 -0
- package/validation/act/repro-metarefresh.js +65 -0
- package/validation/act/score.js +184 -0
- package/validation/act/stage3a-stress-report.md +69 -0
- package/validation/act/stage3b-mapping-proposals.md +94 -0
- package/validation/act/stage3b-triage-draft.md +138 -0
- package/validation/act/surea11y-track-a-2026-09-01.md +50 -0
- package/validation/executors/netWatch.js +180 -90
- package/validation/executors/test.js +17 -2
- package/validation/executors/tests.js +118 -10
- package/validation/jobs/reports/raw/260901T1000-surea11y-validation.json +964 -0
- package/validation/jobs/todo/240101T1200-simple-example.json +14 -6
- package/validation/jobs/todo/240101T1300-shoot-example.json +2 -1
- package/validation/jobs/todo/260821T1900-pour-validation.json +45 -0
- package/validation/jobs/todo/260901T1000-surea11y-validation.json +45 -0
- package/validation/knownFailures.json +1 -0
- package/validation/tests/jobProperties/adbID.json +27 -2
- package/validation/tests/jobProperties/{focOp.json → allCapStyle.json} +56 -53
- package/validation/tests/jobProperties/allCaps.json +25 -0
- package/validation/tests/jobProperties/allHidden.json +133 -13
- package/validation/tests/jobProperties/allSlanted.json +3 -3
- package/validation/tests/jobProperties/altScheme.json +23 -8
- package/validation/tests/jobProperties/attVal.json +57 -57
- package/validation/tests/jobProperties/autocomplete.json +11 -1
- package/validation/tests/jobProperties/bulk.json +6 -1
- package/validation/tests/jobProperties/buttonMenu.json +63 -42
- package/validation/tests/jobProperties/captionLoc.json +1 -6
- package/validation/tests/jobProperties/checkpoint-browser.json +401 -0
- package/validation/tests/jobProperties/checkpoint-page.json +397 -0
- package/validation/tests/jobProperties/checkpoint.json +395 -0
- package/validation/tests/jobProperties/datalistRef.json +20 -5
- package/validation/tests/jobProperties/distortion.json +28 -3
- package/validation/tests/jobProperties/docType.json +2 -2
- package/validation/tests/jobProperties/dupAtt.json +48 -33
- package/validation/tests/jobProperties/elements.json +28 -28
- package/validation/tests/jobProperties/embAc.json +36 -31
- package/validation/tests/jobProperties/focAndOp.json +284 -0
- package/validation/tests/jobProperties/focInd.json +34 -34
- package/validation/tests/jobProperties/focVis.json +1 -1
- package/validation/tests/jobProperties/hover.json +39 -37
- package/validation/tests/jobProperties/hr.json +13 -3
- package/validation/tests/jobProperties/imageLink.json +0 -5
- package/validation/tests/jobProperties/labClash.json +68 -33
- package/validation/tests/jobProperties/legendLoc.json +1 -6
- package/validation/tests/jobProperties/lineHeight.json +2 -2
- package/validation/tests/jobProperties/linkAmb.json +13 -18
- package/validation/tests/jobProperties/linkExt.json +1 -1
- package/validation/tests/jobProperties/linkOldAtt.json +12 -2
- package/validation/tests/jobProperties/linkTo.json +1 -1
- package/validation/tests/jobProperties/linkUl.json +65 -65
- package/validation/tests/jobProperties/miniText.json +7 -2
- package/validation/tests/jobProperties/motion.json +4 -50
- package/validation/tests/jobProperties/nonTable.json +48 -3
- package/validation/tests/jobProperties/optRoleSel.json +12 -2
- package/validation/tests/jobProperties/phOnly.json +6 -16
- package/validation/tests/jobProperties/pseudoP.json +17 -2
- package/validation/tests/jobProperties/radioSet.json +30 -30
- package/validation/tests/jobProperties/role.json +25 -5
- package/validation/tests/jobProperties/secHeading.json +26 -21
- package/validation/tests/jobProperties/styleDiff.json +35 -5
- package/validation/tests/jobProperties/tabNav.json +3 -1
- package/validation/tests/jobProperties/targetsNear.json +237 -0
- package/validation/tests/jobProperties/textNodes.json +29 -29
- package/validation/tests/jobProperties/textSem.json +26 -1
- package/validation/tests/jobProperties/title.json +12 -2
- package/validation/tests/jobProperties/titledEl.json +39 -9
- package/validation/tests/jobProperties/userPath.json +317 -0
- package/validation/tests/jobProperties/zIndex.json +40 -35
- package/validation/tests/targets/allCapStyle/index.html +26 -0
- package/validation/tests/targets/checkpoint/index.html +38 -0
- package/validation/tests/targets/datalistRef/index.html +1 -1
- package/validation/tests/targets/focAndOp/bad.html +29 -0
- package/validation/tests/targets/{focOp → focAndOp}/good.html +3 -1
- package/validation/tests/targets/focInd/bad.html +2 -1
- package/validation/tests/targets/headEl/index.html +10 -1
- package/validation/tests/targets/{targetSmall → targetsNear}/index.html +15 -1
- package/validation/tests/targets/userPath/index.html +42 -0
- package/validation/validateTest.js +61 -9
- package/.claude/settings.local.json +0 -11
- package/.eslintrc.json +0 -41
- package/htmlcs/.eslintrc.json +0 -67
- package/validation/tests/jobProperties/linkTitle.json +0 -127
- package/validation/tests/jobProperties/opFoc.json +0 -164
- package/validation/tests/jobProperties/targetSmall.json +0 -152
- package/validation/tests/jobProperties/targetTiny.json +0 -142
- package/validation/tests/targets/focOp/bad.html +0 -25
- package/validation/tests/targets/linkTitle/index.html +0 -24
- package/validation/tests/targets/opFoc/bad.html +0 -26
- package/validation/tests/targets/opFoc/good.html +0 -23
package/testaro/allCaps.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
"use strict";
|
|
1
2
|
/*
|
|
2
3
|
© 2023 CVS Health and/or one of its affiliates. All rights reserved.
|
|
3
4
|
© 2025–2026 Jonathan Robert Pool.
|
|
@@ -6,185 +7,212 @@
|
|
|
6
7
|
|
|
7
8
|
SPDX-License-Identifier: MIT
|
|
8
9
|
*/
|
|
9
|
-
|
|
10
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
13
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
14
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
15
|
+
}
|
|
16
|
+
Object.defineProperty(o, k2, desc);
|
|
17
|
+
}) : (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
o[k2] = m[k];
|
|
20
|
+
}));
|
|
21
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
22
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
23
|
+
}) : function(o, v) {
|
|
24
|
+
o["default"] = v;
|
|
25
|
+
});
|
|
26
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
27
|
+
var ownKeys = function(o) {
|
|
28
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
29
|
+
var ar = [];
|
|
30
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
31
|
+
return ar;
|
|
32
|
+
};
|
|
33
|
+
return ownKeys(o);
|
|
34
|
+
};
|
|
35
|
+
return function (mod) {
|
|
36
|
+
if (mod && mod.__esModule) return mod;
|
|
37
|
+
var result = {};
|
|
38
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
39
|
+
__setModuleDefault(result, mod);
|
|
40
|
+
return result;
|
|
41
|
+
};
|
|
42
|
+
})();
|
|
43
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
|
+
exports.reporter = void 0;
|
|
45
|
+
// IMPORTS
|
|
46
|
+
const https = __importStar(require("https"));
|
|
47
|
+
// Function to build a standard instance.
|
|
48
|
+
const standard_1 = require("../procs/standard");
|
|
10
49
|
/*
|
|
11
50
|
allCaps
|
|
12
51
|
Related to Tenon rule 153.
|
|
13
52
|
This test reports elements whose text contains upper-case strings that are not intrinsically upper-case (i.e., not acronyms, abbreviations, or terms whose standard form is all-capitals). Claude Haiku classifies qualifying catalog entries and estimates the probability of a rule violation. If the AI call fails, the test falls back to a rule-based check for 8+ consecutive upper-case letters.
|
|
53
|
+
Compiled to allCaps.js by tsc (issue #73); edit this file, not the emitted one.
|
|
14
54
|
*/
|
|
15
|
-
|
|
16
|
-
// IMPORTS
|
|
17
|
-
|
|
18
|
-
const https = require('https');
|
|
19
|
-
|
|
20
55
|
// PARAMETERS
|
|
21
|
-
|
|
22
56
|
const MIN_CONFIDENCE = 0.8;
|
|
23
57
|
const MAX_MARGIN = 100;
|
|
24
58
|
const MAX_TOTAL = 2000;
|
|
25
59
|
const MAX_QUALIFYING = 100;
|
|
26
|
-
|
|
27
60
|
// CONSTANTS
|
|
28
|
-
|
|
29
61
|
const ruleID = 'allCaps';
|
|
30
62
|
const whats = 'Elements have all-capital text';
|
|
31
|
-
|
|
32
63
|
// FUNCTIONS
|
|
33
|
-
|
|
34
64
|
// Returns text limited to MAX_MARGIN characters before the first and after the last uppercase match, capped at MAX_TOTAL characters.
|
|
35
|
-
const getContext = text => {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
65
|
+
const getContext = (text) => {
|
|
66
|
+
const matches = [...text.matchAll(/\p{Lu}{2,}/gu)];
|
|
67
|
+
const first = matches[0];
|
|
68
|
+
const last = matches[matches.length - 1];
|
|
69
|
+
const start = Math.max(0, first.index - MAX_MARGIN);
|
|
70
|
+
const end = Math.min(text.length, last.index + last[0].length + MAX_MARGIN);
|
|
71
|
+
return text.slice(start, end).slice(0, MAX_TOTAL);
|
|
42
72
|
};
|
|
43
|
-
|
|
44
73
|
// Returns violations using the rule-based fallback (8+ consecutive uppercase letters).
|
|
45
|
-
const getRuleBasedViolations = catalog =>
|
|
46
|
-
Object.entries(catalog)
|
|
74
|
+
const getRuleBasedViolations = (catalog) => Object.entries(catalog)
|
|
47
75
|
.filter(([, entry]) => entry.text && /\p{Lu}{8,}/u.test(entry.text))
|
|
48
76
|
.map(([index]) => ({
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
77
|
+
catalogIndex: index,
|
|
78
|
+
what: '[No AI available] Element contains all-capital text'
|
|
79
|
+
}));
|
|
53
80
|
// Sends qualifying entries to Claude Haiku and returns confidence scores.
|
|
54
|
-
const classifyWithAI = entries => new Promise((resolve, reject) => {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
60
|
-
const prompt =
|
|
61
|
-
'Classify HTML elements for an accessibility rule. All-capital text violates the rule UNLESS it is an acronym, abbreviation, or term whose standard form is all-capitals (NASA, WHO, etc.).\n\n'
|
|
62
|
-
+ 'For each element, give a confidence score (0.0–1.0, rounded to one decimal place) for the probability that the element VIOLATES the rule (its all-caps text is NOT intrinsically all-caps).\n\n'
|
|
63
|
-
+ 'Respond with ONLY a JSON array. Each element: {"index": <number>, "confidence": <number>}\n\n'
|
|
64
|
-
+ 'Elements:\n'
|
|
65
|
-
+ JSON.stringify(entries);
|
|
66
|
-
const payload = JSON.stringify({
|
|
67
|
-
model: 'claude-haiku-4-5-20251001',
|
|
68
|
-
max_tokens: 2048,
|
|
69
|
-
messages: [{role: 'user', content: prompt}]
|
|
70
|
-
});
|
|
71
|
-
const options = {
|
|
72
|
-
hostname: 'api.anthropic.com',
|
|
73
|
-
path: '/v1/messages',
|
|
74
|
-
method: 'POST',
|
|
75
|
-
headers: {
|
|
76
|
-
'x-api-key': apiKey,
|
|
77
|
-
'anthropic-version': '2023-06-01',
|
|
78
|
-
'content-type': 'application/json',
|
|
79
|
-
'content-length': Buffer.byteLength(payload)
|
|
81
|
+
const classifyWithAI = (entries) => new Promise((resolve, reject) => {
|
|
82
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
83
|
+
if (!apiKey) {
|
|
84
|
+
reject(new Error('ANTHROPIC_API_KEY not set'));
|
|
85
|
+
return;
|
|
80
86
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
87
|
+
const prompt = 'Classify HTML elements for an accessibility rule. All-capital text violates the rule UNLESS it is an acronym, abbreviation, or term whose standard form is all-capitals (NASA, WHO, etc.).\n\n'
|
|
88
|
+
+ 'For each element, give a confidence score (0.0–1.0, rounded to one decimal place) for the probability that the element VIOLATES the rule (its all-caps text is NOT intrinsically all-caps).\n\n'
|
|
89
|
+
+ 'Respond with ONLY a JSON array. Each element: {"index": <number>, "confidence": <number>}\n\n'
|
|
90
|
+
+ 'Elements:\n'
|
|
91
|
+
+ JSON.stringify(entries);
|
|
92
|
+
const payload = JSON.stringify({
|
|
93
|
+
model: 'claude-haiku-4-5-20251001',
|
|
94
|
+
max_tokens: 2048,
|
|
95
|
+
messages: [{ role: 'user', content: prompt }]
|
|
96
|
+
});
|
|
97
|
+
const options = {
|
|
98
|
+
hostname: 'api.anthropic.com',
|
|
99
|
+
path: '/v1/messages',
|
|
100
|
+
method: 'POST',
|
|
101
|
+
headers: {
|
|
102
|
+
'x-api-key': apiKey,
|
|
103
|
+
'anthropic-version': '2023-06-01',
|
|
104
|
+
'content-type': 'application/json',
|
|
105
|
+
'content-length': Buffer.byteLength(payload)
|
|
91
106
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
107
|
+
};
|
|
108
|
+
const req = https.request(options, res => {
|
|
109
|
+
let body = '';
|
|
110
|
+
res.on('data', chunk => { body += chunk; });
|
|
111
|
+
res.on('end', () => {
|
|
112
|
+
try {
|
|
113
|
+
const parsed = JSON.parse(body);
|
|
114
|
+
if (parsed.error) {
|
|
115
|
+
reject(new Error(parsed.error.message));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const text = parsed.content[0].text;
|
|
119
|
+
const classifications = [...text.matchAll(/\{"index":\s*\d+,\s*"confidence":\s*[01]\.\d{1,2}\}/g)]
|
|
120
|
+
.map(m => {
|
|
121
|
+
const { index, confidence } = JSON.parse(m[0]);
|
|
122
|
+
return { index, confidence: Math.round(confidence * 10) / 10 };
|
|
123
|
+
});
|
|
124
|
+
const { input_tokens, output_tokens } = parsed.usage;
|
|
125
|
+
resolve({ classifications, aiModelUsage: { inputTokens: input_tokens, outputTokens: output_tokens } });
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
reject(new Error(`Haiku response error: ${error.message}`));
|
|
129
|
+
}
|
|
97
130
|
});
|
|
98
|
-
const {input_tokens, output_tokens} = parsed.usage;
|
|
99
|
-
resolve({classifications, aiModelUsage: {inputTokens: input_tokens, outputTokens: output_tokens}});
|
|
100
|
-
}
|
|
101
|
-
catch(error) {
|
|
102
|
-
reject(new Error(`Haiku response error: ${error.message}`));
|
|
103
|
-
}
|
|
104
131
|
});
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
req.end();
|
|
132
|
+
req.on('error', reject);
|
|
133
|
+
req.setTimeout(20000, () => req.destroy(new Error('Haiku API timeout')));
|
|
134
|
+
req.write(payload);
|
|
135
|
+
req.end();
|
|
110
136
|
});
|
|
111
|
-
|
|
112
137
|
// Runs the test and returns the result.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}));
|
|
125
|
-
// If there are none:
|
|
126
|
-
if (!qualifying.length) {
|
|
127
|
-
// Report this.
|
|
128
|
-
return {data, totals, standardInstances};
|
|
129
|
-
}
|
|
130
|
-
// Sort them, with unbiased (Fisher–Yates) randomization.
|
|
131
|
-
const sample = qualifying.slice();
|
|
132
|
-
for (let i = sample.length - 1; i > 0; i--) {
|
|
133
|
-
const j = Math.floor(Math.random() * (i + 1));
|
|
134
|
-
[sample[i], sample[j]] = [sample[j], sample[i]];
|
|
135
|
-
}
|
|
136
|
-
// If their count exceeds the limit on AI assistance:
|
|
137
|
-
if (sample.length > MAX_QUALIFYING) {
|
|
138
|
-
// Truncate them.
|
|
139
|
-
sample.length = MAX_QUALIFYING;
|
|
140
|
-
}
|
|
141
|
-
let violations;
|
|
142
|
-
try {
|
|
143
|
-
// Get AI estimates of the probabilities of their violating the rule.
|
|
144
|
-
const {classifications, aiModelUsage} = await classifyWithAI(sample);
|
|
145
|
-
data.aiModelUsage = aiModelUsage;
|
|
146
|
-
// Treat the entries with above-minimum violation confidence levels as violations.
|
|
147
|
-
violations = classifications
|
|
148
|
-
.filter(({confidence}) => confidence >= MIN_CONFIDENCE)
|
|
149
|
-
.map(({index, confidence}) => ({
|
|
150
|
-
catalogIndex: String(index),
|
|
151
|
-
what: `Claude Haiku has ${Math.round(confidence * 100)}% confidence that the element contains unnecessarily all-capital text`
|
|
138
|
+
const reporter = async (_0, report, _1, withItems) => {
|
|
139
|
+
const data = {};
|
|
140
|
+
const totals = [0, 0, 0, 0];
|
|
141
|
+
const standardInstances = [];
|
|
142
|
+
// Get data on the catalog entries whose text values contain 2+ consecutive capital letters.
|
|
143
|
+
const qualifying = Object.entries(report.catalog)
|
|
144
|
+
.filter(([, entry]) => entry.text && /\p{Lu}{2,}/u.test(entry.text))
|
|
145
|
+
.map(([index, entry]) => ({
|
|
146
|
+
index: Number(index),
|
|
147
|
+
tagName: entry.tagName,
|
|
148
|
+
text: getContext(entry.text)
|
|
152
149
|
}));
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
150
|
+
// If there are none:
|
|
151
|
+
if (!qualifying.length) {
|
|
152
|
+
// Report this.
|
|
153
|
+
return { data, totals, standardInstances };
|
|
154
|
+
}
|
|
155
|
+
// Sort them, with unbiased (Fisher–Yates) randomization.
|
|
156
|
+
const sample = qualifying.slice();
|
|
157
|
+
for (let i = sample.length - 1; i > 0; i--) {
|
|
158
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
159
|
+
[sample[i], sample[j]] = [sample[j], sample[i]];
|
|
162
160
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
161
|
+
// If their count exceeds the limit on AI assistance:
|
|
162
|
+
if (sample.length > MAX_QUALIFYING) {
|
|
163
|
+
// Truncate them.
|
|
164
|
+
sample.length = MAX_QUALIFYING;
|
|
165
|
+
}
|
|
166
|
+
let violations;
|
|
167
|
+
try {
|
|
168
|
+
// Get AI estimates of the probabilities of their violating the rule.
|
|
169
|
+
const { classifications, aiModelUsage } = await classifyWithAI(sample);
|
|
170
|
+
data.aiModelUsage = aiModelUsage;
|
|
171
|
+
// Treat the entries with above-minimum violation confidence levels as violations.
|
|
172
|
+
violations = classifications
|
|
173
|
+
.filter(({ confidence }) => confidence >= MIN_CONFIDENCE)
|
|
174
|
+
.map(({ index, confidence }) => ({
|
|
175
|
+
catalogIndex: String(index),
|
|
176
|
+
what: `Claude Haiku has ${Math.round(confidence * 100)}% confidence that the element contains unnecessarily all-capital text`
|
|
177
|
+
}));
|
|
178
|
+
const evaluated = classifications.length;
|
|
179
|
+
const leftOut = qualifying.length - evaluated;
|
|
180
|
+
// If any entries were truncated out and any AI confidence levels were above-minimum:
|
|
181
|
+
if (leftOut > 0 && evaluated > 0) {
|
|
182
|
+
// Add data about the estimated violation rate among those entries.
|
|
183
|
+
data.leftOut = {
|
|
184
|
+
count: leftOut,
|
|
185
|
+
estimatedViolations: Math.round((violations.length / evaluated) * leftOut)
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
data.aiError = error.message;
|
|
191
|
+
violations = getRuleBasedViolations(report.catalog);
|
|
192
|
+
}
|
|
193
|
+
const estimatedLeftOut = data.leftOut?.estimatedViolations ?? 0;
|
|
194
|
+
// Add the estimated violation count to the totals.
|
|
195
|
+
totals[0] = violations.length + estimatedLeftOut;
|
|
196
|
+
// The estimates are AI judgements, so every instance is uncertain.
|
|
197
|
+
const certainty = { outcome: 'cantTell', uncertainty: 'judgement-required' };
|
|
198
|
+
// If itemization is required:
|
|
199
|
+
if (withItems) {
|
|
200
|
+
// For each entry deemed a violation:
|
|
201
|
+
for (const { catalogIndex, what } of violations) {
|
|
202
|
+
// Add an instance to the standard instances.
|
|
203
|
+
standardInstances.push((0, standard_1.getInstance)({ ruleID, what, ordinalSeverity: 0, catalogIndex, ...certainty }));
|
|
204
|
+
}
|
|
205
|
+
// If any entries were truncated:
|
|
206
|
+
if (estimatedLeftOut) {
|
|
207
|
+
// Add a summary instance for them.
|
|
208
|
+
standardInstances.push((0, standard_1.getInstance)({ ruleID, what: whats, ordinalSeverity: 0, count: estimatedLeftOut, ...certainty }));
|
|
209
|
+
}
|
|
177
210
|
}
|
|
178
|
-
//
|
|
179
|
-
if (
|
|
180
|
-
|
|
181
|
-
|
|
211
|
+
// Otherwise, i.e. if itemization is not required, and if any violations exist:
|
|
212
|
+
else if (totals[0]) {
|
|
213
|
+
// Add a summary instance for them.
|
|
214
|
+
standardInstances.push((0, standard_1.getInstance)({ ruleID, what: whats, ordinalSeverity: 0, count: totals[0], ...certainty }));
|
|
182
215
|
}
|
|
183
|
-
|
|
184
|
-
// Otherwise, i.e. if itemization is not required, and if any violations exist:
|
|
185
|
-
else if (totals[0]) {
|
|
186
|
-
// Add a summary instance for them.
|
|
187
|
-
standardInstances.push({ruleID, what: whats, ordinalSeverity: 0, count: totals[0]});
|
|
188
|
-
}
|
|
189
|
-
return {data, totals, standardInstances};
|
|
216
|
+
return { data, totals, standardInstances };
|
|
190
217
|
};
|
|
218
|
+
exports.reporter = reporter;
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/*
|
|
2
|
+
© 2023 CVS Health and/or one of its affiliates. All rights reserved.
|
|
3
|
+
© 2025–2026 Jonathan Robert Pool.
|
|
4
|
+
|
|
5
|
+
Licensed under the MIT License. See LICENSE file at the project root or https://opensource.org/license/mit/ for details.
|
|
6
|
+
|
|
7
|
+
SPDX-License-Identifier: MIT
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
// IMPORTS
|
|
11
|
+
|
|
12
|
+
import * as https from 'https';
|
|
13
|
+
// Function to build a standard instance.
|
|
14
|
+
import {getInstance} from '../procs/standard';
|
|
15
|
+
import type {Catalog, Report, StandardInstance} from '../types';
|
|
16
|
+
|
|
17
|
+
// TYPES
|
|
18
|
+
|
|
19
|
+
// One catalog entry prepared for classification.
|
|
20
|
+
interface QualifyingEntry {
|
|
21
|
+
index: number;
|
|
22
|
+
tagName: string;
|
|
23
|
+
text: string;
|
|
24
|
+
}
|
|
25
|
+
// One classification decision.
|
|
26
|
+
interface Classification {
|
|
27
|
+
index: number;
|
|
28
|
+
confidence: number;
|
|
29
|
+
}
|
|
30
|
+
// One entry deemed a violation.
|
|
31
|
+
interface Violation {
|
|
32
|
+
catalogIndex: string;
|
|
33
|
+
what: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/*
|
|
37
|
+
allCaps
|
|
38
|
+
Related to Tenon rule 153.
|
|
39
|
+
This test reports elements whose text contains upper-case strings that are not intrinsically upper-case (i.e., not acronyms, abbreviations, or terms whose standard form is all-capitals). Claude Haiku classifies qualifying catalog entries and estimates the probability of a rule violation. If the AI call fails, the test falls back to a rule-based check for 8+ consecutive upper-case letters.
|
|
40
|
+
Compiled to allCaps.js by tsc (issue #73); edit this file, not the emitted one.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
// PARAMETERS
|
|
44
|
+
|
|
45
|
+
const MIN_CONFIDENCE = 0.8;
|
|
46
|
+
const MAX_MARGIN = 100;
|
|
47
|
+
const MAX_TOTAL = 2000;
|
|
48
|
+
const MAX_QUALIFYING = 100;
|
|
49
|
+
|
|
50
|
+
// CONSTANTS
|
|
51
|
+
|
|
52
|
+
const ruleID = 'allCaps';
|
|
53
|
+
const whats = 'Elements have all-capital text';
|
|
54
|
+
|
|
55
|
+
// FUNCTIONS
|
|
56
|
+
|
|
57
|
+
// Returns text limited to MAX_MARGIN characters before the first and after the last uppercase match, capped at MAX_TOTAL characters.
|
|
58
|
+
const getContext = (text: string) => {
|
|
59
|
+
const matches = [...text.matchAll(/\p{Lu}{2,}/gu)];
|
|
60
|
+
const first = matches[0];
|
|
61
|
+
const last = matches[matches.length - 1];
|
|
62
|
+
const start = Math.max(0, first.index - MAX_MARGIN);
|
|
63
|
+
const end = Math.min(text.length, last.index + last[0].length + MAX_MARGIN);
|
|
64
|
+
return text.slice(start, end).slice(0, MAX_TOTAL);
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
// Returns violations using the rule-based fallback (8+ consecutive uppercase letters).
|
|
68
|
+
const getRuleBasedViolations = (catalog: Catalog): Violation[] =>
|
|
69
|
+
Object.entries(catalog)
|
|
70
|
+
.filter(([, entry]) => entry.text && /\p{Lu}{8,}/u.test(entry.text))
|
|
71
|
+
.map(([index]) => ({
|
|
72
|
+
catalogIndex: index,
|
|
73
|
+
what: '[No AI available] Element contains all-capital text'
|
|
74
|
+
}));
|
|
75
|
+
|
|
76
|
+
// Sends qualifying entries to Claude Haiku and returns confidence scores.
|
|
77
|
+
const classifyWithAI = (entries: QualifyingEntry[]) => new Promise<{
|
|
78
|
+
classifications: Classification[];
|
|
79
|
+
aiModelUsage: {inputTokens: number; outputTokens: number};
|
|
80
|
+
}>((resolve, reject) => {
|
|
81
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
82
|
+
if (!apiKey) {
|
|
83
|
+
reject(new Error('ANTHROPIC_API_KEY not set'));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const prompt =
|
|
87
|
+
'Classify HTML elements for an accessibility rule. All-capital text violates the rule UNLESS it is an acronym, abbreviation, or term whose standard form is all-capitals (NASA, WHO, etc.).\n\n'
|
|
88
|
+
+ 'For each element, give a confidence score (0.0–1.0, rounded to one decimal place) for the probability that the element VIOLATES the rule (its all-caps text is NOT intrinsically all-caps).\n\n'
|
|
89
|
+
+ 'Respond with ONLY a JSON array. Each element: {"index": <number>, "confidence": <number>}\n\n'
|
|
90
|
+
+ 'Elements:\n'
|
|
91
|
+
+ JSON.stringify(entries);
|
|
92
|
+
const payload = JSON.stringify({
|
|
93
|
+
model: 'claude-haiku-4-5-20251001',
|
|
94
|
+
max_tokens: 2048,
|
|
95
|
+
messages: [{role: 'user', content: prompt}]
|
|
96
|
+
});
|
|
97
|
+
const options = {
|
|
98
|
+
hostname: 'api.anthropic.com',
|
|
99
|
+
path: '/v1/messages',
|
|
100
|
+
method: 'POST',
|
|
101
|
+
headers: {
|
|
102
|
+
'x-api-key': apiKey,
|
|
103
|
+
'anthropic-version': '2023-06-01',
|
|
104
|
+
'content-type': 'application/json',
|
|
105
|
+
'content-length': Buffer.byteLength(payload)
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
const req = https.request(options, res => {
|
|
109
|
+
let body = '';
|
|
110
|
+
res.on('data', chunk => { body += chunk; });
|
|
111
|
+
res.on('end', () => {
|
|
112
|
+
try {
|
|
113
|
+
const parsed = JSON.parse(body);
|
|
114
|
+
if (parsed.error) {
|
|
115
|
+
reject(new Error(parsed.error.message));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const text = parsed.content[0].text;
|
|
119
|
+
const classifications = [...(text as string).matchAll(/\{"index":\s*\d+,\s*"confidence":\s*[01]\.\d{1,2}\}/g)]
|
|
120
|
+
.map(m => {
|
|
121
|
+
const {index, confidence} = JSON.parse(m[0]);
|
|
122
|
+
return {index, confidence: Math.round(confidence * 10) / 10};
|
|
123
|
+
});
|
|
124
|
+
const {input_tokens, output_tokens} = parsed.usage;
|
|
125
|
+
resolve({classifications, aiModelUsage: {inputTokens: input_tokens, outputTokens: output_tokens}});
|
|
126
|
+
}
|
|
127
|
+
catch(error) {
|
|
128
|
+
reject(new Error(`Haiku response error: ${(error as Error).message}`));
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
req.on('error', reject);
|
|
133
|
+
req.setTimeout(20000, () => req.destroy(new Error('Haiku API timeout')));
|
|
134
|
+
req.write(payload);
|
|
135
|
+
req.end();
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// Runs the test and returns the result.
|
|
139
|
+
export const reporter = async (_0: unknown, report: Report, _1: unknown, withItems: boolean) => {
|
|
140
|
+
const data: {
|
|
141
|
+
aiModelUsage?: {inputTokens: number; outputTokens: number};
|
|
142
|
+
leftOut?: {count: number; estimatedViolations: number};
|
|
143
|
+
aiError?: string;
|
|
144
|
+
} = {};
|
|
145
|
+
const totals = [0, 0, 0, 0];
|
|
146
|
+
const standardInstances: StandardInstance[] = [];
|
|
147
|
+
// Get data on the catalog entries whose text values contain 2+ consecutive capital letters.
|
|
148
|
+
const qualifying = Object.entries(report.catalog as Catalog)
|
|
149
|
+
.filter(([, entry]) => entry.text && /\p{Lu}{2,}/u.test(entry.text))
|
|
150
|
+
.map(([index, entry]) => ({
|
|
151
|
+
index: Number(index),
|
|
152
|
+
tagName: entry.tagName,
|
|
153
|
+
text: getContext(entry.text as string)
|
|
154
|
+
}));
|
|
155
|
+
// If there are none:
|
|
156
|
+
if (!qualifying.length) {
|
|
157
|
+
// Report this.
|
|
158
|
+
return {data, totals, standardInstances};
|
|
159
|
+
}
|
|
160
|
+
// Sort them, with unbiased (Fisher–Yates) randomization.
|
|
161
|
+
const sample = qualifying.slice();
|
|
162
|
+
for (let i = sample.length - 1; i > 0; i--) {
|
|
163
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
164
|
+
[sample[i], sample[j]] = [sample[j], sample[i]];
|
|
165
|
+
}
|
|
166
|
+
// If their count exceeds the limit on AI assistance:
|
|
167
|
+
if (sample.length > MAX_QUALIFYING) {
|
|
168
|
+
// Truncate them.
|
|
169
|
+
sample.length = MAX_QUALIFYING;
|
|
170
|
+
}
|
|
171
|
+
let violations: Violation[];
|
|
172
|
+
try {
|
|
173
|
+
// Get AI estimates of the probabilities of their violating the rule.
|
|
174
|
+
const {classifications, aiModelUsage} = await classifyWithAI(sample);
|
|
175
|
+
data.aiModelUsage = aiModelUsage;
|
|
176
|
+
// Treat the entries with above-minimum violation confidence levels as violations.
|
|
177
|
+
violations = classifications
|
|
178
|
+
.filter(({confidence}) => confidence >= MIN_CONFIDENCE)
|
|
179
|
+
.map(({index, confidence}) => ({
|
|
180
|
+
catalogIndex: String(index),
|
|
181
|
+
what: `Claude Haiku has ${Math.round(confidence * 100)}% confidence that the element contains unnecessarily all-capital text`
|
|
182
|
+
}));
|
|
183
|
+
const evaluated = classifications.length;
|
|
184
|
+
const leftOut = qualifying.length - evaluated;
|
|
185
|
+
// If any entries were truncated out and any AI confidence levels were above-minimum:
|
|
186
|
+
if (leftOut > 0 && evaluated > 0) {
|
|
187
|
+
// Add data about the estimated violation rate among those entries.
|
|
188
|
+
data.leftOut = {
|
|
189
|
+
count: leftOut,
|
|
190
|
+
estimatedViolations: Math.round((violations.length / evaluated) * leftOut)
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
catch(error) {
|
|
195
|
+
data.aiError = (error as Error).message;
|
|
196
|
+
violations = getRuleBasedViolations(report.catalog as Catalog);
|
|
197
|
+
}
|
|
198
|
+
const estimatedLeftOut = data.leftOut?.estimatedViolations ?? 0;
|
|
199
|
+
// Add the estimated violation count to the totals.
|
|
200
|
+
totals[0] = violations.length + estimatedLeftOut;
|
|
201
|
+
// The estimates are AI judgements, so every instance is uncertain.
|
|
202
|
+
const certainty = {outcome: 'cantTell', uncertainty: 'judgement-required'} as const;
|
|
203
|
+
// If itemization is required:
|
|
204
|
+
if (withItems) {
|
|
205
|
+
// For each entry deemed a violation:
|
|
206
|
+
for (const {catalogIndex, what} of violations) {
|
|
207
|
+
// Add an instance to the standard instances.
|
|
208
|
+
standardInstances.push(
|
|
209
|
+
getInstance({ruleID, what, ordinalSeverity: 0, catalogIndex, ... certainty})
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
// If any entries were truncated:
|
|
213
|
+
if (estimatedLeftOut) {
|
|
214
|
+
// Add a summary instance for them.
|
|
215
|
+
standardInstances.push(
|
|
216
|
+
getInstance({ruleID, what: whats, ordinalSeverity: 0, count: estimatedLeftOut, ... certainty})
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// Otherwise, i.e. if itemization is not required, and if any violations exist:
|
|
221
|
+
else if (totals[0]) {
|
|
222
|
+
// Add a summary instance for them.
|
|
223
|
+
standardInstances.push(
|
|
224
|
+
getInstance({ruleID, what: whats, ordinalSeverity: 0, count: totals[0], ... certainty})
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
return {data, totals, standardInstances};
|
|
228
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import type { Report } from '../types';
|
|
3
|
+
export declare const reporter: (page: Page, report: Report) => Promise<{
|
|
4
|
+
data: {};
|
|
5
|
+
totals: number[];
|
|
6
|
+
standardInstances: {
|
|
7
|
+
ruleID: string;
|
|
8
|
+
what: string;
|
|
9
|
+
ordinalSeverity: number;
|
|
10
|
+
count: number;
|
|
11
|
+
catalogIndex: string;
|
|
12
|
+
}[];
|
|
13
|
+
}>;
|