ultra11y 2.31.2 → 2.31.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.opencode/plugins/ultra11y.js +1 -1
- package/package.json +1 -1
- package/scripts/build-pack-rgaa.mjs +31 -0
- package/scripts/ultra11y.mjs +183 -56
- package/scripts/vendor/wcag-2.2-fr.json +1 -1
- package/scripts/vendor/wcag-2.2-sc-universe.json +1 -1
- package/skills/review-a11y/SKILL.md +1 -1
- package/skills/review-a11y/scripts/ultra11y.mjs +183 -56
- package/skills/ultra11y/SKILL.md +10 -5
- package/skills/ultra11y/references/pages.md +7 -3
- package/skills/ultra11y/scripts/ultra11y.mjs +183 -56
- package/src/adjudicate.ts +77 -12
- package/src/auditor.ts +2 -2
- package/src/cli.ts +26 -10
- package/src/data/standards/rgaa.json +10 -5
- package/src/data/wcag-universe.json +1 -1
- package/src/llm.ts +23 -2
- package/src/pages-report.ts +51 -12
- package/src/prd.ts +3 -2
- package/src/report.ts +70 -19
- package/src/standards/derive.ts +33 -2
- package/src/standards/pack.ts +13 -0
- package/src/standards/types.ts +13 -0
- package/src/types.ts +1 -1
package/src/adjudicate.ts
CHANGED
|
@@ -77,6 +77,18 @@ export interface AdjudicationItem {
|
|
|
77
77
|
justification: string; // REQUIRED for C and NA
|
|
78
78
|
reason: string | null; // REQUIRED for a still-`manual` verdict ("needs-rendered-dom" | "undecidable")
|
|
79
79
|
findings: AgentFinding[]; // REQUIRED (≥1, groundable, each with a normativeRef) for NC
|
|
80
|
+
// What the agent CLEARED, for a C (or ruled out of scope, for an NA). The mirror image of
|
|
81
|
+
// `findings`, and required for the same reason: without a slot to cite in, a conforming
|
|
82
|
+
// verdict was pure prose, and the gate could only ever check that the prose was non-empty
|
|
83
|
+
// — so a model answering "C" to everything with the justification "x" passed, and the
|
|
84
|
+
// report published 91 conformant criteria nobody had assessed.
|
|
85
|
+
//
|
|
86
|
+
// Each citation must ground (the file/line/snippet must resolve, exactly like an NC
|
|
87
|
+
// finding) AND match one of this item's own `evidence` anchors by file+line — that pairing
|
|
88
|
+
// is what turns "cite something real" into "cite the evidence you were shown". When the
|
|
89
|
+
// harvester found no evidence at all there is nothing to clear, and the honest verdicts
|
|
90
|
+
// are `manual` or `NA`, never `C`.
|
|
91
|
+
citations?: Evidence[];
|
|
80
92
|
// Non-normative good practices the agent noted on this criterion — folded back into the
|
|
81
93
|
// audit as ADVISORY findings (grounded exactly like an NC finding, but never affecting
|
|
82
94
|
// status: they cannot flip the criterion to NC nor enter conformancePct). Optional.
|
|
@@ -427,12 +439,29 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
427
439
|
// means every pack criterion that derives `manual` — the pack's own granularity, which is
|
|
428
440
|
// what the worklist was built at.
|
|
429
441
|
const packMode = !isCore(adj.standard);
|
|
442
|
+
const open = new Set<string>();
|
|
430
443
|
if (packMode) {
|
|
431
444
|
for (const pc of derivePackResults(audit, adj.standard)) {
|
|
432
|
-
if (pc.status
|
|
445
|
+
if (pc.status !== "manual") continue;
|
|
446
|
+
open.add(pc.id);
|
|
447
|
+
if (!byId.has(pc.id)) issues.push(`criterion ${pc.id}: missing from the adjudication (coverage gap)`);
|
|
433
448
|
}
|
|
434
449
|
} else {
|
|
435
|
-
for (const r of audit.residualRisks)
|
|
450
|
+
for (const r of audit.residualRisks) {
|
|
451
|
+
open.add(r.criteriaId);
|
|
452
|
+
if (!byId.has(r.criteriaId)) issues.push(`criterion ${r.criteriaId}: missing from the adjudication (coverage gap)`);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// …and the other direction. The coverage check above only proves every open criterion was
|
|
457
|
+
// ruled on; it says nothing about a SURPLUS item. That mattered: the fold resolved a
|
|
458
|
+
// criterion by id against the whole audit, so an extra `{criteriaId: "3.1.1", verdict:
|
|
459
|
+
// "C"}` overwrote a non-conformity the deterministic engine had decided — no finding, no
|
|
460
|
+
// citation, no normativeRef. Adjudication may only ever decide what the engine left open.
|
|
461
|
+
for (const it of adj.items) {
|
|
462
|
+
if (!open.has(it.criteriaId)) {
|
|
463
|
+
issues.push(`criterion ${it.criteriaId}: not open for adjudication — the engine already decided it, or it is not part of ${adj.standard}`);
|
|
464
|
+
}
|
|
436
465
|
}
|
|
437
466
|
|
|
438
467
|
// Per-item fail-closed validation.
|
|
@@ -443,6 +472,36 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
443
472
|
issues.push(`criterion ${it.criteriaId}: unadjudicated (verdict is null)`);
|
|
444
473
|
} else if (v === "C" || v === "NA") {
|
|
445
474
|
if (!it.justification || !it.justification.trim()) issues.push(`criterion ${it.criteriaId}: a ${v} verdict requires a justification`);
|
|
475
|
+
// A clearing verdict is gated exactly like an accusing one. Before this, the only
|
|
476
|
+
// check was "the justification is a non-empty string", so `"x"` cleared a criterion —
|
|
477
|
+
// and a model answering C to everything published a conformance nobody had assessed.
|
|
478
|
+
const cites = it.citations ?? [];
|
|
479
|
+
if (it.evidence.length === 0) {
|
|
480
|
+
// Nothing was harvested for this criterion, so there is nothing the agent could have
|
|
481
|
+
// read to clear it. `NA` is still legitimate (the honest "no element in scope is
|
|
482
|
+
// concerned"); `C` is not — it must stay `manual` and be ruled on against the
|
|
483
|
+
// criterion's own tests, from a rendered capture or by hand.
|
|
484
|
+
if (v === "C") {
|
|
485
|
+
issues.push(
|
|
486
|
+
`criterion ${it.criteriaId}: a C verdict needs evidence to cite, and none was harvested for this criterion — record "manual" (reason "undecidable"), or "NA" if nothing in scope is concerned`,
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
} else if (cites.length === 0) {
|
|
490
|
+
issues.push(
|
|
491
|
+
`criterion ${it.criteriaId}: a ${v} verdict must cite at least one of the ${it.evidence.length} evidence item(s) it was shown (citations: [{file, line, …}])`,
|
|
492
|
+
);
|
|
493
|
+
} else {
|
|
494
|
+
// Each citation must name evidence THIS item carried. Grounding alone would only
|
|
495
|
+
// prove the anchor exists somewhere in the tree; the pairing proves the agent ruled
|
|
496
|
+
// on what it was actually given.
|
|
497
|
+
const anchors = new Set(it.evidence.map((e) => `${e.file}:${e.line}`));
|
|
498
|
+
for (const c of cites) {
|
|
499
|
+
if (!anchors.has(`${c.file}:${c.line}`)) {
|
|
500
|
+
issues.push(`criterion ${it.criteriaId}: citation ${c.file}:${c.line} is not among this criterion's harvested evidence (fabricated?)`);
|
|
501
|
+
}
|
|
502
|
+
groundInputs.push({ file: c.file, line: c.line, selector: c.selector, snippet: c.snippet });
|
|
503
|
+
}
|
|
504
|
+
}
|
|
446
505
|
} else if (v === "NC") {
|
|
447
506
|
if (!it.findings || it.findings.length === 0) issues.push(`criterion ${it.criteriaId}: an NC verdict requires at least one groundable finding`);
|
|
448
507
|
for (const f of it.findings ?? []) {
|
|
@@ -471,7 +530,8 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
471
530
|
for (const rec of it.recommendations ?? []) groundInputs.push({ file: rec.file, line: rec.line, selector: rec.selector, snippet: rec.snippet });
|
|
472
531
|
}
|
|
473
532
|
|
|
474
|
-
// Content-level grounding of every agent NC finding
|
|
533
|
+
// Content-level grounding of every agent NC finding, every C/NA citation, and every
|
|
534
|
+
// recommendation — the same check, whichever direction the verdict points.
|
|
475
535
|
const grounding = groundItems(groundInputs, { cwd: opts.cwd });
|
|
476
536
|
for (const gi of grounding.issues) issues.push(gi);
|
|
477
537
|
|
|
@@ -596,7 +656,12 @@ function recomputeTallies(a: AuditResult): void {
|
|
|
596
656
|
g.na = inG.filter((c) => c.status === "NA").length;
|
|
597
657
|
g.manual = inG.filter((c) => c.status === "manual").length;
|
|
598
658
|
}
|
|
599
|
-
|
|
659
|
+
// `conformancePct` is labelled everywhere as the AUTOMATIC static-check pass rate, so an
|
|
660
|
+
// agent's C must not enter it — otherwise one adjudication pass turns a judgement into a
|
|
661
|
+
// machine-verified number, which is exactly the claim the label denies. An agent NC does
|
|
662
|
+
// count: a non-conformity is evidenced (grounded finding + normativeRef) and reporting
|
|
663
|
+
// it lowers the rate, which is the safe direction.
|
|
664
|
+
const decided = a.criteria.filter((c) => c.status === "NC" || (c.status === "C" && c.decidedBy !== "agent"));
|
|
600
665
|
const conform = decided.filter((c) => c.status === "C").length;
|
|
601
666
|
a.conformancePct = decided.length === 0 ? 100 : Math.round((conform / decided.length) * 100);
|
|
602
667
|
}
|
|
@@ -611,13 +676,13 @@ const T = {
|
|
|
611
676
|
intro:
|
|
612
677
|
"Pour CHAQUE critère, lisez les évidences ci-dessous (extraites de la source auditée) et attribuez un verdict dans `ADJUDICATE.todo.json` (champ `verdict`) :",
|
|
613
678
|
verdicts: [
|
|
614
|
-
"- `C` — conforme (renseignez `justification`) ;",
|
|
679
|
+
"- `C` — conforme (renseignez `justification` ET `citations[]` : les évidences que vous avez levées, `file`/`line` recopiés depuis la liste du critère) ;",
|
|
615
680
|
"- `NC` — non conforme (ajoutez au moins un `findings[]` : file/line/message, avec un `snippet` groundable ET un `normativeRef` citant le test précis échoué) ;",
|
|
616
|
-
"- `NA` — non applicable (renseignez `justification`) ;",
|
|
681
|
+
"- `NA` — non applicable (renseignez `justification` ; si des évidences sont présentées, citez-les aussi dans `citations[]` pour dire lesquelles sortent du périmètre) ;",
|
|
617
682
|
"- `manual` — indécidable statiquement (renseignez `reason` : `needs-rendered-dom` → `scan`, ou `undecidable`).",
|
|
618
683
|
],
|
|
619
|
-
rule: "> Ne signalez une NC que si un test précis du référentiel actif échoue — citez-le (`normativeRef`). Une bonne pratique sans test normatif est une recommandation (`recommendations[]`, non normative). Une simple préoccupation UX n'est ni l'un ni l'autre.",
|
|
620
|
-
then: "Puis : `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (échoue si un verdict manque
|
|
684
|
+
rule: "> Ne signalez une NC que si un test précis du référentiel actif échoue — citez-le (`normativeRef`). Une bonne pratique sans test normatif est une recommandation (`recommendations[]`, non normative). Une simple préoccupation UX n'est ni l'un ni l'autre.\n>\n> Symétriquement, un `C` se cite comme une NC : il faut nommer dans `citations[]` les évidences levées. **Un critère présenté sans aucune évidence ne peut pas être `C`** — c'est `manual` (`undecidable`), ou `NA` si rien n'est concerné.",
|
|
685
|
+
then: "Puis : `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (échoue si un verdict manque sa justification, ses citations, son finding ou sa raison).",
|
|
621
686
|
evidence: "Évidences",
|
|
622
687
|
none: "(aucune évidence automatique — décidez depuis la source, ou laissez `manual` avec une raison)",
|
|
623
688
|
questions: "À vérifier manuellement",
|
|
@@ -635,13 +700,13 @@ const T = {
|
|
|
635
700
|
title: "# Criteria adjudication (ultra11y)",
|
|
636
701
|
intro: "For EACH criterion, read the evidence below (harvested from the audited source) and set a verdict in `ADJUDICATE.todo.json` (field `verdict`):",
|
|
637
702
|
verdicts: [
|
|
638
|
-
"- `C` — conformant (fill `justification`);",
|
|
703
|
+
"- `C` — conformant (fill `justification` AND `citations[]`: the evidence you cleared, `file`/`line` copied from the criterion's own list);",
|
|
639
704
|
"- `NC` — non-conformant (add at least one `findings[]`: file/line/message, with a groundable `snippet` AND a `normativeRef` citing the precise failed test);",
|
|
640
|
-
"- `NA` — not applicable (fill `justification`);",
|
|
705
|
+
"- `NA` — not applicable (fill `justification`; when evidence is presented, cite it too in `citations[]` to say which items are out of scope);",
|
|
641
706
|
"- `manual` — not statically decidable (fill `reason`: `needs-rendered-dom` → `scan`, or `undecidable`).",
|
|
642
707
|
],
|
|
643
|
-
rule: "> Report NC only if a precise test of the active standard fails — cite it (`normativeRef`). A good practice without a normative test is a recommendation (`recommendations[]`, non-normative). A purely UX concern is neither.",
|
|
644
|
-
then: "Then: `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (fails if any verdict lacks its justification
|
|
708
|
+
rule: "> Report NC only if a precise test of the active standard fails — cite it (`normativeRef`). A good practice without a normative test is a recommendation (`recommendations[]`, non-normative). A purely UX concern is neither.\n>\n> A `C` is cited the same way an NC is: name the evidence you cleared in `citations[]`. **A criterion presented with no evidence at all cannot be `C`** — record `manual` (`undecidable`), or `NA` if nothing in scope is concerned.",
|
|
709
|
+
then: "Then: `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (fails if any verdict lacks its justification, citations, finding or reason).",
|
|
645
710
|
evidence: "Evidence",
|
|
646
711
|
none: "(no automatic evidence — decide from source, or leave `manual` with a reason)",
|
|
647
712
|
questions: "To verify manually",
|
package/src/auditor.ts
CHANGED
|
@@ -11,7 +11,7 @@ import type { AuditResult, Finding, Lang, Severity } from "./types.js";
|
|
|
11
11
|
import { prdUnits, partitionUnits, effortOf, guidanceFor, guidanceExampleBlock, acceptanceCriteria, type PrdUnit, type PrdFile } from "./prd.js";
|
|
12
12
|
import { getSC, guidelineTitle, principleTitle, techniques as scTechniques } from "./wcag.js";
|
|
13
13
|
import { resolveMessage, resolveRemediation, resolveNote } from "./messages.js";
|
|
14
|
-
import { type StandardId, isCore, loadPack, standardLabel, themeName, vocabularyFor } from "./standards/index.js";
|
|
14
|
+
import { type StandardId, isCore, loadPack, packTestIds, standardLabel, themeName, vocabularyFor } from "./standards/index.js";
|
|
15
15
|
|
|
16
16
|
const SEV_ORDER: Severity[] = ["bloquant", "majeur", "mineur"];
|
|
17
17
|
const ICON: Record<Severity, string> = { bloquant: "🔴", majeur: "🟠", mineur: "🟡" };
|
|
@@ -186,7 +186,7 @@ export function renderAuditorUnit(unit: PrdUnit, standard: StandardId, lang: Lan
|
|
|
186
186
|
const pc = pack.criteria.find((c) => c.id === unit.criteriaId);
|
|
187
187
|
if (pc) out.push(`**${v.theme}** : ${pc.theme}. ${themeName(pack, pc.theme, lang) ?? ""}`.trimEnd());
|
|
188
188
|
out.push(`**${v.criterion}** : ${unit.criteriaId} — ${unit.title}`);
|
|
189
|
-
const testNums =
|
|
189
|
+
const testNums = packTestIds(pack, unit.criteriaId);
|
|
190
190
|
if (testNums.length) out.push(`**${v.test}(s)** : ${testNums.join(" · ")}`);
|
|
191
191
|
if (unit.refs.length) out.push(`**WCAG** : ${unit.refs.map((sc) => `${sc}${scLevel(sc)}`).join(" · ")}`);
|
|
192
192
|
}
|
package/src/cli.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { realpathSync, writeFileSync, mkdirSync, existsSync, readFileSync, appendFileSync } from "node:fs";
|
|
1
|
+
import { realpathSync, writeFileSync, mkdirSync, existsSync, readFileSync, appendFileSync, copyFileSync } from "node:fs";
|
|
2
2
|
import { join, relative, sep, dirname } from "node:path";
|
|
3
3
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4
4
|
import { VERSION, type Lang, type AuditResult, type DynamicResult, type SampleConfig, type Severity } from "./types.js";
|
|
@@ -1157,14 +1157,30 @@ async function cmdPages(p: ParsedArgs): Promise<number> {
|
|
|
1157
1157
|
const split = p.flags.split === "page";
|
|
1158
1158
|
const outDir = typeof p.flags.out === "string" && p.flags.out ? (p.flags.out as string) : undefined;
|
|
1159
1159
|
|
|
1160
|
-
// The screenshot lives beside the snapshot,
|
|
1161
|
-
//
|
|
1162
|
-
//
|
|
1163
|
-
|
|
1160
|
+
// The screenshot lives beside the snapshot, in `.ultra11y/pages/`. Referencing it
|
|
1161
|
+
// relatively is right when the report is read in place, and wrong the moment the report
|
|
1162
|
+
// DIRECTORY travels on its own: CI uploads `audits/` as the artifact, and a
|
|
1163
|
+
// `../../.ultra11y/…` link then points outside it — every image broken in the artifact
|
|
1164
|
+
// the reviewer actually opens. So when writing to a directory we copy each screenshot
|
|
1165
|
+
// into it (`assets/<page-id>.png`) and link that; stdout mode keeps the relative
|
|
1166
|
+
// reference, since there is no output directory to be self-contained.
|
|
1167
|
+
const shotOf = (id: string): string => join(PAGES_DIR, id, "screen.png");
|
|
1168
|
+
const shotsRelative = (fileDir: string): Map<string, string> => {
|
|
1164
1169
|
const m = new Map<string, string>();
|
|
1165
1170
|
for (const pg of derived) {
|
|
1166
|
-
|
|
1167
|
-
|
|
1171
|
+
if (existsSync(shotOf(pg.id))) m.set(pg.id, relative(fileDir, shotOf(pg.id)).split("\\").join("/"));
|
|
1172
|
+
}
|
|
1173
|
+
return m;
|
|
1174
|
+
};
|
|
1175
|
+
const shotsCopiedInto = (dir: string): Map<string, string> => {
|
|
1176
|
+
const m = new Map<string, string>();
|
|
1177
|
+
const assets = join(dir, "assets");
|
|
1178
|
+
for (const pg of derived) {
|
|
1179
|
+
const src = shotOf(pg.id);
|
|
1180
|
+
if (!existsSync(src)) continue;
|
|
1181
|
+
mkdirSync(assets, { recursive: true });
|
|
1182
|
+
copyFileSync(src, join(assets, `${pg.id}.png`));
|
|
1183
|
+
m.set(pg.id, `./assets/${pg.id}.png`);
|
|
1168
1184
|
}
|
|
1169
1185
|
return m;
|
|
1170
1186
|
};
|
|
@@ -1172,19 +1188,19 @@ async function cmdPages(p: ParsedArgs): Promise<number> {
|
|
|
1172
1188
|
if (!outDir) {
|
|
1173
1189
|
// No --out: stream the whole document to stdout. Screenshots are resolved against the
|
|
1174
1190
|
// CWD, which is where a reader piping this would sit.
|
|
1175
|
-
console.log(renderPagesDocument(result, derived, { standard, lang, screenshots:
|
|
1191
|
+
console.log(renderPagesDocument(result, derived, { standard, lang, screenshots: shotsRelative(".") }));
|
|
1176
1192
|
return 0;
|
|
1177
1193
|
}
|
|
1178
1194
|
|
|
1179
1195
|
mkdirSync(outDir, { recursive: true });
|
|
1180
1196
|
if (!split) {
|
|
1181
1197
|
const file = join(outDir, `pages-${result.date}.md`);
|
|
1182
|
-
writeFileSync(file, `${renderPagesDocument(result, derived, { standard, lang, screenshots:
|
|
1198
|
+
writeFileSync(file, `${renderPagesDocument(result, derived, { standard, lang, screenshots: shotsCopiedInto(outDir) })}\n`);
|
|
1183
1199
|
console.log(file);
|
|
1184
1200
|
return 0;
|
|
1185
1201
|
}
|
|
1186
1202
|
|
|
1187
|
-
const shots =
|
|
1203
|
+
const shots = shotsCopiedInto(outDir);
|
|
1188
1204
|
// Sheets are `page-<id>.md`, never `<id>.md`: a page whose id is `index` — the ordinary id
|
|
1189
1205
|
// for an `index.html` target — would otherwise be written to the same path as the index and
|
|
1190
1206
|
// one of the two would silently vanish. The prefix makes the collision impossible by
|
|
@@ -1258,7 +1258,8 @@
|
|
|
1258
1258
|
"wcag": ["1.4.2"],
|
|
1259
1259
|
"appliesTo": {
|
|
1260
1260
|
"ruleIds": ["autoplay-media", "axe:no-autoplay-audio"]
|
|
1261
|
-
}
|
|
1261
|
+
},
|
|
1262
|
+
"judgment": true
|
|
1262
1263
|
},
|
|
1263
1264
|
{
|
|
1264
1265
|
"id": "4.11",
|
|
@@ -1897,7 +1898,8 @@
|
|
|
1897
1898
|
"wcag": ["3.1.1"],
|
|
1898
1899
|
"appliesTo": {
|
|
1899
1900
|
"ruleIds": ["axe:html-lang-valid", "lang-invalid"]
|
|
1900
|
-
}
|
|
1901
|
+
},
|
|
1902
|
+
"judgment": true
|
|
1901
1903
|
},
|
|
1902
1904
|
{
|
|
1903
1905
|
"id": "8.5",
|
|
@@ -1933,7 +1935,8 @@
|
|
|
1933
1935
|
"wcag": ["2.4.2"],
|
|
1934
1936
|
"appliesTo": {
|
|
1935
1937
|
"ruleIds": []
|
|
1936
|
-
}
|
|
1938
|
+
},
|
|
1939
|
+
"judgment": true
|
|
1937
1940
|
},
|
|
1938
1941
|
{
|
|
1939
1942
|
"id": "8.7",
|
|
@@ -3327,7 +3330,8 @@
|
|
|
3327
3330
|
"wcag": ["1.1.1", "1.3.1", "1.3.2", "2.4.1", "2.4.3", "3.1.1", "4.1.2"],
|
|
3328
3331
|
"appliesTo": {
|
|
3329
3332
|
"ruleIds": []
|
|
3330
|
-
}
|
|
3333
|
+
},
|
|
3334
|
+
"judgment": true
|
|
3331
3335
|
},
|
|
3332
3336
|
{
|
|
3333
3337
|
"id": "13.4",
|
|
@@ -3349,7 +3353,8 @@
|
|
|
3349
3353
|
"wcag": ["1.1.1", "1.3.1", "1.3.2", "2.4.1", "2.4.3", "3.1.1", "4.1.2"],
|
|
3350
3354
|
"appliesTo": {
|
|
3351
3355
|
"ruleIds": []
|
|
3352
|
-
}
|
|
3356
|
+
},
|
|
3357
|
+
"judgment": true
|
|
3353
3358
|
},
|
|
3354
3359
|
{
|
|
3355
3360
|
"id": "13.5",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"wcagVersion": "2.2",
|
|
3
3
|
"source": "https://www.w3.org/TR/WCAG22/",
|
|
4
4
|
"criteriaSource": "https://github.com/w3c/wcag",
|
|
5
|
-
"provenance": "Full WCAG 2.x SC universe (all levels incl. AAA, and the removed 4.1.1 Parsing) fetched from raw.githubusercontent.com/w3c/wcag@main on 2026-08-
|
|
5
|
+
"provenance": "Full WCAG 2.x SC universe (all levels incl. AAA, and the removed 4.1.1 Parsing) fetched from raw.githubusercontent.com/w3c/wcag@main on 2026-08-12 via `node scripts/build-standards.mjs --refresh-universe`. Classification: core-AA = ships in src/data/wcag.json (the shipped WCAG 2.2 AA core); out-of-core = WCAG AAA; removed = obsolete (4.1.1).",
|
|
6
6
|
"criteria": [
|
|
7
7
|
{
|
|
8
8
|
"id": "1.1.1",
|
package/src/llm.ts
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// changes: the engine's "no keys, no install" promise holds everywhere else.
|
|
21
21
|
//
|
|
22
22
|
// Zero dependencies: global `fetch`, no SDK.
|
|
23
|
-
import type { AdjudicationItem, AgentFinding, CriterionVerdict } from "./adjudicate.js";
|
|
23
|
+
import type { AdjudicationItem, AgentFinding, CriterionVerdict, Evidence } from "./adjudicate.js";
|
|
24
24
|
|
|
25
25
|
export const DEFAULT_MODEL = "claude-sonnet-5";
|
|
26
26
|
const API_VERSION = "2023-06-01";
|
|
@@ -97,6 +97,21 @@ const VERDICT_TOOL = {
|
|
|
97
97
|
required: ["file", "line", "message", "normativeRef"],
|
|
98
98
|
},
|
|
99
99
|
},
|
|
100
|
+
citations: {
|
|
101
|
+
type: "array",
|
|
102
|
+
description:
|
|
103
|
+
"Required for C and NA when evidence was presented: the evidence items you cleared (or ruled out of scope). Copy `file`, `line` and `snippet` VERBATIM from the evidence list of this criterion — a citation that is not among them, or that no longer matches the source, is rejected.",
|
|
104
|
+
items: {
|
|
105
|
+
type: "object",
|
|
106
|
+
properties: {
|
|
107
|
+
file: { type: "string" },
|
|
108
|
+
line: { type: "number" },
|
|
109
|
+
selector: { type: "string" },
|
|
110
|
+
snippet: { type: "string" },
|
|
111
|
+
},
|
|
112
|
+
required: ["file", "line"],
|
|
113
|
+
},
|
|
114
|
+
},
|
|
100
115
|
recommendations: {
|
|
101
116
|
type: "array",
|
|
102
117
|
description: "Non-normative good practices. They never change a status.",
|
|
@@ -126,7 +141,7 @@ const SYSTEM = `You are an accessibility auditor ruling on the criteria a static
|
|
|
126
141
|
Rules, in order of importance:
|
|
127
142
|
1. NEVER assert conformity you did not verify. "manual" with a reason is always available and is a correct answer.
|
|
128
143
|
2. An NC must cite a real file:line taken from the evidence you were given, and the criterion's OWN numbered test as normativeRef. A citation that does not resolve against the real source is rejected downstream and wastes the whole batch.
|
|
129
|
-
3. C and NA require a justification that says what you saw,
|
|
144
|
+
3. C and NA require a justification that says what you saw, AND a "citations" array naming the evidence items you cleared — file and line copied verbatim from the evidence presented for that criterion. A criterion presented with NO evidence cannot be C: record "manual" (reason "undecidable"), or NA if nothing in scope is concerned.
|
|
130
145
|
4. A criterion that needs a rendered page (computed contrast, visible focus, zoom, reflow) and was given only source evidence is "manual" with reason "needs-rendered-dom".
|
|
131
146
|
5. Rule only on the criteria presented. Never introduce another.`;
|
|
132
147
|
|
|
@@ -146,6 +161,7 @@ export interface RawVerdict {
|
|
|
146
161
|
justification?: string;
|
|
147
162
|
reason?: string;
|
|
148
163
|
findings?: AgentFinding[];
|
|
164
|
+
citations?: Evidence[];
|
|
149
165
|
recommendations?: AgentFinding[];
|
|
150
166
|
}
|
|
151
167
|
|
|
@@ -252,6 +268,11 @@ export function applyRawVerdicts(items: AdjudicationItem[], verdicts: RawVerdict
|
|
|
252
268
|
item.justification = v.justification ?? "";
|
|
253
269
|
item.reason = v.verdict === "manual" ? (v.reason ?? null) : null;
|
|
254
270
|
item.findings = v.verdict === "NC" ? (v.findings ?? []) : [];
|
|
271
|
+
// Citations belong to the clearing verdicts, the way findings belong to NC. Dropping
|
|
272
|
+
// them here (as this function once dropped everything a non-NC verdict carried) would
|
|
273
|
+
// make every model-produced C fail the gate that now requires them.
|
|
274
|
+
if (v.verdict === "C" || v.verdict === "NA") item.citations = v.citations ?? [];
|
|
275
|
+
else delete item.citations;
|
|
255
276
|
if (v.recommendations?.length) item.recommendations = v.recommendations;
|
|
256
277
|
filled++;
|
|
257
278
|
}
|
package/src/pages-report.ts
CHANGED
|
@@ -19,12 +19,14 @@
|
|
|
19
19
|
// `verify` (AUDITOR_OCCURRENCE, src/verify.ts).
|
|
20
20
|
//
|
|
21
21
|
// The screenshot is the one thing the page report has that no other output does. It is
|
|
22
|
-
//
|
|
23
|
-
//
|
|
22
|
+
// passed in as an href, never resolved here: `.ultra11y/pages/<id>/screen.png` is already
|
|
23
|
+
// on disk, and the caller decides whether to reference it relatively or copy it beside the
|
|
24
|
+
// report (it does copy, so an uploaded `audits/` artefact keeps its images), which keeps
|
|
25
|
+
// this module pure and testable.
|
|
24
26
|
import { prdUnits } from "./prd.js";
|
|
25
27
|
import { renderAuditorUnit } from "./auditor.js";
|
|
26
28
|
import { pageView } from "./pages.js";
|
|
27
|
-
import { CORE, type StandardId, derivePackResults, isCore, loadPack, themeName, titlePlain } from "./standards/index.js";
|
|
29
|
+
import { CORE, type StandardId, derivePackResults, isCore, loadPack, packTestIds, themeName, titlePlain } from "./standards/index.js";
|
|
28
30
|
import type { AuditResult, Lang, PageResult, Status } from "./types.js";
|
|
29
31
|
import { compareSC, scTitle } from "./wcag.js";
|
|
30
32
|
|
|
@@ -61,9 +63,13 @@ const L = {
|
|
|
61
63
|
viewport: "Fenêtre",
|
|
62
64
|
producer: "Producteur",
|
|
63
65
|
auth: "Authentification requise",
|
|
64
|
-
rate: "Taux de
|
|
66
|
+
rate: "Taux de réussite automatique (vérifications statiques)",
|
|
65
67
|
rateNote: "sous-ensemble décidable : C ÷ (C + NC)",
|
|
66
68
|
tally: (c: number, nc: number, na: number, m: number) => `${c} conforme(s) · ${nc} non conforme(s) · ${na} non applicable(s) · ${m} à évaluer`,
|
|
69
|
+
coverage: (decided: number, total: number) =>
|
|
70
|
+
`Couverture : ${decided}/${total} critère(s) évalué(s) — le taux ci-dessus ne porte que sur eux, et ne dit rien des ${total - decided} autres.`,
|
|
71
|
+
tests: "Tests",
|
|
72
|
+
agentMark: "`C*` : conformité tranchée par l'agent IA à partir des évidences citées (gaté), et non prouvée par le moteur déterministe.",
|
|
67
73
|
screenshotAlt: (n: string) => `Capture d'écran de la page ${n}`,
|
|
68
74
|
noScreenshot: "Aucune capture d'écran pour cette page (le producteur n'en a pas fourni) — le tier pixel est donc inactif ici.",
|
|
69
75
|
gridTitle: "Grille des critères",
|
|
@@ -101,9 +107,13 @@ const L = {
|
|
|
101
107
|
viewport: "Viewport",
|
|
102
108
|
producer: "Producer",
|
|
103
109
|
auth: "Authentication required",
|
|
104
|
-
rate: "
|
|
110
|
+
rate: "Automatic static-check pass rate",
|
|
105
111
|
rateNote: "decidable subset: C ÷ (C + NC)",
|
|
106
112
|
tally: (c: number, nc: number, na: number, m: number) => `${c} conforming · ${nc} non-conforming · ${na} not applicable · ${m} to assess`,
|
|
113
|
+
coverage: (decided: number, total: number) =>
|
|
114
|
+
`Coverage: ${decided}/${total} criteria assessed — the rate above covers only those, and says nothing about the other ${total - decided}.`,
|
|
115
|
+
tests: "Tests",
|
|
116
|
+
agentMark: "`C*`: conformity ruled by the AI agent from the evidence it cited (gated), not proven by the deterministic engine.",
|
|
107
117
|
screenshotAlt: (n: string) => `Screenshot of the ${n} page`,
|
|
108
118
|
noScreenshot: "No screenshot for this page (the producer supplied none) — the pixel tier is therefore inactive here.",
|
|
109
119
|
gridTitle: "Criteria grid",
|
|
@@ -144,6 +154,12 @@ interface Row {
|
|
|
144
154
|
label: string;
|
|
145
155
|
group: string;
|
|
146
156
|
status: Status;
|
|
157
|
+
// The criterion's own numbered tests (pack standards only — the WCAG core's analogue is
|
|
158
|
+
// its techniques, which are advisory rather than the thing you rule on). Rendered in the
|
|
159
|
+
// grid so a sheet says WHAT has to be checked for every criterion, not only for the ones
|
|
160
|
+
// that happened to trigger a non-conformity.
|
|
161
|
+
tests: string[];
|
|
162
|
+
decidedBy?: "engine" | "agent" | "scan";
|
|
147
163
|
}
|
|
148
164
|
|
|
149
165
|
/** The criterion rows for ONE page, in the active standard's own vocabulary and order.
|
|
@@ -152,15 +168,24 @@ function rowsFor(result: AuditResult, page: PageResult, standard: StandardId, la
|
|
|
152
168
|
if (isCore(standard)) {
|
|
153
169
|
return [...page.criteria]
|
|
154
170
|
.sort((a, b) => compareSC(a.id, b.id))
|
|
155
|
-
.map((c) => ({
|
|
171
|
+
.map((c) => ({
|
|
172
|
+
id: c.id,
|
|
173
|
+
label: `${c.id} ${scTitle(c.id, lang) ?? ""}`.trim(),
|
|
174
|
+
group: c.guideline,
|
|
175
|
+
status: c.status,
|
|
176
|
+
tests: [],
|
|
177
|
+
decidedBy: c.decidedBy,
|
|
178
|
+
}));
|
|
156
179
|
}
|
|
157
180
|
const pack = loadPack(standard);
|
|
158
|
-
const byId = new Map(derivePackResults(pageView(result, page), standard).map((r) => [r.id, r
|
|
181
|
+
const byId = new Map(derivePackResults(pageView(result, page), standard).map((r) => [r.id, r]));
|
|
159
182
|
return pack.criteria.map((pc) => ({
|
|
160
183
|
id: pc.id,
|
|
161
184
|
label: `${pc.id} — ${titlePlain(pack, pc, lang)}`,
|
|
162
185
|
group: `${pc.theme}. ${themeName(pack, pc.theme, lang) ?? ""}`.trim(),
|
|
163
|
-
status: byId.get(pc.id) ?? "manual",
|
|
186
|
+
status: byId.get(pc.id)?.status ?? "manual",
|
|
187
|
+
tests: packTestIds(pack, pc.id),
|
|
188
|
+
decidedBy: byId.get(pc.id)?.decidedBy,
|
|
164
189
|
}));
|
|
165
190
|
}
|
|
166
191
|
|
|
@@ -202,7 +227,10 @@ export function renderPageReport(result: AuditResult, page: PageResult, opts: Pa
|
|
|
202
227
|
const rows = rowsFor(result, page, standard, lang);
|
|
203
228
|
const t = tally(rows);
|
|
204
229
|
out.push(`- **${s.rate}** : **${ratePct(rows)} %** _(${s.rateNote})_`);
|
|
205
|
-
out.push(`- ${s.tally(t.c, t.nc, t.na, t.manual)}
|
|
230
|
+
out.push(`- ${s.tally(t.c, t.nc, t.na, t.manual)}`);
|
|
231
|
+
// The rate alone reads as a verdict on the page. Naming the denominator next to it is
|
|
232
|
+
// what stops "100 %" over six decided criteria from being quoted as a conformant page.
|
|
233
|
+
out.push(`- ${s.coverage(t.c + t.nc, rows.length)}`, "");
|
|
206
234
|
|
|
207
235
|
// A source-only page cannot earn conformity by silence. Say so ON THE PAGE, not only in a
|
|
208
236
|
// legend the reader may never reach — this is the sentence that stops a clean-looking
|
|
@@ -213,17 +241,28 @@ export function renderPageReport(result: AuditResult, page: PageResult, opts: Pa
|
|
|
213
241
|
if (shot) out.push(``, "");
|
|
214
242
|
else out.push(`_${s.noScreenshot}_`, "");
|
|
215
243
|
|
|
244
|
+
// The grid carries the criterion's own numbered TESTS, not just its title and status:
|
|
245
|
+
// those tests are what has to be checked, and listing them only inside the
|
|
246
|
+
// non-conformity blocks meant a sheet said nothing about the work still to do on the
|
|
247
|
+
// ~99 criteria that never triggered. Test ids go in backticks and are never followed by
|
|
248
|
+
// an em dash — `check`'s criterion scanner matches `(\d+\.\d+)\s*—` and would capture
|
|
249
|
+
// "2.1" out of "6.2.1 — …".
|
|
250
|
+
const withTests = rows.some((r) => r.tests.length);
|
|
216
251
|
out.push(`${h}# ${s.gridTitle}`, "", `> ${s.gridNote}`, "");
|
|
217
|
-
out.push(`| ${s.criterion} | ${s.status}
|
|
252
|
+
out.push(withTests ? `| ${s.criterion} | ${s.tests} | ${s.status} |` : `| ${s.criterion} | ${s.status} |`);
|
|
253
|
+
out.push(withTests ? "| --- | --- | --- |" : "| --- | --- |");
|
|
218
254
|
let group = "";
|
|
219
255
|
for (const row of rows) {
|
|
220
256
|
if (row.group !== group) {
|
|
221
257
|
group = row.group;
|
|
222
|
-
out.push(`| **${group}** | |`);
|
|
258
|
+
out.push(withTests ? `| **${group}** | | |` : `| **${group}** | |`);
|
|
223
259
|
}
|
|
224
|
-
|
|
260
|
+
const mark = row.decidedBy === "agent" && row.status === "C" ? `${MARK[row.status]}*` : MARK[row.status];
|
|
261
|
+
if (withTests) out.push(`| ${row.label} | ${row.tests.map((t) => `\`${t}\``).join(" ")} | ${mark} |`);
|
|
262
|
+
else out.push(`| ${row.label} | ${mark} |`);
|
|
225
263
|
}
|
|
226
264
|
out.push("", `> ${s.manualWarn}`, "");
|
|
265
|
+
if (rows.some((r) => r.decidedBy === "agent" && r.status === "C")) out.push(`> ${s.agentMark}`, "");
|
|
227
266
|
|
|
228
267
|
// The non-conformities, rendered by the ONE auditor block (invariant 2). Feeding
|
|
229
268
|
// `prdUnits` the page view is what scopes them to this page without a second grouping.
|
package/src/prd.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
loadPack,
|
|
15
15
|
derivePackResults,
|
|
16
16
|
packConformancePct,
|
|
17
|
+
packTestIds,
|
|
17
18
|
standardLabel,
|
|
18
19
|
themeName,
|
|
19
20
|
titlePlain as packTitlePlain,
|
|
@@ -363,8 +364,8 @@ export function toRuleOnSection(r: AuditResult, standard: StandardId, lang: Lang
|
|
|
363
364
|
for (const pc of manual) {
|
|
364
365
|
const crit = pack.criteria.find((c) => c.id === pc.id);
|
|
365
366
|
const title = crit ? packTitlePlain(pack, crit, lang) : pc.id;
|
|
366
|
-
const tests =
|
|
367
|
-
out.push(`- [ ] **${pack.name} ${pc.id}** — ${title}${tests.length ? ` · ${s.tests}: ${tests.map((
|
|
367
|
+
const tests = packTestIds(pack, pc.id);
|
|
368
|
+
out.push(`- [ ] **${pack.name} ${pc.id}** — ${title}${tests.length ? ` · ${s.tests}: ${tests.map((t) => `\`${t}\``).join(" ")}` : ""}`);
|
|
368
369
|
}
|
|
369
370
|
out.push("");
|
|
370
371
|
return out;
|