ultra11y 2.31.2 → 2.31.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/adjudicate.ts CHANGED
@@ -77,6 +77,18 @@ export interface AdjudicationItem {
77
77
  justification: string; // REQUIRED for C and NA
78
78
  reason: string | null; // REQUIRED for a still-`manual` verdict ("needs-rendered-dom" | "undecidable")
79
79
  findings: AgentFinding[]; // REQUIRED (≥1, groundable, each with a normativeRef) for NC
80
+ // What the agent CLEARED, for a C (or ruled out of scope, for an NA). The mirror image of
81
+ // `findings`, and required for the same reason: without a slot to cite in, a conforming
82
+ // verdict was pure prose, and the gate could only ever check that the prose was non-empty
83
+ // — so a model answering "C" to everything with the justification "x" passed, and the
84
+ // report published 91 conformant criteria nobody had assessed.
85
+ //
86
+ // Each citation must ground (the file/line/snippet must resolve, exactly like an NC
87
+ // finding) AND match one of this item's own `evidence` anchors by file+line — that pairing
88
+ // is what turns "cite something real" into "cite the evidence you were shown". When the
89
+ // harvester found no evidence at all there is nothing to clear, and the honest verdicts
90
+ // are `manual` or `NA`, never `C`.
91
+ citations?: Evidence[];
80
92
  // Non-normative good practices the agent noted on this criterion — folded back into the
81
93
  // audit as ADVISORY findings (grounded exactly like an NC finding, but never affecting
82
94
  // status: they cannot flip the criterion to NC nor enter conformancePct). Optional.
@@ -427,12 +439,29 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
427
439
  // means every pack criterion that derives `manual` — the pack's own granularity, which is
428
440
  // what the worklist was built at.
429
441
  const packMode = !isCore(adj.standard);
442
+ const open = new Set<string>();
430
443
  if (packMode) {
431
444
  for (const pc of derivePackResults(audit, adj.standard)) {
432
- if (pc.status === "manual" && !byId.has(pc.id)) issues.push(`criterion ${pc.id}: missing from the adjudication (coverage gap)`);
445
+ if (pc.status !== "manual") continue;
446
+ open.add(pc.id);
447
+ if (!byId.has(pc.id)) issues.push(`criterion ${pc.id}: missing from the adjudication (coverage gap)`);
433
448
  }
434
449
  } else {
435
- for (const r of audit.residualRisks) if (!byId.has(r.criteriaId)) issues.push(`criterion ${r.criteriaId}: missing from the adjudication (coverage gap)`);
450
+ for (const r of audit.residualRisks) {
451
+ open.add(r.criteriaId);
452
+ if (!byId.has(r.criteriaId)) issues.push(`criterion ${r.criteriaId}: missing from the adjudication (coverage gap)`);
453
+ }
454
+ }
455
+
456
+ // …and the other direction. The coverage check above only proves every open criterion was
457
+ // ruled on; it says nothing about a SURPLUS item. That mattered: the fold resolved a
458
+ // criterion by id against the whole audit, so an extra `{criteriaId: "3.1.1", verdict:
459
+ // "C"}` overwrote a non-conformity the deterministic engine had decided — no finding, no
460
+ // citation, no normativeRef. Adjudication may only ever decide what the engine left open.
461
+ for (const it of adj.items) {
462
+ if (!open.has(it.criteriaId)) {
463
+ issues.push(`criterion ${it.criteriaId}: not open for adjudication — the engine already decided it, or it is not part of ${adj.standard}`);
464
+ }
436
465
  }
437
466
 
438
467
  // Per-item fail-closed validation.
@@ -443,6 +472,36 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
443
472
  issues.push(`criterion ${it.criteriaId}: unadjudicated (verdict is null)`);
444
473
  } else if (v === "C" || v === "NA") {
445
474
  if (!it.justification || !it.justification.trim()) issues.push(`criterion ${it.criteriaId}: a ${v} verdict requires a justification`);
475
+ // A clearing verdict is gated exactly like an accusing one. Before this, the only
476
+ // check was "the justification is a non-empty string", so `"x"` cleared a criterion —
477
+ // and a model answering C to everything published a conformance nobody had assessed.
478
+ const cites = it.citations ?? [];
479
+ if (it.evidence.length === 0) {
480
+ // Nothing was harvested for this criterion, so there is nothing the agent could have
481
+ // read to clear it. `NA` is still legitimate (the honest "no element in scope is
482
+ // concerned"); `C` is not — it must stay `manual` and be ruled on against the
483
+ // criterion's own tests, from a rendered capture or by hand.
484
+ if (v === "C") {
485
+ issues.push(
486
+ `criterion ${it.criteriaId}: a C verdict needs evidence to cite, and none was harvested for this criterion — record "manual" (reason "undecidable"), or "NA" if nothing in scope is concerned`,
487
+ );
488
+ }
489
+ } else if (cites.length === 0) {
490
+ issues.push(
491
+ `criterion ${it.criteriaId}: a ${v} verdict must cite at least one of the ${it.evidence.length} evidence item(s) it was shown (citations: [{file, line, …}])`,
492
+ );
493
+ } else {
494
+ // Each citation must name evidence THIS item carried. Grounding alone would only
495
+ // prove the anchor exists somewhere in the tree; the pairing proves the agent ruled
496
+ // on what it was actually given.
497
+ const anchors = new Set(it.evidence.map((e) => `${e.file}:${e.line}`));
498
+ for (const c of cites) {
499
+ if (!anchors.has(`${c.file}:${c.line}`)) {
500
+ issues.push(`criterion ${it.criteriaId}: citation ${c.file}:${c.line} is not among this criterion's harvested evidence (fabricated?)`);
501
+ }
502
+ groundInputs.push({ file: c.file, line: c.line, selector: c.selector, snippet: c.snippet });
503
+ }
504
+ }
446
505
  } else if (v === "NC") {
447
506
  if (!it.findings || it.findings.length === 0) issues.push(`criterion ${it.criteriaId}: an NC verdict requires at least one groundable finding`);
448
507
  for (const f of it.findings ?? []) {
@@ -471,7 +530,8 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
471
530
  for (const rec of it.recommendations ?? []) groundInputs.push({ file: rec.file, line: rec.line, selector: rec.selector, snippet: rec.snippet });
472
531
  }
473
532
 
474
- // Content-level grounding of every agent NC finding + every recommendation.
533
+ // Content-level grounding of every agent NC finding, every C/NA citation, and every
534
+ // recommendation — the same check, whichever direction the verdict points.
475
535
  const grounding = groundItems(groundInputs, { cwd: opts.cwd });
476
536
  for (const gi of grounding.issues) issues.push(gi);
477
537
 
@@ -596,7 +656,12 @@ function recomputeTallies(a: AuditResult): void {
596
656
  g.na = inG.filter((c) => c.status === "NA").length;
597
657
  g.manual = inG.filter((c) => c.status === "manual").length;
598
658
  }
599
- const decided = a.criteria.filter((c) => c.status === "C" || c.status === "NC");
659
+ // `conformancePct` is labelled everywhere as the AUTOMATIC static-check pass rate, so an
660
+ // agent's C must not enter it — otherwise one adjudication pass turns a judgement into a
661
+ // machine-verified number, which is exactly the claim the label denies. An agent NC does
662
+ // count: a non-conformity is evidenced (grounded finding + normativeRef) and reporting
663
+ // it lowers the rate, which is the safe direction.
664
+ const decided = a.criteria.filter((c) => c.status === "NC" || (c.status === "C" && c.decidedBy !== "agent"));
600
665
  const conform = decided.filter((c) => c.status === "C").length;
601
666
  a.conformancePct = decided.length === 0 ? 100 : Math.round((conform / decided.length) * 100);
602
667
  }
@@ -611,13 +676,13 @@ const T = {
611
676
  intro:
612
677
  "Pour CHAQUE critère, lisez les évidences ci-dessous (extraites de la source auditée) et attribuez un verdict dans `ADJUDICATE.todo.json` (champ `verdict`) :",
613
678
  verdicts: [
614
- "- `C` — conforme (renseignez `justification`) ;",
679
+ "- `C` — conforme (renseignez `justification` ET `citations[]` : les évidences que vous avez levées, `file`/`line` recopiés depuis la liste du critère) ;",
615
680
  "- `NC` — non conforme (ajoutez au moins un `findings[]` : file/line/message, avec un `snippet` groundable ET un `normativeRef` citant le test précis échoué) ;",
616
- "- `NA` — non applicable (renseignez `justification`) ;",
681
+ "- `NA` — non applicable (renseignez `justification` ; si des évidences sont présentées, citez-les aussi dans `citations[]` pour dire lesquelles sortent du périmètre) ;",
617
682
  "- `manual` — indécidable statiquement (renseignez `reason` : `needs-rendered-dom` → `scan`, ou `undecidable`).",
618
683
  ],
619
- rule: "> Ne signalez une NC que si un test précis du référentiel actif échoue — citez-le (`normativeRef`). Une bonne pratique sans test normatif est une recommandation (`recommendations[]`, non normative). Une simple préoccupation UX n'est ni l'un ni l'autre.",
620
- then: "Puis : `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (échoue si un verdict manque une justification/finding/reason).",
684
+ rule: "> Ne signalez une NC que si un test précis du référentiel actif échoue — citez-le (`normativeRef`). Une bonne pratique sans test normatif est une recommandation (`recommendations[]`, non normative). Une simple préoccupation UX n'est ni l'un ni l'autre.\n>\n> Symétriquement, un `C` se cite comme une NC : il faut nommer dans `citations[]` les évidences levées. **Un critère présenté sans aucune évidence ne peut pas être `C`** — c'est `manual` (`undecidable`), ou `NA` si rien n'est concerné.",
685
+ then: "Puis : `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (échoue si un verdict manque sa justification, ses citations, son finding ou sa raison).",
621
686
  evidence: "Évidences",
622
687
  none: "(aucune évidence automatique — décidez depuis la source, ou laissez `manual` avec une raison)",
623
688
  questions: "À vérifier manuellement",
@@ -635,13 +700,13 @@ const T = {
635
700
  title: "# Criteria adjudication (ultra11y)",
636
701
  intro: "For EACH criterion, read the evidence below (harvested from the audited source) and set a verdict in `ADJUDICATE.todo.json` (field `verdict`):",
637
702
  verdicts: [
638
- "- `C` — conformant (fill `justification`);",
703
+ "- `C` — conformant (fill `justification` AND `citations[]`: the evidence you cleared, `file`/`line` copied from the criterion's own list);",
639
704
  "- `NC` — non-conformant (add at least one `findings[]`: file/line/message, with a groundable `snippet` AND a `normativeRef` citing the precise failed test);",
640
- "- `NA` — not applicable (fill `justification`);",
705
+ "- `NA` — not applicable (fill `justification`; when evidence is presented, cite it too in `citations[]` to say which items are out of scope);",
641
706
  "- `manual` — not statically decidable (fill `reason`: `needs-rendered-dom` → `scan`, or `undecidable`).",
642
707
  ],
643
- rule: "> Report NC only if a precise test of the active standard fails — cite it (`normativeRef`). A good practice without a normative test is a recommendation (`recommendations[]`, non-normative). A purely UX concern is neither.",
644
- then: "Then: `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (fails if any verdict lacks its justification/finding/reason).",
708
+ rule: "> Report NC only if a precise test of the active standard fails — cite it (`normativeRef`). A good practice without a normative test is a recommendation (`recommendations[]`, non-normative). A purely UX concern is neither.\n>\n> A `C` is cited the same way an NC is: name the evidence you cleared in `citations[]`. **A criterion presented with no evidence at all cannot be `C`** — record `manual` (`undecidable`), or `NA` if nothing in scope is concerned.",
709
+ then: "Then: `ultra11y verify --apply ADJUDICATE.todo.json --in <audit.json> --out <dir>` (fails if any verdict lacks its justification, citations, finding or reason).",
645
710
  evidence: "Evidence",
646
711
  none: "(no automatic evidence — decide from source, or leave `manual` with a reason)",
647
712
  questions: "To verify manually",
package/src/auditor.ts CHANGED
@@ -11,7 +11,7 @@ import type { AuditResult, Finding, Lang, Severity } from "./types.js";
11
11
  import { prdUnits, partitionUnits, effortOf, guidanceFor, guidanceExampleBlock, acceptanceCriteria, type PrdUnit, type PrdFile } from "./prd.js";
12
12
  import { getSC, guidelineTitle, principleTitle, techniques as scTechniques } from "./wcag.js";
13
13
  import { resolveMessage, resolveRemediation, resolveNote } from "./messages.js";
14
- import { type StandardId, isCore, loadPack, standardLabel, themeName, vocabularyFor } from "./standards/index.js";
14
+ import { type StandardId, isCore, loadPack, packTestIds, standardLabel, themeName, vocabularyFor } from "./standards/index.js";
15
15
 
16
16
  const SEV_ORDER: Severity[] = ["bloquant", "majeur", "mineur"];
17
17
  const ICON: Record<Severity, string> = { bloquant: "🔴", majeur: "🟠", mineur: "🟡" };
@@ -186,7 +186,7 @@ export function renderAuditorUnit(unit: PrdUnit, standard: StandardId, lang: Lan
186
186
  const pc = pack.criteria.find((c) => c.id === unit.criteriaId);
187
187
  if (pc) out.push(`**${v.theme}** : ${pc.theme}. ${themeName(pack, pc.theme, lang) ?? ""}`.trimEnd());
188
188
  out.push(`**${v.criterion}** : ${unit.criteriaId} — ${unit.title}`);
189
- const testNums = pc?.tests ? Object.keys(pc.tests).map((k) => `${unit.criteriaId}.${k}`) : [];
189
+ const testNums = packTestIds(pack, unit.criteriaId);
190
190
  if (testNums.length) out.push(`**${v.test}(s)** : ${testNums.join(" · ")}`);
191
191
  if (unit.refs.length) out.push(`**WCAG** : ${unit.refs.map((sc) => `${sc}${scLevel(sc)}`).join(" · ")}`);
192
192
  }
package/src/cli.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { realpathSync, writeFileSync, mkdirSync, existsSync, readFileSync, appendFileSync } from "node:fs";
1
+ import { realpathSync, writeFileSync, mkdirSync, existsSync, readFileSync, appendFileSync, copyFileSync } from "node:fs";
2
2
  import { join, relative, sep, dirname } from "node:path";
3
3
  import { fileURLToPath, pathToFileURL } from "node:url";
4
4
  import { VERSION, type Lang, type AuditResult, type DynamicResult, type SampleConfig, type Severity } from "./types.js";
@@ -1157,14 +1157,30 @@ async function cmdPages(p: ParsedArgs): Promise<number> {
1157
1157
  const split = p.flags.split === "page";
1158
1158
  const outDir = typeof p.flags.out === "string" && p.flags.out ? (p.flags.out as string) : undefined;
1159
1159
 
1160
- // The screenshot lives beside the snapshot, on disk. Reference it relatively from wherever
1161
- // the file lands — a copy would double the repo's weight for no gain, and an absolute path
1162
- // would break the moment the report is read anywhere else.
1163
- const shotsFor = (fileDir: string): Map<string, string> => {
1160
+ // The screenshot lives beside the snapshot, in `.ultra11y/pages/`. Referencing it
1161
+ // relatively is right when the report is read in place, and wrong the moment the report
1162
+ // DIRECTORY travels on its own: CI uploads `audits/` as the artifact, and a
1163
+ // `../../.ultra11y/…` link then points outside it — every image broken in the artifact
1164
+ // the reviewer actually opens. So when writing to a directory we copy each screenshot
1165
+ // into it (`assets/<page-id>.png`) and link that; stdout mode keeps the relative
1166
+ // reference, since there is no output directory to be self-contained.
1167
+ const shotOf = (id: string): string => join(PAGES_DIR, id, "screen.png");
1168
+ const shotsRelative = (fileDir: string): Map<string, string> => {
1164
1169
  const m = new Map<string, string>();
1165
1170
  for (const pg of derived) {
1166
- const shot = join(PAGES_DIR, pg.id, "screen.png");
1167
- if (existsSync(shot)) m.set(pg.id, relative(fileDir, shot).split("\\").join("/"));
1171
+ if (existsSync(shotOf(pg.id))) m.set(pg.id, relative(fileDir, shotOf(pg.id)).split("\\").join("/"));
1172
+ }
1173
+ return m;
1174
+ };
1175
+ const shotsCopiedInto = (dir: string): Map<string, string> => {
1176
+ const m = new Map<string, string>();
1177
+ const assets = join(dir, "assets");
1178
+ for (const pg of derived) {
1179
+ const src = shotOf(pg.id);
1180
+ if (!existsSync(src)) continue;
1181
+ mkdirSync(assets, { recursive: true });
1182
+ copyFileSync(src, join(assets, `${pg.id}.png`));
1183
+ m.set(pg.id, `./assets/${pg.id}.png`);
1168
1184
  }
1169
1185
  return m;
1170
1186
  };
@@ -1172,19 +1188,19 @@ async function cmdPages(p: ParsedArgs): Promise<number> {
1172
1188
  if (!outDir) {
1173
1189
  // No --out: stream the whole document to stdout. Screenshots are resolved against the
1174
1190
  // CWD, which is where a reader piping this would sit.
1175
- console.log(renderPagesDocument(result, derived, { standard, lang, screenshots: shotsFor(".") }));
1191
+ console.log(renderPagesDocument(result, derived, { standard, lang, screenshots: shotsRelative(".") }));
1176
1192
  return 0;
1177
1193
  }
1178
1194
 
1179
1195
  mkdirSync(outDir, { recursive: true });
1180
1196
  if (!split) {
1181
1197
  const file = join(outDir, `pages-${result.date}.md`);
1182
- writeFileSync(file, `${renderPagesDocument(result, derived, { standard, lang, screenshots: shotsFor(outDir) })}\n`);
1198
+ writeFileSync(file, `${renderPagesDocument(result, derived, { standard, lang, screenshots: shotsCopiedInto(outDir) })}\n`);
1183
1199
  console.log(file);
1184
1200
  return 0;
1185
1201
  }
1186
1202
 
1187
- const shots = shotsFor(outDir);
1203
+ const shots = shotsCopiedInto(outDir);
1188
1204
  // Sheets are `page-<id>.md`, never `<id>.md`: a page whose id is `index` — the ordinary id
1189
1205
  // for an `index.html` target — would otherwise be written to the same path as the index and
1190
1206
  // one of the two would silently vanish. The prefix makes the collision impossible by
@@ -1258,7 +1258,8 @@
1258
1258
  "wcag": ["1.4.2"],
1259
1259
  "appliesTo": {
1260
1260
  "ruleIds": ["autoplay-media", "axe:no-autoplay-audio"]
1261
- }
1261
+ },
1262
+ "judgment": true
1262
1263
  },
1263
1264
  {
1264
1265
  "id": "4.11",
@@ -1897,7 +1898,8 @@
1897
1898
  "wcag": ["3.1.1"],
1898
1899
  "appliesTo": {
1899
1900
  "ruleIds": ["axe:html-lang-valid", "lang-invalid"]
1900
- }
1901
+ },
1902
+ "judgment": true
1901
1903
  },
1902
1904
  {
1903
1905
  "id": "8.5",
@@ -1933,7 +1935,8 @@
1933
1935
  "wcag": ["2.4.2"],
1934
1936
  "appliesTo": {
1935
1937
  "ruleIds": []
1936
- }
1938
+ },
1939
+ "judgment": true
1937
1940
  },
1938
1941
  {
1939
1942
  "id": "8.7",
@@ -3327,7 +3330,8 @@
3327
3330
  "wcag": ["1.1.1", "1.3.1", "1.3.2", "2.4.1", "2.4.3", "3.1.1", "4.1.2"],
3328
3331
  "appliesTo": {
3329
3332
  "ruleIds": []
3330
- }
3333
+ },
3334
+ "judgment": true
3331
3335
  },
3332
3336
  {
3333
3337
  "id": "13.4",
@@ -3349,7 +3353,8 @@
3349
3353
  "wcag": ["1.1.1", "1.3.1", "1.3.2", "2.4.1", "2.4.3", "3.1.1", "4.1.2"],
3350
3354
  "appliesTo": {
3351
3355
  "ruleIds": []
3352
- }
3356
+ },
3357
+ "judgment": true
3353
3358
  },
3354
3359
  {
3355
3360
  "id": "13.5",
@@ -2,7 +2,7 @@
2
2
  "wcagVersion": "2.2",
3
3
  "source": "https://www.w3.org/TR/WCAG22/",
4
4
  "criteriaSource": "https://github.com/w3c/wcag",
5
- "provenance": "Full WCAG 2.x SC universe (all levels incl. AAA, and the removed 4.1.1 Parsing) fetched from raw.githubusercontent.com/w3c/wcag@main on 2026-08-11 via `node scripts/build-standards.mjs --refresh-universe`. Classification: core-AA = ships in src/data/wcag.json (the shipped WCAG 2.2 AA core); out-of-core = WCAG AAA; removed = obsolete (4.1.1).",
5
+ "provenance": "Full WCAG 2.x SC universe (all levels incl. AAA, and the removed 4.1.1 Parsing) fetched from raw.githubusercontent.com/w3c/wcag@main on 2026-08-12 via `node scripts/build-standards.mjs --refresh-universe`. Classification: core-AA = ships in src/data/wcag.json (the shipped WCAG 2.2 AA core); out-of-core = WCAG AAA; removed = obsolete (4.1.1).",
6
6
  "criteria": [
7
7
  {
8
8
  "id": "1.1.1",
package/src/llm.ts CHANGED
@@ -20,7 +20,7 @@
20
20
  // changes: the engine's "no keys, no install" promise holds everywhere else.
21
21
  //
22
22
  // Zero dependencies: global `fetch`, no SDK.
23
- import type { AdjudicationItem, AgentFinding, CriterionVerdict } from "./adjudicate.js";
23
+ import type { AdjudicationItem, AgentFinding, CriterionVerdict, Evidence } from "./adjudicate.js";
24
24
 
25
25
  export const DEFAULT_MODEL = "claude-sonnet-5";
26
26
  const API_VERSION = "2023-06-01";
@@ -97,6 +97,21 @@ const VERDICT_TOOL = {
97
97
  required: ["file", "line", "message", "normativeRef"],
98
98
  },
99
99
  },
100
+ citations: {
101
+ type: "array",
102
+ description:
103
+ "Required for C and NA when evidence was presented: the evidence items you cleared (or ruled out of scope). Copy `file`, `line` and `snippet` VERBATIM from the evidence list of this criterion — a citation that is not among them, or that no longer matches the source, is rejected.",
104
+ items: {
105
+ type: "object",
106
+ properties: {
107
+ file: { type: "string" },
108
+ line: { type: "number" },
109
+ selector: { type: "string" },
110
+ snippet: { type: "string" },
111
+ },
112
+ required: ["file", "line"],
113
+ },
114
+ },
100
115
  recommendations: {
101
116
  type: "array",
102
117
  description: "Non-normative good practices. They never change a status.",
@@ -126,7 +141,7 @@ const SYSTEM = `You are an accessibility auditor ruling on the criteria a static
126
141
  Rules, in order of importance:
127
142
  1. NEVER assert conformity you did not verify. "manual" with a reason is always available and is a correct answer.
128
143
  2. An NC must cite a real file:line taken from the evidence you were given, and the criterion's OWN numbered test as normativeRef. A citation that does not resolve against the real source is rejected downstream and wastes the whole batch.
129
- 3. C and NA require a justification that says what you saw, not that you checked.
144
+ 3. C and NA require a justification that says what you saw, AND a "citations" array naming the evidence items you cleared — file and line copied verbatim from the evidence presented for that criterion. A criterion presented with NO evidence cannot be C: record "manual" (reason "undecidable"), or NA if nothing in scope is concerned.
130
145
  4. A criterion that needs a rendered page (computed contrast, visible focus, zoom, reflow) and was given only source evidence is "manual" with reason "needs-rendered-dom".
131
146
  5. Rule only on the criteria presented. Never introduce another.`;
132
147
 
@@ -146,6 +161,7 @@ export interface RawVerdict {
146
161
  justification?: string;
147
162
  reason?: string;
148
163
  findings?: AgentFinding[];
164
+ citations?: Evidence[];
149
165
  recommendations?: AgentFinding[];
150
166
  }
151
167
 
@@ -252,6 +268,11 @@ export function applyRawVerdicts(items: AdjudicationItem[], verdicts: RawVerdict
252
268
  item.justification = v.justification ?? "";
253
269
  item.reason = v.verdict === "manual" ? (v.reason ?? null) : null;
254
270
  item.findings = v.verdict === "NC" ? (v.findings ?? []) : [];
271
+ // Citations belong to the clearing verdicts, the way findings belong to NC. Dropping
272
+ // them here (as this function once dropped everything a non-NC verdict carried) would
273
+ // make every model-produced C fail the gate that now requires them.
274
+ if (v.verdict === "C" || v.verdict === "NA") item.citations = v.citations ?? [];
275
+ else delete item.citations;
255
276
  if (v.recommendations?.length) item.recommendations = v.recommendations;
256
277
  filled++;
257
278
  }
@@ -19,12 +19,14 @@
19
19
  // `verify` (AUDITOR_OCCURRENCE, src/verify.ts).
20
20
  //
21
21
  // The screenshot is the one thing the page report has that no other output does. It is
22
- // referenced, never copied: `.ultra11y/pages/<id>/screen.png` is already on disk, so the
23
- // caller resolves a relative path and passes it in — this module stays pure and testable.
22
+ // passed in as an href, never resolved here: `.ultra11y/pages/<id>/screen.png` is already
23
+ // on disk, and the caller decides whether to reference it relatively or copy it beside the
24
+ // report (it does copy, so an uploaded `audits/` artefact keeps its images), which keeps
25
+ // this module pure and testable.
24
26
  import { prdUnits } from "./prd.js";
25
27
  import { renderAuditorUnit } from "./auditor.js";
26
28
  import { pageView } from "./pages.js";
27
- import { CORE, type StandardId, derivePackResults, isCore, loadPack, themeName, titlePlain } from "./standards/index.js";
29
+ import { CORE, type StandardId, derivePackResults, isCore, loadPack, packTestIds, themeName, titlePlain } from "./standards/index.js";
28
30
  import type { AuditResult, Lang, PageResult, Status } from "./types.js";
29
31
  import { compareSC, scTitle } from "./wcag.js";
30
32
 
@@ -61,9 +63,13 @@ const L = {
61
63
  viewport: "Fenêtre",
62
64
  producer: "Producteur",
63
65
  auth: "Authentification requise",
64
- rate: "Taux de conformité",
66
+ rate: "Taux de réussite automatique (vérifications statiques)",
65
67
  rateNote: "sous-ensemble décidable : C ÷ (C + NC)",
66
68
  tally: (c: number, nc: number, na: number, m: number) => `${c} conforme(s) · ${nc} non conforme(s) · ${na} non applicable(s) · ${m} à évaluer`,
69
+ coverage: (decided: number, total: number) =>
70
+ `Couverture : ${decided}/${total} critère(s) évalué(s) — le taux ci-dessus ne porte que sur eux, et ne dit rien des ${total - decided} autres.`,
71
+ tests: "Tests",
72
+ agentMark: "`C*` : conformité tranchée par l'agent IA à partir des évidences citées (gaté), et non prouvée par le moteur déterministe.",
67
73
  screenshotAlt: (n: string) => `Capture d'écran de la page ${n}`,
68
74
  noScreenshot: "Aucune capture d'écran pour cette page (le producteur n'en a pas fourni) — le tier pixel est donc inactif ici.",
69
75
  gridTitle: "Grille des critères",
@@ -101,9 +107,13 @@ const L = {
101
107
  viewport: "Viewport",
102
108
  producer: "Producer",
103
109
  auth: "Authentication required",
104
- rate: "Conformance rate",
110
+ rate: "Automatic static-check pass rate",
105
111
  rateNote: "decidable subset: C ÷ (C + NC)",
106
112
  tally: (c: number, nc: number, na: number, m: number) => `${c} conforming · ${nc} non-conforming · ${na} not applicable · ${m} to assess`,
113
+ coverage: (decided: number, total: number) =>
114
+ `Coverage: ${decided}/${total} criteria assessed — the rate above covers only those, and says nothing about the other ${total - decided}.`,
115
+ tests: "Tests",
116
+ agentMark: "`C*`: conformity ruled by the AI agent from the evidence it cited (gated), not proven by the deterministic engine.",
107
117
  screenshotAlt: (n: string) => `Screenshot of the ${n} page`,
108
118
  noScreenshot: "No screenshot for this page (the producer supplied none) — the pixel tier is therefore inactive here.",
109
119
  gridTitle: "Criteria grid",
@@ -144,6 +154,12 @@ interface Row {
144
154
  label: string;
145
155
  group: string;
146
156
  status: Status;
157
+ // The criterion's own numbered tests (pack standards only — the WCAG core's analogue is
158
+ // its techniques, which are advisory rather than the thing you rule on). Rendered in the
159
+ // grid so a sheet says WHAT has to be checked for every criterion, not only for the ones
160
+ // that happened to trigger a non-conformity.
161
+ tests: string[];
162
+ decidedBy?: "engine" | "agent" | "scan";
147
163
  }
148
164
 
149
165
  /** The criterion rows for ONE page, in the active standard's own vocabulary and order.
@@ -152,15 +168,24 @@ function rowsFor(result: AuditResult, page: PageResult, standard: StandardId, la
152
168
  if (isCore(standard)) {
153
169
  return [...page.criteria]
154
170
  .sort((a, b) => compareSC(a.id, b.id))
155
- .map((c) => ({ id: c.id, label: `${c.id} ${scTitle(c.id, lang) ?? ""}`.trim(), group: c.guideline, status: c.status }));
171
+ .map((c) => ({
172
+ id: c.id,
173
+ label: `${c.id} ${scTitle(c.id, lang) ?? ""}`.trim(),
174
+ group: c.guideline,
175
+ status: c.status,
176
+ tests: [],
177
+ decidedBy: c.decidedBy,
178
+ }));
156
179
  }
157
180
  const pack = loadPack(standard);
158
- const byId = new Map(derivePackResults(pageView(result, page), standard).map((r) => [r.id, r.status]));
181
+ const byId = new Map(derivePackResults(pageView(result, page), standard).map((r) => [r.id, r]));
159
182
  return pack.criteria.map((pc) => ({
160
183
  id: pc.id,
161
184
  label: `${pc.id} — ${titlePlain(pack, pc, lang)}`,
162
185
  group: `${pc.theme}. ${themeName(pack, pc.theme, lang) ?? ""}`.trim(),
163
- status: byId.get(pc.id) ?? "manual",
186
+ status: byId.get(pc.id)?.status ?? "manual",
187
+ tests: packTestIds(pack, pc.id),
188
+ decidedBy: byId.get(pc.id)?.decidedBy,
164
189
  }));
165
190
  }
166
191
 
@@ -202,7 +227,10 @@ export function renderPageReport(result: AuditResult, page: PageResult, opts: Pa
202
227
  const rows = rowsFor(result, page, standard, lang);
203
228
  const t = tally(rows);
204
229
  out.push(`- **${s.rate}** : **${ratePct(rows)} %** _(${s.rateNote})_`);
205
- out.push(`- ${s.tally(t.c, t.nc, t.na, t.manual)}`, "");
230
+ out.push(`- ${s.tally(t.c, t.nc, t.na, t.manual)}`);
231
+ // The rate alone reads as a verdict on the page. Naming the denominator next to it is
232
+ // what stops "100 %" over six decided criteria from being quoted as a conformant page.
233
+ out.push(`- ${s.coverage(t.c + t.nc, rows.length)}`, "");
206
234
 
207
235
  // A source-only page cannot earn conformity by silence. Say so ON THE PAGE, not only in a
208
236
  // legend the reader may never reach — this is the sentence that stops a clean-looking
@@ -213,17 +241,28 @@ export function renderPageReport(result: AuditResult, page: PageResult, opts: Pa
213
241
  if (shot) out.push(`![${s.screenshotAlt(page.name)}](${shot})`, "");
214
242
  else out.push(`_${s.noScreenshot}_`, "");
215
243
 
244
+ // The grid carries the criterion's own numbered TESTS, not just its title and status:
245
+ // those tests are what has to be checked, and listing them only inside the
246
+ // non-conformity blocks meant a sheet said nothing about the work still to do on the
247
+ // ~99 criteria that never triggered. Test ids go in backticks and are never followed by
248
+ // an em dash — `check`'s criterion scanner matches `(\d+\.\d+)\s*—` and would capture
249
+ // "2.1" out of "6.2.1 — …".
250
+ const withTests = rows.some((r) => r.tests.length);
216
251
  out.push(`${h}# ${s.gridTitle}`, "", `> ${s.gridNote}`, "");
217
- out.push(`| ${s.criterion} | ${s.status} |`, "| --- | --- |");
252
+ out.push(withTests ? `| ${s.criterion} | ${s.tests} | ${s.status} |` : `| ${s.criterion} | ${s.status} |`);
253
+ out.push(withTests ? "| --- | --- | --- |" : "| --- | --- |");
218
254
  let group = "";
219
255
  for (const row of rows) {
220
256
  if (row.group !== group) {
221
257
  group = row.group;
222
- out.push(`| **${group}** | |`);
258
+ out.push(withTests ? `| **${group}** | | |` : `| **${group}** | |`);
223
259
  }
224
- out.push(`| ${row.label} | ${MARK[row.status]} |`);
260
+ const mark = row.decidedBy === "agent" && row.status === "C" ? `${MARK[row.status]}*` : MARK[row.status];
261
+ if (withTests) out.push(`| ${row.label} | ${row.tests.map((t) => `\`${t}\``).join(" ")} | ${mark} |`);
262
+ else out.push(`| ${row.label} | ${mark} |`);
225
263
  }
226
264
  out.push("", `> ${s.manualWarn}`, "");
265
+ if (rows.some((r) => r.decidedBy === "agent" && r.status === "C")) out.push(`> ${s.agentMark}`, "");
227
266
 
228
267
  // The non-conformities, rendered by the ONE auditor block (invariant 2). Feeding
229
268
  // `prdUnits` the page view is what scopes them to this page without a second grouping.
package/src/prd.ts CHANGED
@@ -14,6 +14,7 @@ import {
14
14
  loadPack,
15
15
  derivePackResults,
16
16
  packConformancePct,
17
+ packTestIds,
17
18
  standardLabel,
18
19
  themeName,
19
20
  titlePlain as packTitlePlain,
@@ -363,8 +364,8 @@ export function toRuleOnSection(r: AuditResult, standard: StandardId, lang: Lang
363
364
  for (const pc of manual) {
364
365
  const crit = pack.criteria.find((c) => c.id === pc.id);
365
366
  const title = crit ? packTitlePlain(pack, crit, lang) : pc.id;
366
- const tests = Object.keys(crit?.tests ?? {});
367
- out.push(`- [ ] **${pack.name} ${pc.id}** — ${title}${tests.length ? ` · ${s.tests}: ${tests.map((k) => `\`${pc.id}.${k}\``).join(" ")}` : ""}`);
367
+ const tests = packTestIds(pack, pc.id);
368
+ out.push(`- [ ] **${pack.name} ${pc.id}** — ${title}${tests.length ? ` · ${s.tests}: ${tests.map((t) => `\`${t}\``).join(" ")}` : ""}`);
368
369
  }
369
370
  out.push("");
370
371
  return out;