ultra11y 5.3.3 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.opencode/plugins/ultra11y.js +1 -1
- package/package.json +1 -1
- package/scripts/ultra11y.mjs +1721 -1272
- package/skills/review-a11y/SKILL.md +1 -1
- package/skills/review-a11y/scripts/ultra11y.mjs +1721 -1272
- package/skills/ultra11y/SKILL.md +2 -2
- package/skills/ultra11y/references/ci.md +46 -4
- package/skills/ultra11y/references/judgment.md +50 -12
- package/skills/ultra11y/references/pages.md +13 -4
- package/skills/ultra11y/scripts/ultra11y.mjs +1721 -1272
- package/src/adjudicate.ts +229 -41
- package/src/audit.ts +163 -2
- package/src/cli.ts +216 -18
- package/src/ledger.ts +250 -0
- package/src/orchestrate-templates.ts +2 -2
- package/src/report.ts +24 -6
- package/src/rules/rendered.ts +39 -1
- package/src/types.ts +10 -3
package/src/adjudicate.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// C/NA, no ungroundable NC, no reasonless manual, full coverage of the residual set. The
|
|
11
11
|
// decisions are the AGENT's, statically, gated — not a deferral to a human.
|
|
12
12
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
13
|
-
import { join } from "node:path";
|
|
13
|
+
import { join, resolve } from "node:path";
|
|
14
14
|
import type { AuditResult, Automatability, Finding, Lang, PackCriterionAdjudication, ResidualRisk, Severity, Status } from "./types.js";
|
|
15
15
|
import { SCHEMA_VERSION } from "./types.js";
|
|
16
16
|
import { discover } from "./discover.js";
|
|
@@ -20,7 +20,7 @@ import { type Doc, type El, elementsByTag, attr, textContent, ancestors, snippet
|
|
|
20
20
|
import { parseInlineStyle } from "./color.js";
|
|
21
21
|
import { ADJUDICATION } from "./adjudication-data.js";
|
|
22
22
|
import { scTitle, getSC, hasSC, techniquesFor, allSC, guidelineTitle } from "./wcag.js";
|
|
23
|
-
import {
|
|
23
|
+
import { groundFinding, type GroundingSummary } from "./grounding.js";
|
|
24
24
|
import {
|
|
25
25
|
type StandardId,
|
|
26
26
|
CORE,
|
|
@@ -331,7 +331,13 @@ function docsForAudit(audit: AuditResult, cwd?: string): Doc[] {
|
|
|
331
331
|
const docs: Doc[] = [];
|
|
332
332
|
for (const f of files) {
|
|
333
333
|
try {
|
|
334
|
-
|
|
334
|
+
// `resolve`, not `join`: a cwd resolves RELATIVE paths and must leave an absolute one
|
|
335
|
+
// alone. `join("/repo", "/tmp/x/page.html")` produced `/repo/tmp/x/page.html`, which is
|
|
336
|
+
// unreadable — and the catch below swallows it, so every criterion silently arrived with
|
|
337
|
+
// ZERO evidence. The gate then refused each C verdict for "no evidence harvested",
|
|
338
|
+
// blaming the adjudicator for a path bug. Grounding has always used `resolve` for the
|
|
339
|
+
// same reason; these two must agree, since one harvests the anchors the other checks.
|
|
340
|
+
docs.push(parseSource(readText(cwd ? resolve(cwd, f) : f), f));
|
|
335
341
|
} catch {
|
|
336
342
|
/* unreadable — skip, mirrors runAudit */
|
|
337
343
|
}
|
|
@@ -413,6 +419,13 @@ export interface ApplyAdjudicationResult {
|
|
|
413
419
|
issues: string[];
|
|
414
420
|
applied: number;
|
|
415
421
|
stillManual: number;
|
|
422
|
+
/** Criteria whose verdict was REFUSED by the gate and therefore not applied. They stay
|
|
423
|
+
* « to assess », each carrying the refusal as its residual reason. Always 0 in strict mode,
|
|
424
|
+
* where a single refusal discards the whole fold. */
|
|
425
|
+
rejected: number;
|
|
426
|
+
/** The ids behind `rejected`, in file order — so a caller can name them without re-parsing
|
|
427
|
+
* `issues` (one criterion can raise several). */
|
|
428
|
+
rejectedCriteria: string[];
|
|
416
429
|
grounding: GroundingSummary;
|
|
417
430
|
}
|
|
418
431
|
|
|
@@ -504,11 +517,51 @@ function normativeRefResolves(ref: string | undefined, standard: StandardId, ite
|
|
|
504
517
|
return itemCriterionId === undefined || critId === itemCriterionId;
|
|
505
518
|
}
|
|
506
519
|
|
|
507
|
-
/** Fold an adjudication file back into the audit.
|
|
508
|
-
*
|
|
509
|
-
*
|
|
510
|
-
|
|
520
|
+
/** Fold an adjudication file back into the audit. Returns a NEW AuditResult with the decided
|
|
521
|
+
* statuses, agent findings, recomputed conformancePct, a shrunk residual set, and the
|
|
522
|
+
* `adjudicated` marker.
|
|
523
|
+
*
|
|
524
|
+
* FAIL-CLOSED PER VERDICT, which is not the same thing as fail-closed per FILE. Every check
|
|
525
|
+
* below is unchanged and no refused verdict is ever applied — but a refusal now costs its own
|
|
526
|
+
* criterion and nothing else. The all-or-nothing fold was measured to be worse than useless:
|
|
527
|
+
* a CI run that filled 95 of 96 verdicts correctly had all 96 discarded, so the audit paid a
|
|
528
|
+
* full adjudication and published « to assess » across the board. Since the point of the gate
|
|
529
|
+
* is to keep an unproven verdict out of the report — not to punish the ones that proved
|
|
530
|
+
* themselves — a refused criterion simply stays « to assess », carrying the refusal as its
|
|
531
|
+
* residual reason so the next run knows what to fix.
|
|
532
|
+
*
|
|
533
|
+
* `strict: true` restores the old file-level behaviour for a caller that genuinely wants
|
|
534
|
+
* all-or-nothing (a conformance deliverable signed off in one pass). */
|
|
535
|
+
export function applyAdjudication(
|
|
536
|
+
audit: AuditResult,
|
|
537
|
+
adj: AdjudicationFile,
|
|
538
|
+
opts: {
|
|
539
|
+
cwd?: string;
|
|
540
|
+
strict?: boolean;
|
|
541
|
+
/** Criteria the caller KNOWS this adjudication does not cover, each with the reason to
|
|
542
|
+
* record. A ledger replay supplies them (« stale », « never adjudicated ») so an absence
|
|
543
|
+
* it fully expected is not reported as a coverage violation — while the criterion still
|
|
544
|
+
* stays to assess, carrying that reason instead of a blank cell. */
|
|
545
|
+
residualReasons?: Record<string, string>;
|
|
546
|
+
} = {},
|
|
547
|
+
): ApplyAdjudicationResult {
|
|
511
548
|
const issues: string[] = [];
|
|
549
|
+
const expected = opts.residualReasons ?? {};
|
|
550
|
+
// Per-criterion attribution. The gate used to collect one flat list, which is why it could
|
|
551
|
+
// only ever reject the whole file: an issue did not know which verdict it condemned.
|
|
552
|
+
const itemIssues = new Map<string, string[]>();
|
|
553
|
+
// Criteria that will NOT fold and must keep `manual` with a reason — the refused ones plus
|
|
554
|
+
// the ones the caller declared uncovered.
|
|
555
|
+
const notFolded = new Set<string>();
|
|
556
|
+
const blame = (criteriaId: string, issue: string) => {
|
|
557
|
+
issues.push(issue);
|
|
558
|
+
notFolded.add(criteriaId);
|
|
559
|
+
const list = itemIssues.get(criteriaId);
|
|
560
|
+
if (list) list.push(issue);
|
|
561
|
+
else itemIssues.set(criteriaId, [issue]);
|
|
562
|
+
};
|
|
563
|
+
/** An open criterion this adjudication deliberately does not carry. Not a gate violation. */
|
|
564
|
+
const uncovered = (criteriaId: string) => notFolded.add(criteriaId);
|
|
512
565
|
// Spelling first, decisions second: "na" is NA, and rejecting the run over the case of a
|
|
513
566
|
// verdict taught the caller nothing about accessibility. Everything below stays fail-closed.
|
|
514
567
|
canonicalizeAdjudication(adj);
|
|
@@ -523,12 +576,16 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
523
576
|
for (const pc of derivePackResults(audit, adj.standard)) {
|
|
524
577
|
if (pc.status !== "manual") continue;
|
|
525
578
|
open.add(pc.id);
|
|
526
|
-
if (
|
|
579
|
+
if (byId.has(pc.id)) continue;
|
|
580
|
+
if (Object.hasOwn(expected, pc.id)) uncovered(pc.id);
|
|
581
|
+
else blame(pc.id, `criterion ${pc.id}: missing from the adjudication (coverage gap)`);
|
|
527
582
|
}
|
|
528
583
|
} else {
|
|
529
584
|
for (const r of audit.residualRisks) {
|
|
530
585
|
open.add(r.criteriaId);
|
|
531
|
-
if (
|
|
586
|
+
if (byId.has(r.criteriaId)) continue;
|
|
587
|
+
if (Object.hasOwn(expected, r.criteriaId)) uncovered(r.criteriaId);
|
|
588
|
+
else blame(r.criteriaId, `criterion ${r.criteriaId}: missing from the adjudication (coverage gap)`);
|
|
532
589
|
}
|
|
533
590
|
}
|
|
534
591
|
|
|
@@ -539,18 +596,25 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
539
596
|
// citation, no normativeRef. Adjudication may only ever decide what the engine left open.
|
|
540
597
|
for (const it of adj.items) {
|
|
541
598
|
if (!open.has(it.criteriaId)) {
|
|
542
|
-
|
|
599
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: not open for adjudication — the engine already decided it, or it is not part of ${adj.standard}`);
|
|
543
600
|
}
|
|
544
601
|
}
|
|
545
602
|
|
|
546
|
-
// Per-item fail-closed validation.
|
|
547
|
-
|
|
603
|
+
// Per-item fail-closed validation. Grounding inputs are collected PER ITEM (not into one
|
|
604
|
+
// flat list) for the same reason as `blame`: a failed citation has to condemn its own
|
|
605
|
+
// criterion and no other.
|
|
606
|
+
const groundInputs = new Map<string, { file: string; line: number; selector?: string; snippet?: string }[]>();
|
|
607
|
+
const toGround = (criteriaId: string, g: { file: string; line: number; selector?: string; snippet?: string }) => {
|
|
608
|
+
const list = groundInputs.get(criteriaId);
|
|
609
|
+
if (list) list.push(g);
|
|
610
|
+
else groundInputs.set(criteriaId, [g]);
|
|
611
|
+
};
|
|
548
612
|
for (const it of adj.items) {
|
|
549
613
|
const v = it.verdict;
|
|
550
614
|
if (v === null) {
|
|
551
|
-
|
|
615
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: unadjudicated (verdict is null)`);
|
|
552
616
|
} else if (v === "C" || v === "NA") {
|
|
553
|
-
if (!it.justification || !it.justification.trim())
|
|
617
|
+
if (!it.justification || !it.justification.trim()) blame(it.criteriaId, `criterion ${it.criteriaId}: a ${v} verdict requires a justification`);
|
|
554
618
|
// A clearing verdict is gated exactly like an accusing one. Before this, the only
|
|
555
619
|
// check was "the justification is a non-empty string", so `"x"` cleared a criterion —
|
|
556
620
|
// and a model answering C to everything published a conformance nobody had assessed.
|
|
@@ -561,12 +625,14 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
561
625
|
// concerned"); `C` is not — it must stay `manual` and be ruled on against the
|
|
562
626
|
// criterion's own tests, from a rendered capture or by hand.
|
|
563
627
|
if (v === "C") {
|
|
564
|
-
|
|
628
|
+
blame(
|
|
629
|
+
it.criteriaId,
|
|
565
630
|
`criterion ${it.criteriaId}: a C verdict needs evidence to cite, and none was harvested for this criterion — record "manual" (reason "undecidable"), or "NA" if nothing in scope is concerned`,
|
|
566
631
|
);
|
|
567
632
|
}
|
|
568
633
|
} else if (cites.length === 0) {
|
|
569
|
-
|
|
634
|
+
blame(
|
|
635
|
+
it.criteriaId,
|
|
570
636
|
`criterion ${it.criteriaId}: a ${v} verdict must cite at least one of the ${it.evidence.length} evidence item(s) it was shown (citations: [{file, line, …}])`,
|
|
571
637
|
);
|
|
572
638
|
} else {
|
|
@@ -576,50 +642,75 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
576
642
|
const anchors = new Set(it.evidence.map((e) => `${e.file}:${e.line}`));
|
|
577
643
|
for (const c of cites) {
|
|
578
644
|
if (!anchors.has(`${c.file}:${c.line}`)) {
|
|
579
|
-
|
|
645
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: citation ${c.file}:${c.line} is not among this criterion's harvested evidence (fabricated?)`);
|
|
580
646
|
}
|
|
581
|
-
|
|
647
|
+
toGround(it.criteriaId, { file: c.file, line: c.line, selector: c.selector, snippet: c.snippet });
|
|
582
648
|
}
|
|
583
649
|
}
|
|
584
650
|
} else if (v === "NC") {
|
|
585
|
-
if (!it.findings || it.findings.length === 0)
|
|
651
|
+
if (!it.findings || it.findings.length === 0) blame(it.criteriaId, `criterion ${it.criteriaId}: an NC verdict requires at least one groundable finding`);
|
|
586
652
|
for (const f of it.findings ?? []) {
|
|
587
653
|
// FAIL-CLOSED: every NC finding must cite a precise, resolvable test of the active
|
|
588
654
|
// standard. A good practice with no normative test is a recommendation, not an NC.
|
|
589
655
|
if (!f.normativeRef || !f.normativeRef.trim()) {
|
|
590
|
-
|
|
656
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: an NC finding requires a normativeRef citing the failed test of the active standard`);
|
|
591
657
|
} else if (!normativeRefResolves(f.normativeRef, adj.standard, isCore(adj.standard) ? undefined : it.criteriaId)) {
|
|
592
|
-
|
|
658
|
+
blame(
|
|
659
|
+
it.criteriaId,
|
|
593
660
|
isCore(adj.standard)
|
|
594
661
|
? `criterion ${it.criteriaId}: normativeRef "${f.normativeRef}" does not resolve to a test of ${adj.standard} (fabricated?)`
|
|
595
662
|
: `criterion ${it.criteriaId}: normativeRef "${f.normativeRef}" is not a test of ${adj.standard} ${it.criteriaId} — cite one of its own tests (e.g. "${it.criteriaId}.1"); a WCAG id looks alike but denotes an unrelated test`,
|
|
596
663
|
);
|
|
597
664
|
}
|
|
598
|
-
|
|
665
|
+
toGround(it.criteriaId, { file: f.file, line: f.line, selector: f.selector, snippet: f.snippet });
|
|
599
666
|
}
|
|
600
667
|
} else if (v === "manual") {
|
|
601
668
|
if (!it.reason || !MANUAL_REASONS.has(it.reason))
|
|
602
|
-
|
|
669
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: a manual verdict requires reason ∈ {${MANUAL_REASON_VALUES.join(", ")}}`);
|
|
603
670
|
} else {
|
|
604
671
|
// Name the vocabulary in the rejection. The caller is usually a model that has just
|
|
605
672
|
// spent a whole worklist filling this file; "unknown verdict" alone told it nothing
|
|
606
673
|
// about how to be right on the retry.
|
|
607
|
-
|
|
674
|
+
blame(it.criteriaId, `criterion ${it.criteriaId}: unknown verdict "${String(v)}" — expected one of ${VERDICTS.join(" | ")}`);
|
|
608
675
|
}
|
|
609
676
|
// Recommendations are independent of the verdict (a C criterion may still carry a good
|
|
610
677
|
// practice) and are grounded exactly like an NC finding — no normativeRef required, as
|
|
611
678
|
// a recommendation has no normative test by definition.
|
|
612
|
-
for (const rec of it.recommendations ?? [])
|
|
679
|
+
for (const rec of it.recommendations ?? []) toGround(it.criteriaId, { file: rec.file, line: rec.line, selector: rec.selector, snippet: rec.snippet });
|
|
613
680
|
}
|
|
614
681
|
|
|
615
682
|
// Content-level grounding of every agent NC finding, every C/NA citation, and every
|
|
616
|
-
// recommendation — the same check, whichever direction the verdict points.
|
|
617
|
-
|
|
618
|
-
|
|
683
|
+
// recommendation — the same check, whichever direction the verdict points. Walked per
|
|
684
|
+
// criterion so a failure is attributable; the aggregate summary is identical to what the
|
|
685
|
+
// flat `groundItems` produced.
|
|
686
|
+
const grounding: GroundingSummary = { grounded: 0, moved: 0, failed: 0, issues: [] };
|
|
687
|
+
for (const [criteriaId, inputs] of groundInputs) {
|
|
688
|
+
for (const input of inputs) {
|
|
689
|
+
const r = groundFinding(input, { cwd: opts.cwd });
|
|
690
|
+
if (r.ok) {
|
|
691
|
+
grounding.grounded++;
|
|
692
|
+
if (r.moved) grounding.moved++;
|
|
693
|
+
} else {
|
|
694
|
+
grounding.failed++;
|
|
695
|
+
if (r.issue) {
|
|
696
|
+
grounding.issues.push(r.issue);
|
|
697
|
+
blame(criteriaId, r.issue);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
}
|
|
619
702
|
|
|
620
|
-
|
|
621
|
-
|
|
703
|
+
// STRICT: the historical file-level fold. One refusal, nothing lands.
|
|
704
|
+
if (opts.strict && issues.length) {
|
|
705
|
+
return { ok: false, audit, issues, applied: 0, stillManual: 0, rejected: 0, rejectedCriteria: [], grounding };
|
|
622
706
|
}
|
|
707
|
+
// PARTIAL: a refused criterion is dropped, the rest folds. `rejectedCriteria` keeps file
|
|
708
|
+
// order and de-duplicates, since one criterion can raise several issues. It counts only the
|
|
709
|
+
// criteria the gate REFUSED — an uncovered one the caller already declared is not a refusal.
|
|
710
|
+
const rejectedCriteria = adj.items.map((it) => it.criteriaId).filter((id) => itemIssues.has(id));
|
|
711
|
+
for (const id of itemIssues.keys()) if (!rejectedCriteria.includes(id)) rejectedCriteria.push(id);
|
|
712
|
+
const rejectedSet = notFolded;
|
|
713
|
+
const rejectedWhy = (id: string) => expected[id] ?? residualRejectedReason(itemIssues.get(id)?.[0] ?? "refused by the gate");
|
|
623
714
|
|
|
624
715
|
// Apply: clone the audit, update the decided criteria + append agent findings.
|
|
625
716
|
const next: AuditResult = structuredClone(audit);
|
|
@@ -636,6 +727,20 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
636
727
|
if (packMode) {
|
|
637
728
|
const decided: PackCriterionAdjudication[] = [];
|
|
638
729
|
for (const it of adj.items) {
|
|
730
|
+
// A refused verdict does not land. It is recorded as still-`manual` carrying the
|
|
731
|
+
// refusal, so the page grid and the report say WHY this criterion is still to assess
|
|
732
|
+
// instead of leaving a blank cell.
|
|
733
|
+
if (rejectedSet.has(it.criteriaId)) {
|
|
734
|
+
if (!open.has(it.criteriaId)) continue; // surplus item: not ours to record at all
|
|
735
|
+
decided.push({
|
|
736
|
+
id: it.criteriaId,
|
|
737
|
+
status: "manual",
|
|
738
|
+
reason: "undecidable",
|
|
739
|
+
justification: rejectedWhy(it.criteriaId),
|
|
740
|
+
findings: [],
|
|
741
|
+
});
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
639
744
|
if (it.verdict === "manual") {
|
|
640
745
|
stillManual++;
|
|
641
746
|
decided.push({
|
|
@@ -660,17 +765,25 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
660
765
|
decidedBy: "agent",
|
|
661
766
|
});
|
|
662
767
|
}
|
|
768
|
+
// An OPEN criterion the file never mentioned — a coverage gap, or one the caller declared
|
|
769
|
+
// uncovered — also has to say why it is still to assess; the loop above only walks the
|
|
770
|
+
// items that exist.
|
|
771
|
+
for (const id of notFolded) {
|
|
772
|
+
if (byId.has(id) || !open.has(id)) continue;
|
|
773
|
+
decided.push({ id, status: "manual", reason: "undecidable", justification: rejectedWhy(id), findings: [] });
|
|
774
|
+
}
|
|
663
775
|
next.packAdjudication = { standard: adj.standard, criteria: decided };
|
|
664
776
|
// The agent's findings still join the flat list so grounding, `check` and the reports can
|
|
665
777
|
// resolve them — but they never touch a WCAG criterion's status.
|
|
666
778
|
next.findings = [...next.findings, ...newFindings];
|
|
667
|
-
next.adjudicated = { date: adj.auditDate, applied, stillManual };
|
|
668
|
-
return { ok:
|
|
779
|
+
next.adjudicated = { date: adj.auditDate, applied, stillManual, ...(rejectedCriteria.length ? { rejected: rejectedCriteria.length } : {}) };
|
|
780
|
+
return { ok: issues.length === 0, audit: next, issues, applied, stillManual, rejected: rejectedCriteria.length, rejectedCriteria, grounding };
|
|
669
781
|
}
|
|
670
782
|
|
|
671
783
|
for (const it of adj.items) {
|
|
672
784
|
const c = critById.get(it.criteriaId);
|
|
673
785
|
if (!c) continue; // an item for a non-residual criterion — ignore (coverage already gated)
|
|
786
|
+
if (rejectedSet.has(it.criteriaId)) continue; // refused: the criterion keeps its `manual` status
|
|
674
787
|
if (it.verdict === "manual") {
|
|
675
788
|
c.status = "manual";
|
|
676
789
|
c.decidedBy = "agent";
|
|
@@ -696,7 +809,7 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
696
809
|
// gets its recommendations, and an NC item's reset `c.findings` keeps them appended last.
|
|
697
810
|
for (const it of adj.items) {
|
|
698
811
|
const c = critById.get(it.criteriaId);
|
|
699
|
-
if (!c) continue;
|
|
812
|
+
if (!c || rejectedSet.has(it.criteriaId)) continue;
|
|
700
813
|
for (const rec of it.recommendations ?? []) {
|
|
701
814
|
const f = agentFinding(it.criteriaId, rec, true);
|
|
702
815
|
c.findings.push(f);
|
|
@@ -705,11 +818,16 @@ export function applyAdjudication(audit: AuditResult, adj: AdjudicationFile, opt
|
|
|
705
818
|
}
|
|
706
819
|
|
|
707
820
|
next.findings = [...next.findings, ...newFindings];
|
|
708
|
-
// Residual set now holds
|
|
709
|
-
|
|
821
|
+
// Residual set now holds every criterion still to assess: the ones ruled `manual` by the
|
|
822
|
+
// adjudication, AND the ones whose verdict the gate refused. A refused criterion keeps its
|
|
823
|
+
// place in the residual set with the refusal as its reason — dropping it would report it as
|
|
824
|
+
// decided, and blanking the reason is exactly the empty cell this release is removing.
|
|
825
|
+
next.residualRisks = next.residualRisks
|
|
826
|
+
.filter((r) => byId.get(r.criteriaId)?.verdict === "manual" || rejectedSet.has(r.criteriaId))
|
|
827
|
+
.map((r) => (rejectedSet.has(r.criteriaId) ? { ...r, reason: rejectedWhy(r.criteriaId) } : r));
|
|
710
828
|
recomputeTallies(next);
|
|
711
|
-
next.adjudicated = { date: adj.auditDate, applied, stillManual };
|
|
712
|
-
return { ok:
|
|
829
|
+
next.adjudicated = { date: adj.auditDate, applied, stillManual, ...(rejectedCriteria.length ? { rejected: rejectedCriteria.length } : {}) };
|
|
830
|
+
return { ok: issues.length === 0, audit: next, issues, applied, stillManual, rejected: rejectedCriteria.length, rejectedCriteria, grounding };
|
|
713
831
|
}
|
|
714
832
|
|
|
715
833
|
function agentFinding(criteriaId: string, f: AgentFinding, advisory = false): Finding {
|
|
@@ -750,6 +868,10 @@ function recomputeTallies(a: AuditResult): void {
|
|
|
750
868
|
|
|
751
869
|
const residualScanReason = () => "Rendering criterion — decide on the rendered DOM (`scan`).";
|
|
752
870
|
const residualUndecidableReason = () => "Left as an explicit residual risk (not decidable from the available evidence).";
|
|
871
|
+
/** Why a criterion is still to assess after a PARTIAL fold: its verdict was refused. Carries
|
|
872
|
+
* the first refusal verbatim, so the reader (and the next adjudication pass) sees the actual
|
|
873
|
+
* contract violation rather than a bare « to assess ». */
|
|
874
|
+
const residualRejectedReason = (why: string) => `Adjudication refused by the gate — ${why.replace(/^criterion \S+: /, "")}. Re-adjudicate this criterion.`;
|
|
753
875
|
|
|
754
876
|
// ---- worklist file rendering ----
|
|
755
877
|
const T = {
|
|
@@ -909,11 +1031,18 @@ function plainTest(s: string): string {
|
|
|
909
1031
|
return s.replace(/\[([^\]]+)\]\(#[^)]*\)/g, "$1");
|
|
910
1032
|
}
|
|
911
1033
|
|
|
912
|
-
|
|
1034
|
+
/** `preamble: false` renders the criterion sections ALONE.
|
|
1035
|
+
*
|
|
1036
|
+
* The preamble names the file to write into and the command to fold with, which is right for
|
|
1037
|
+
* the one combined document and wrong for a per-criterion brief: repeated 96 times it is 96
|
|
1038
|
+
* copies of instructions that contradict a CI adjudicator's toolset (it has no shell) and
|
|
1039
|
+
* point at the wrong file. The briefs carry evidence and protocol; the instructions live once,
|
|
1040
|
+
* in the worklist's `contract` field and the dispatch contract. */
|
|
1041
|
+
export function formatAdjudication(items: AdjudicationItem[], lang: Lang = "en", standard: StandardId = CORE, opts: { preamble?: boolean } = {}): string {
|
|
913
1042
|
const s = T[lang];
|
|
914
1043
|
const pack = isCore(standard) ? undefined : loadPack(standard);
|
|
915
|
-
const out: string[] = [s.title, "", s.intro, "", ...s.verdicts, "", s.rule, "", s.then, ""];
|
|
916
|
-
if (pack) out.push(`> ${s.packIntro(pack.name)}`, "");
|
|
1044
|
+
const out: string[] = opts.preamble === false ? [] : [s.title, "", s.intro, "", ...s.verdicts, "", s.rule, "", s.then, ""];
|
|
1045
|
+
if (pack && opts.preamble !== false) out.push(`> ${s.packIntro(pack.name)}`, "");
|
|
917
1046
|
for (const it of items) {
|
|
918
1047
|
out.push(`## ${pack ? `${pack.name} ` : ""}${it.criteriaId}${it.title ? ` — ${it.title}` : ""} _(${it.automatability})_`);
|
|
919
1048
|
out.push("", `> ${s.evidence} (${it.evidence.length}${it.evidenceTruncated ? ` / ${it.evidenceTruncated.total}` : ""}):`, "");
|
|
@@ -970,6 +1099,10 @@ export function formatAdjudication(items: AdjudicationItem[], lang: Lang = "en",
|
|
|
970
1099
|
export interface WriteAdjudicationResult {
|
|
971
1100
|
todoPath: string;
|
|
972
1101
|
mdPath: string;
|
|
1102
|
+
/** The verdicts-only file, for an adjudicator that cannot edit half a megabyte of JSON. */
|
|
1103
|
+
verdictsPath: string;
|
|
1104
|
+
/** One small Markdown brief per criterion, for one that cannot read it either. */
|
|
1105
|
+
itemsDir: string;
|
|
973
1106
|
count: number;
|
|
974
1107
|
}
|
|
975
1108
|
|
|
@@ -992,5 +1125,60 @@ export function writeAdjudication(
|
|
|
992
1125
|
};
|
|
993
1126
|
writeFileSync(todoPath, JSON.stringify(file, null, 2) + "\n");
|
|
994
1127
|
writeFileSync(mdPath, formatAdjudication(items, opts.lang ?? "en", opts.standard));
|
|
995
|
-
|
|
1128
|
+
|
|
1129
|
+
// THE SPLIT SURFACE, for an adjudicator that cannot shell out.
|
|
1130
|
+
//
|
|
1131
|
+
// The two documents above are sized for a session with a shell: 96 RGAA criteria carrying
|
|
1132
|
+
// 30 evidence anchors each come to ~540 KB of JSON and ~470 KB of Markdown, and the runbook
|
|
1133
|
+
// tells the reader to edit the JSON in place and then run the engine to fold it. That is
|
|
1134
|
+
// fine locally. In CI the adjudicator is handed Read/Grep/Glob/Edit/Write and nothing else,
|
|
1135
|
+
// and against those tools the same instructions are impossible: reading either document
|
|
1136
|
+
// swamps the context, editing 96 verdicts inside half a megabyte of JSON is 96 exact-match
|
|
1137
|
+
// edits, and every prescribed `node …` command is denied. Measured on a real run: 17
|
|
1138
|
+
// permission denials, 75 of 424 turns used, and a todo file that came back untouched — so
|
|
1139
|
+
// the fail-closed fold correctly discarded all 96 and every criterion stayed "to assess".
|
|
1140
|
+
//
|
|
1141
|
+
// So the same worklist is also written as one small file to WRITE and one small file per
|
|
1142
|
+
// criterion to READ. Additive on purpose: the two documents above keep their exact shape and
|
|
1143
|
+
// every existing caller, gate and test is untouched.
|
|
1144
|
+
const verdictsPath = join(outDir, "ADJUDICATE.verdicts.json");
|
|
1145
|
+
writeFileSync(verdictsPath, JSON.stringify({ ...file, items: slimAdjudicationItems(items) }, null, 2) + "\n");
|
|
1146
|
+
const itemsDir = join(outDir, "adjudicate");
|
|
1147
|
+
mkdirSync(itemsDir, { recursive: true });
|
|
1148
|
+
for (const it of items) {
|
|
1149
|
+
writeFileSync(join(itemsDir, `${it.criteriaId}.md`), formatAdjudication([it], opts.lang ?? "en", opts.standard, { preamble: false }));
|
|
1150
|
+
}
|
|
1151
|
+
return { todoPath, mdPath, verdictsPath, itemsDir, count: items.length };
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/** The worklist with the evidence stripped: what an adjudicator has to WRITE, without the
|
|
1155
|
+
* bulk it only has to READ. Every field the fold requires — verdict, justification, reason,
|
|
1156
|
+
* findings, citations, recommendations — is kept, so a filled slim file carries a complete
|
|
1157
|
+
* decision. The evidence comes back at fold time via `hydrateAdjudication`. */
|
|
1158
|
+
export function slimAdjudicationItems(items: AdjudicationItem[]): AdjudicationItem[] {
|
|
1159
|
+
return items.map((it) => {
|
|
1160
|
+
const { evidence: _evidence, evidenceTruncated: _truncated, ...rest } = it;
|
|
1161
|
+
return { ...rest, evidence: [] } as AdjudicationItem;
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
/** Put the evidence back on a slim adjudication, re-derived from the audit it was built from.
|
|
1166
|
+
*
|
|
1167
|
+
* Safe because the worklist is a pure function of the audit: same audit, same harvesters, same
|
|
1168
|
+
* anchors. This is what lets `verify --apply` accept a verdicts-only file WITHOUT relaxing a
|
|
1169
|
+
* single check — the citation gate still matches each cited anchor against this criterion's
|
|
1170
|
+
* own harvested evidence, exactly as it does for an inline worklist.
|
|
1171
|
+
*
|
|
1172
|
+
* An item the re-derivation does not know is left alone: it is a surplus verdict, and the
|
|
1173
|
+
* coverage check in `applyAdjudication` is what must reject it. */
|
|
1174
|
+
export function hydrateAdjudication(adj: AdjudicationFile, audit: AuditResult, opts: { cwd?: string } = {}): void {
|
|
1175
|
+
const needs = adj.items.filter((it) => !it.evidence || it.evidence.length === 0);
|
|
1176
|
+
if (!needs.length) return;
|
|
1177
|
+
const derived = new Map(buildAdjudicationWorklist(audit, { ...opts, standard: adj.standard }).map((it) => [it.criteriaId, it]));
|
|
1178
|
+
for (const it of needs) {
|
|
1179
|
+
const full = derived.get(it.criteriaId);
|
|
1180
|
+
if (!full) continue;
|
|
1181
|
+
it.evidence = full.evidence;
|
|
1182
|
+
if (full.evidenceTruncated) it.evidenceTruncated = full.evidenceTruncated;
|
|
1183
|
+
}
|
|
996
1184
|
}
|