ultra11y 5.3.3 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.opencode/plugins/ultra11y.js +1 -1
- package/package.json +1 -1
- package/scripts/ultra11y.mjs +1721 -1272
- package/skills/review-a11y/SKILL.md +1 -1
- package/skills/review-a11y/scripts/ultra11y.mjs +1721 -1272
- package/skills/ultra11y/SKILL.md +2 -2
- package/skills/ultra11y/references/ci.md +46 -4
- package/skills/ultra11y/references/judgment.md +50 -12
- package/skills/ultra11y/references/pages.md +13 -4
- package/skills/ultra11y/scripts/ultra11y.mjs +1721 -1272
- package/src/adjudicate.ts +229 -41
- package/src/audit.ts +163 -2
- package/src/cli.ts +216 -18
- package/src/ledger.ts +250 -0
- package/src/orchestrate-templates.ts +2 -2
- package/src/report.ts +24 -6
- package/src/rules/rendered.ts +39 -1
- package/src/types.ts +10 -3
package/src/audit.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { attachSignals, isSnapshotDom, snapshotPageId } from "./snapshot.js";
|
|
|
12
12
|
import { attr, elementsByTag, type Doc, type CaptureProvenance } from "./parse/html.js";
|
|
13
13
|
import { CAPTURES_DIR, computeCaptureCoverage, enrichCaptureOrigins, isUnderDir, readCaptureDir, capturesForSources } from "./capture.js";
|
|
14
14
|
import { isFullDocument } from "./rules/rule.js";
|
|
15
|
+
import { renderedTestedScs } from "./rules/rendered.js";
|
|
15
16
|
import { runRules } from "./rules/registry.js";
|
|
16
17
|
import { runCrossRules } from "./rules/cross-registry.js";
|
|
17
18
|
import { listPacks } from "./standards/registry.js";
|
|
@@ -64,12 +65,141 @@ const APPLICABLE: Record<string, (d: Doc) => boolean> = {
|
|
|
64
65
|
"3.1.1": (d) => isFullDocument(d), // Language of Page — html-lang-missing / lang-invalid
|
|
65
66
|
};
|
|
66
67
|
|
|
67
|
-
|
|
68
|
+
// ---- SUBJECT MATTER: what makes a NON-static criterion applicable at all -----------------
|
|
69
|
+
//
|
|
70
|
+
// A judgment or rendering criterion used to have exactly two outcomes: `NC` when a rule fired,
|
|
71
|
+
// `manual` otherwise. So a repository containing no audio and no video still reported all five
|
|
72
|
+
// time-based-media criteria as « to assess » — and under RGAA, whose theme 4 projects from
|
|
73
|
+
// them, that is 13 criteria a reader has to work through to discover there was never anything
|
|
74
|
+
// to look at. Measured on a real 300-file audit: 96 of 106 criteria « to assess », whole themes
|
|
75
|
+
// among them applicable to nothing in scope.
|
|
76
|
+
//
|
|
77
|
+
// « Not applicable » is a real, normative verdict, and the engine can prove it for a criterion
|
|
78
|
+
// whose SUBJECT MATTER is a thing you can look for in the source. So: no media element in the
|
|
79
|
+
// whole scope ⇒ the media criteria are NA, with a justification naming the observation.
|
|
80
|
+
//
|
|
81
|
+
// Three rules keep this honest, because a wrong NA is far worse than an honest « to assess » —
|
|
82
|
+
// it is a non-conformity hidden inside a report someone signs:
|
|
83
|
+
//
|
|
84
|
+
// 1. OVER-INCLUSION IS THE SAFE DIRECTION. A predicate answers "could this criterion
|
|
85
|
+
// possibly apply here?", so it must say YES on anything ambiguous. `<object>`/`<embed>`/
|
|
86
|
+
// `<iframe>` are usually not media, and they are all listed anyway: their presence merely
|
|
87
|
+
// keeps a criterion open, which costs nothing but a line in the worklist.
|
|
88
|
+
// 2. APPLICABILITY IS OR-FOLDED ACROSS THE WHOLE SCOPE (see `foldDoc`). One document with a
|
|
89
|
+
// `<video>` keeps the media criteria open for the entire audit. NA means "nothing,
|
|
90
|
+
// anywhere in what was audited" — never "nothing in this file".
|
|
91
|
+
// 3. THE VERDICT IS SCOPE-BOUND AND SAYS SO. The justification states what was searched and
|
|
92
|
+
// not found, so it is falsifiable by a reader. A scope that cannot see rendered output
|
|
93
|
+
// (an opaque component library) is already flagged on the audit itself, and `check`'s
|
|
94
|
+
// unjustified-NA gate still applies.
|
|
95
|
+
const SUBJECT_MATTER: Record<string, (d: Doc) => boolean> = {
|
|
96
|
+
// ---- Time-based media (WCAG 1.2.x → RGAA theme 4) ----
|
|
97
|
+
// Captions, transcripts, audio description and sign language all presuppose a media element.
|
|
98
|
+
// `track` counts on its own: a stray <track> means media is being assembled somewhere.
|
|
99
|
+
"1.2.1": hasMedia,
|
|
100
|
+
"1.2.2": hasMedia,
|
|
101
|
+
"1.2.3": hasMedia,
|
|
102
|
+
"1.2.4": hasMedia,
|
|
103
|
+
"1.2.5": hasMedia,
|
|
104
|
+
// ---- Motion actuation (WCAG 2.5.4) ----
|
|
105
|
+
// Applicable only where device motion drives the interface, which is a source-visible API.
|
|
106
|
+
"2.5.4": hasMotionApi,
|
|
107
|
+
// ---- Forms (WCAG 3.3.x) ----
|
|
108
|
+
// Error identification, labels and instructions, error suggestion, error prevention,
|
|
109
|
+
// redundant entry and accessible authentication all presuppose user input. 3.3.4 and 3.3.8
|
|
110
|
+
// are narrower still (legal/financial transactions, authentication), so "any form control"
|
|
111
|
+
// is the deliberately over-inclusive predicate for them.
|
|
112
|
+
"3.3.1": hasFormControl,
|
|
113
|
+
"3.3.2": hasFormControl,
|
|
114
|
+
"3.3.3": hasFormControl,
|
|
115
|
+
"3.3.4": hasFormControl,
|
|
116
|
+
"3.3.7": hasFormControl,
|
|
117
|
+
"3.3.8": hasFormControl,
|
|
118
|
+
// Identify Input Purpose — about autocomplete on fields collecting user data.
|
|
119
|
+
"1.3.5": hasFormControl,
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/** Media, and everything that could be carrying media. Over-inclusive on purpose (rule 1). */
|
|
123
|
+
function hasMedia(d: Doc): boolean {
|
|
124
|
+
return has(d, "audio", "video", "track", "source", "object", "embed", "iframe", "marquee", "canvas");
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Device-motion input: the only thing WCAG 2.5.4 is about. */
|
|
128
|
+
function hasMotionApi(d: Doc): boolean {
|
|
129
|
+
return /\b(?:devicemotion|deviceorientation|DeviceMotionEvent|DeviceOrientationEvent|Accelerometer|Gyroscope|requestPermission)\b/i.test(d.source);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Any control that takes user input, native or delegated. Over-inclusive on purpose. */
|
|
133
|
+
function hasFormControl(d: Doc): boolean {
|
|
134
|
+
if (has(d, "input", "select", "textarea", "form", "fieldset", "output", "datalist")) return true;
|
|
135
|
+
// A custom widget standing in for a field, and a submit control that implies one.
|
|
136
|
+
return d.elements.some((e) => {
|
|
137
|
+
const role = (e.attribs.role ?? "").toLowerCase();
|
|
138
|
+
if (role && /^(textbox|combobox|listbox|checkbox|radio|radiogroup|slider|spinbutton|searchbox|switch|form)$/.test(role)) return true;
|
|
139
|
+
return e.attribs.contenteditable !== undefined;
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ---- WHY a criterion is still to assess, and WHERE its evidence would come from -----------
|
|
144
|
+
//
|
|
145
|
+
// One generic sentence per tier used to answer for all 52 undecided criteria: "needs a rendered
|
|
146
|
+
// DOM (contrast, focus visibility, zoom/reflow, target size)" was printed against 1.4.5 (images
|
|
147
|
+
// of text) as readily as against 1.4.3 (contrast), naming four measurements of which at most
|
|
148
|
+
// one was the criterion's. A reader could not tell what was missing, and neither could the next
|
|
149
|
+
// run — so « to assess » read as a shrug rather than as an instruction.
|
|
150
|
+
//
|
|
151
|
+
// Each entry names the measurement that decides the criterion and the command that produces it.
|
|
152
|
+
// Absent ⇒ the tier default below, which is still true, just unspecific.
|
|
153
|
+
const RESIDUAL_TRAIL: Record<string, string> = {
|
|
154
|
+
// Rendering criteria the SNAPSHOT tier decides offline, from computed styles + stylesheets.
|
|
155
|
+
"1.3.4":
|
|
156
|
+
"Decided on a page's own stylesheets (an orientation media query that rotates the document) — record a snapshot with `scan --sample` (or an E2E capture), then re-audit.",
|
|
157
|
+
"1.4.1": "Decided on computed colours (a link distinguished by colour alone) — record a snapshot with `scan --sample`, then re-audit.",
|
|
158
|
+
"1.4.3":
|
|
159
|
+
"Decided on computed text/background colours, with a screenshot fallback where the backdrop is an image — record a snapshot with `scan --sample`, then re-audit.",
|
|
160
|
+
"1.4.11": "Decided on computed colours of UI components and graphics — record a snapshot with `scan --sample`, then re-audit.",
|
|
161
|
+
"2.4.7":
|
|
162
|
+
"Decided on the page's own stylesheets (a `:focus` rule that removes the indicator without restoring one), or by the live focus probe — record a snapshot with `scan --sample`, then re-audit.",
|
|
163
|
+
// Rendering criteria that need a LIVE browser: they are measured by acting on the page.
|
|
164
|
+
"1.4.4": "Needs a live browser: the page is re-measured at 200% zoom — `scan <url> --runtime local --merge <audit.json>`.",
|
|
165
|
+
"1.4.10": "Needs a live browser: the page is re-measured in a 320px viewport (reflow) — `scan <url> --merge <audit.json>`.",
|
|
166
|
+
"1.4.12": "Needs a live browser: text spacing overrides are injected and the layout re-measured — `scan <url> --runtime local --merge <audit.json>`.",
|
|
167
|
+
"1.4.13":
|
|
168
|
+
"Needs a live browser: hover/focus-triggered content is opened and probed for dismiss/hover/persist — `scan <url> --runtime local --merge <audit.json>`.",
|
|
169
|
+
// axe's own `target-size` rule can FAIL this on a live page, but no tier certifies it, so a
|
|
170
|
+
// clean scan leaves it open rather than conforming. Say both halves.
|
|
171
|
+
"2.5.8":
|
|
172
|
+
"A live scan can fail this (axe `target-size`) but never certify it — `scan <url> --runtime local --merge <audit.json>`, then adjudicate what is left.",
|
|
173
|
+
"4.1.3":
|
|
174
|
+
"Needs a live browser WITH interaction: a status message is triggered and the live region observed — `scan <url> --runtime local --merge <audit.json>` (interactions are on by default).",
|
|
175
|
+
// Rendering criteria no automated tier measures — say so, rather than pointing at `scan` and
|
|
176
|
+
// letting the reader discover it changes nothing.
|
|
177
|
+
"1.4.5":
|
|
178
|
+
"No automated tier decides this: whether text is presented as an image is a reading of each image's content. Adjudicate it (`verify --manual`) against the images the audit lists.",
|
|
179
|
+
"2.1.2": "No automated tier decides this: escaping a keyboard trap has to be attempted by hand, on each focusable region.",
|
|
180
|
+
"2.3.1": "No automated tier decides this: flashing has to be observed over time on the rendered page.",
|
|
181
|
+
"2.4.11": "No automated tier decides this: whether a focused element stays unobscured depends on the sticky headers and overlays in play on each screen.",
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
function residualReason(automatability: string, sc?: string): string {
|
|
185
|
+
const trail = sc ? RESIDUAL_TRAIL[sc] : undefined;
|
|
186
|
+
if (trail) return trail;
|
|
68
187
|
return automatability === "needs-rendering"
|
|
69
188
|
? "Needs a rendered DOM (contrast, focus visibility, zoom/reflow, target size) — decide via `scan`."
|
|
70
189
|
: "Judgement criterion — adjudicated by the agent from source/context (`verify --manual`, gated).";
|
|
71
190
|
}
|
|
72
191
|
|
|
192
|
+
/** Why a non-static criterion is NA — the observation, so a reader can falsify it. Scope-bound
|
|
193
|
+
* by construction (`n` files audited), which is exactly the claim being made. */
|
|
194
|
+
function subjectMatterReason(sc: string, files: number): string {
|
|
195
|
+
const scope = `nothing in the ${files} file(s) audited is concerned`;
|
|
196
|
+
if (sc.startsWith("1.2."))
|
|
197
|
+
return `No time-based media in scope: ${scope} — no <audio>, <video>, <track>, <object>, <embed>, <iframe> or <canvas> element was found.`;
|
|
198
|
+
if (sc === "2.5.4") return `No motion actuation in scope: ${scope} — no device-motion or device-orientation API is used.`;
|
|
199
|
+
if (sc === "1.3.5" || sc.startsWith("3.3.")) return `No user input in scope: ${scope} — no form control (native, ARIA or contenteditable) was found.`;
|
|
200
|
+
return `No element in scope is concerned by this success criterion (${scope}).`;
|
|
201
|
+
}
|
|
202
|
+
|
|
73
203
|
// Streaming accumulator: parse → run rules → fold → DISCARD each Doc, so the
|
|
74
204
|
// engine never holds a whole repo in memory. The only cross-document state is
|
|
75
205
|
// (a) the flat findings list (bounded by finding count, not source size) and
|
|
@@ -87,6 +217,10 @@ interface Accum {
|
|
|
87
217
|
sfcExts: Set<string>; // which SFC extensions were seen
|
|
88
218
|
captures: { file: string; provenance: CaptureProvenance }[]; // rendered capture files audited
|
|
89
219
|
langCounts: Map<string, number>; // <html lang> primary subtags seen, for repo-language detection
|
|
220
|
+
// Success criteria the RENDERED tier measured, from the snapshots ingested in this run
|
|
221
|
+
// (src/rules/rendered.ts renderedTestedScs). Stamped onto scope.scan.testedScs so the
|
|
222
|
+
// partial-audit banner names what is genuinely missing instead of every rendering criterion.
|
|
223
|
+
renderedScs: Set<string>;
|
|
90
224
|
}
|
|
91
225
|
|
|
92
226
|
// Precompute the static success criteria + their applicability predicates once.
|
|
@@ -94,6 +228,13 @@ const STATIC_PREDS: ReadonlyArray<readonly [string, (d: Doc) => boolean]> = allS
|
|
|
94
228
|
.filter((c) => c.automatability === "static")
|
|
95
229
|
.map((c) => [c.sc, APPLICABLE[c.sc] ?? isFullDocument] as const);
|
|
96
230
|
|
|
231
|
+
// The same fold, for the non-static criteria whose subject matter is source-visible. Kept as a
|
|
232
|
+
// separate list so the `static` contract above ("clean ⇒ C") is untouched: a criterion here can
|
|
233
|
+
// only ever gain `NA`, never `C`.
|
|
234
|
+
const SUBJECT_PREDS: ReadonlyArray<readonly [string, (d: Doc) => boolean]> = allSC()
|
|
235
|
+
.filter((c) => c.automatability !== "static" && SUBJECT_MATTER[c.sc] !== undefined)
|
|
236
|
+
.map((c) => [c.sc, SUBJECT_MATTER[c.sc]!] as const);
|
|
237
|
+
|
|
97
238
|
function newAccum(): Accum {
|
|
98
239
|
return {
|
|
99
240
|
byCriterion: new Map(),
|
|
@@ -107,6 +248,7 @@ function newAccum(): Accum {
|
|
|
107
248
|
sfcExts: new Set(),
|
|
108
249
|
captures: [],
|
|
109
250
|
langCounts: new Map(),
|
|
251
|
+
renderedScs: new Set(),
|
|
110
252
|
};
|
|
111
253
|
}
|
|
112
254
|
|
|
@@ -166,6 +308,14 @@ export function foldDoc(acc: Accum, doc: Doc, graph?: DepGraph): void {
|
|
|
166
308
|
for (const [id, pred] of STATIC_PREDS) {
|
|
167
309
|
if (!acc.applicable.get(id) && pred(doc)) acc.applicable.set(id, true);
|
|
168
310
|
}
|
|
311
|
+
// OR-folded across the whole scope: one document carrying the subject matter keeps the
|
|
312
|
+
// criterion open for the entire audit (see SUBJECT_MATTER, rule 2).
|
|
313
|
+
for (const [id, pred] of SUBJECT_PREDS) {
|
|
314
|
+
if (!acc.applicable.get(id) && pred(doc)) acc.applicable.set(id, true);
|
|
315
|
+
}
|
|
316
|
+
// A page snapshot just let the rendered tier MEASURE some criteria, offline and with no
|
|
317
|
+
// browser. Record which, so the report stops claiming they were never tested.
|
|
318
|
+
if (doc.signals) for (const sc of renderedTestedScs(doc.signals)) acc.renderedScs.add(sc);
|
|
169
319
|
if (doc.opaqueComponents?.length) {
|
|
170
320
|
for (const lib of doc.opaqueComponents) acc.opaqueLibs.add(lib);
|
|
171
321
|
acc.opaqueFiles++;
|
|
@@ -215,11 +365,17 @@ function finalize(acc: Accum, inputs: string[], extra: FinalizeExtra = {}): Audi
|
|
|
215
365
|
} else if (normativeFs.length > 0) {
|
|
216
366
|
// a rule on a needs-rendering / judgment SC raised a DEFINITE failure
|
|
217
367
|
status = "NC";
|
|
368
|
+
} else if (SUBJECT_MATTER[c.sc] !== undefined && !(acc.applicable.get(c.sc) ?? false)) {
|
|
369
|
+
// The criterion's SUBJECT MATTER is absent from the entire scope, so there is nothing to
|
|
370
|
+
// judge and nothing to render — « not applicable » is the accurate verdict, not « to
|
|
371
|
+
// assess ». Never a `C`: this says the criterion does not apply, never that it is met.
|
|
372
|
+
status = "NA";
|
|
373
|
+
justification = subjectMatterReason(c.sc, acc.fileCount);
|
|
218
374
|
} else {
|
|
219
375
|
// engine can't decide — leave it for the agent to adjudicate (`verify --manual`,
|
|
220
376
|
// gated) or the `scan` tier (rendering criteria); never a silent conforming.
|
|
221
377
|
status = "manual";
|
|
222
|
-
residualRisks.push({ criteriaId: c.sc, reason: residualReason(c.automatability), automatability: c.automatability });
|
|
378
|
+
residualRisks.push({ criteriaId: c.sc, reason: residualReason(c.automatability, c.sc), automatability: c.automatability });
|
|
223
379
|
}
|
|
224
380
|
criteria.push({ id: c.sc, guideline: c.guideline, status, findings: fs, ...(justification ? { justification } : {}) });
|
|
225
381
|
}
|
|
@@ -262,6 +418,11 @@ function finalize(acc: Accum, inputs: string[], extra: FinalizeExtra = {}): Audi
|
|
|
262
418
|
}
|
|
263
419
|
: {}),
|
|
264
420
|
...(acc.langCounts.size ? { langs: [...acc.langCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([lang]) => lang) } : {}),
|
|
421
|
+
// Coverage the RENDERED tier earned from the snapshots this run ingested. Same field the
|
|
422
|
+
// live-browser merge stamps (`mergeDynamic`), because it is the same claim: these criteria
|
|
423
|
+
// were measured. Omitted when no snapshot carried usable signals, so a source-only audit
|
|
424
|
+
// is byte-for-byte unchanged and still says the rendering criteria are untested.
|
|
425
|
+
...(acc.renderedScs.size ? { scan: { testedScs: [...acc.renderedScs].sort() } } : {}),
|
|
265
426
|
// The pages this run genuinely read. Written UNCONDITIONALLY — `[]` is the whole point,
|
|
266
427
|
// because "this audit read no page" is exactly the claim a source-only run needs to make,
|
|
267
428
|
// and an omit-when-empty field would say nothing precisely then. See scope.pagesAudited.
|
package/src/cli.ts
CHANGED
|
@@ -42,7 +42,15 @@ import { runCriteria, renderCriteriaReference } from "./criteria.js";
|
|
|
42
42
|
import { checkReport, checkSemantic } from "./check.js";
|
|
43
43
|
import { buildWorklist, writeWorklist, applyVerdicts, VERIFY_MAX, type VerifyItem } from "./verify.js";
|
|
44
44
|
import { groundItems } from "./grounding.js";
|
|
45
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
buildAdjudicationWorklist,
|
|
47
|
+
formatAdjudication,
|
|
48
|
+
writeAdjudication,
|
|
49
|
+
applyAdjudication,
|
|
50
|
+
hydrateAdjudication,
|
|
51
|
+
type AdjudicationFile,
|
|
52
|
+
} from "./adjudicate.js";
|
|
53
|
+
import { entriesFrom, isLedger, ledgerPath, mergeLedger, readLedger, replayLedger, type VerdictLedger, writeLedger } from "./ledger.js";
|
|
46
54
|
import { BATCH_SIZE, apiKeyFromEnv, applyRawVerdicts, judgeAll, modelFromEnv } from "./llm.js";
|
|
47
55
|
import { runScan, runScanMany, runCrawlScan, runSampleScan, mergeDynamic, mergeSnapshotAudit, cleanDynamic, dockerAvailable } from "./scan.js";
|
|
48
56
|
import { runScanLocal, runScanManyLocal, runCrawlScanLocal, runSampleScanLocal, localAvailable } from "./scan-local.js";
|
|
@@ -368,6 +376,13 @@ Options:
|
|
|
368
376
|
--generate criteria: emit the bundled references/criteria.md (WCAG 2.2 AA)
|
|
369
377
|
--apply <file> verify: reduce a filled verdicts file to a pass/fail gate
|
|
370
378
|
(requires --report — coverage is re-derived from the report, uncapped)
|
|
379
|
+
Also accepts an ADJUDICATION (--in <audit.json>), or a VERDICT LEDGER,
|
|
380
|
+
which it REPLAYS onto the audit with no model in the loop
|
|
381
|
+
--ledger <file> verify --apply / judge --apply: record the verdicts the fold ACCEPTED,
|
|
382
|
+
so a later run replays them for free (default:
|
|
383
|
+
.ultra11y/verdicts/<standard>.json). Replay re-derives the evidence
|
|
384
|
+
and re-runs the same gate; a verdict whose evidence changed is
|
|
385
|
+
dropped as stale and its criterion says so
|
|
371
386
|
--max-verify <n> verify: cap the worklist size; 0 = no cap (default: 40)
|
|
372
387
|
--verdicts <file> check --semantic: the adjudicated verdicts artifact
|
|
373
388
|
(default: VERIFY.todo.json next to the report)
|
|
@@ -430,6 +445,10 @@ Options:
|
|
|
430
445
|
--apply judge: fold the verdicts straight into the audit, through the SAME
|
|
431
446
|
fail-closed gate an agent's verdicts pass (no unjustified C/NA, no
|
|
432
447
|
ungroundable NC, no unadjudicated criterion)
|
|
448
|
+
--strict verify --apply / judge --apply: all-or-nothing fold — one refused
|
|
449
|
+
verdict discards the whole adjudication. The default folds PER
|
|
450
|
+
VERDICT: a refusal costs its own criterion, which stays to assess
|
|
451
|
+
carrying the refusal, and every verdict that proved itself lands
|
|
433
452
|
--semantic verify: fold the support-check into one pass
|
|
434
453
|
check: engage the semantic gate — requires an adjudicated verdicts
|
|
435
454
|
artifact (fails closed when absent) and re-grounds every passing
|
|
@@ -508,6 +527,10 @@ const VALUE_FLAGS = new Set([
|
|
|
508
527
|
"dedup",
|
|
509
528
|
"only",
|
|
510
529
|
"standard",
|
|
530
|
+
// `verify --apply` / `judge --apply`: where to RECORD the verdicts the fold accepted, so a
|
|
531
|
+
// later run can replay them without a model (src/ledger.ts). A path; empty falls back to the
|
|
532
|
+
// standard's default location under .ultra11y/verdicts/.
|
|
533
|
+
"ledger",
|
|
511
534
|
"baseline",
|
|
512
535
|
"fail-on",
|
|
513
536
|
"split",
|
|
@@ -592,6 +615,9 @@ const BOOLEAN_FLAGS = new Set([
|
|
|
592
615
|
"generate",
|
|
593
616
|
"semantic",
|
|
594
617
|
"manual",
|
|
618
|
+
// `verify --apply` / `judge --apply`: restore the all-or-nothing fold, where one refused
|
|
619
|
+
// verdict discards the whole adjudication. The default is per-verdict.
|
|
620
|
+
"strict",
|
|
595
621
|
"no-technical",
|
|
596
622
|
"override",
|
|
597
623
|
"local",
|
|
@@ -2486,10 +2512,14 @@ function cmdVerify(p: ParsedArgs): number {
|
|
|
2486
2512
|
return 2;
|
|
2487
2513
|
}
|
|
2488
2514
|
// Dispatch on shape: an OBJECT with kind:"adjudication" is a manual-criteria adjudication
|
|
2489
|
-
// (src/adjudicate.ts);
|
|
2515
|
+
// (src/adjudicate.ts); kind:"verdict-ledger" is a stored one being REPLAYED (src/ledger.ts,
|
|
2516
|
+
// no model in the loop); a plain ARRAY is the classic NC-verdicts worklist.
|
|
2490
2517
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && (parsed as { kind?: string }).kind === "adjudication") {
|
|
2491
2518
|
return applyAdjudicationFile(p, parsed as AdjudicationFile, lang);
|
|
2492
2519
|
}
|
|
2520
|
+
if (isLedger(parsed)) {
|
|
2521
|
+
return replayLedgerFile(p, parsed, lang);
|
|
2522
|
+
}
|
|
2493
2523
|
if (!Array.isArray(parsed)) {
|
|
2494
2524
|
console.error("ultra11y verify: --apply must be a JSON array of verdicts, or an adjudication object.");
|
|
2495
2525
|
return 2;
|
|
@@ -2617,6 +2647,109 @@ function cmdVerify(p: ParsedArgs): number {
|
|
|
2617
2647
|
return 0;
|
|
2618
2648
|
}
|
|
2619
2649
|
|
|
2650
|
+
/** Where to record the verdicts a fold accepted. `--ledger <path>` names a file; `--ledger`
|
|
2651
|
+
* bare takes the standard's default location. Absent ⇒ nothing is recorded, so a caller who
|
|
2652
|
+
* never asked for a ledger never gets a surprise file in their tree. */
|
|
2653
|
+
function ledgerTarget(p: ParsedArgs, standard: StandardId): string | undefined {
|
|
2654
|
+
const flag = p.flags.ledger;
|
|
2655
|
+
if (flag === undefined || flag === false) return undefined;
|
|
2656
|
+
return typeof flag === "string" && flag ? flag : ledgerPath(standard);
|
|
2657
|
+
}
|
|
2658
|
+
|
|
2659
|
+
/** `verify --apply <ledger.json> --in <audit.json> --out <dir>` — REPLAY stored verdicts onto
|
|
2660
|
+
* a fresh audit, with no model in the loop.
|
|
2661
|
+
*
|
|
2662
|
+
* This is the parity mechanism: the same `applyAdjudication` gate runs, on evidence re-derived
|
|
2663
|
+
* from the audit in front of it, so a stored verdict has to prove itself again exactly as it
|
|
2664
|
+
* did the day it was recorded. A verdict whose evidence has since changed is dropped as stale
|
|
2665
|
+
* and its criterion returns to « to assess », saying so. */
|
|
2666
|
+
function replayLedgerFile(p: ParsedArgs, ledger: VerdictLedger, lang: Lang): number {
|
|
2667
|
+
const inFlag = p.flags.in;
|
|
2668
|
+
if (typeof inFlag !== "string" || !inFlag) {
|
|
2669
|
+
console.error(
|
|
2670
|
+
lang === "fr"
|
|
2671
|
+
? "ultra11y verify : --apply <registre> exige --in <audit.json> (l'audit à mettre à jour)."
|
|
2672
|
+
: "ultra11y verify: --apply <ledger> requires --in <audit.json> (the audit to update).",
|
|
2673
|
+
);
|
|
2674
|
+
return 2;
|
|
2675
|
+
}
|
|
2676
|
+
let audit: AuditResult;
|
|
2677
|
+
try {
|
|
2678
|
+
audit = JSON.parse(readText(inFlag)) as AuditResult;
|
|
2679
|
+
} catch {
|
|
2680
|
+
console.error(`ultra11y verify: --in file not found or not valid JSON: ${inFlag}.`);
|
|
2681
|
+
return 2;
|
|
2682
|
+
}
|
|
2683
|
+
const cwd = typeof p.flags.cwd === "string" ? (p.flags.cwd as string) : undefined;
|
|
2684
|
+
const standard = resolveStandard(p.flags.standard) ?? ledger.standard;
|
|
2685
|
+
const rp = replayLedger(audit, ledger, { cwd, standard });
|
|
2686
|
+
const r = applyAdjudication(audit, rp.adj, { cwd, strict: p.flags.strict === true, residualReasons: rp.residualReasons });
|
|
2687
|
+
|
|
2688
|
+
if (!r.ok && (p.flags.strict === true || r.applied + r.stillManual === 0)) {
|
|
2689
|
+
if (p.flags.json) console.log(JSON.stringify({ ...r, replay: rp }, null, 2));
|
|
2690
|
+
else {
|
|
2691
|
+
console.error(lang === "fr" ? `✗ Registre rejeté (${r.issues.length} problème(s)) :` : `✗ Ledger rejected (${r.issues.length} issue(s)):`);
|
|
2692
|
+
for (const i of r.issues) console.error(` ✗ ${i}`);
|
|
2693
|
+
}
|
|
2694
|
+
return 1;
|
|
2695
|
+
}
|
|
2696
|
+
const out = typeof p.flags.out === "string" ? (p.flags.out as string) : ".";
|
|
2697
|
+
mkdirSync(out, { recursive: true });
|
|
2698
|
+
const auditPath = join(out, "audit-latest.json");
|
|
2699
|
+
writeFileSync(auditPath, `${JSON.stringify(r.audit, null, 2)}\n`);
|
|
2700
|
+
|
|
2701
|
+
if (p.flags.json)
|
|
2702
|
+
console.log(
|
|
2703
|
+
JSON.stringify(
|
|
2704
|
+
{
|
|
2705
|
+
ok: true,
|
|
2706
|
+
auditPath,
|
|
2707
|
+
applied: r.applied,
|
|
2708
|
+
stillManual: r.stillManual,
|
|
2709
|
+
rejected: r.rejected,
|
|
2710
|
+
rejectedCriteria: r.rejectedCriteria,
|
|
2711
|
+
replayed: rp.fresh.length,
|
|
2712
|
+
stale: rp.stale,
|
|
2713
|
+
obsolete: rp.obsolete,
|
|
2714
|
+
missing: rp.missing,
|
|
2715
|
+
issues: r.issues,
|
|
2716
|
+
conformancePct: r.audit.conformancePct,
|
|
2717
|
+
grounding: r.grounding,
|
|
2718
|
+
},
|
|
2719
|
+
null,
|
|
2720
|
+
2,
|
|
2721
|
+
),
|
|
2722
|
+
);
|
|
2723
|
+
else {
|
|
2724
|
+
console.log(
|
|
2725
|
+
lang === "fr"
|
|
2726
|
+
? `✓ ${r.applied} verdict(s) rejoué(s) depuis le registre, ${r.stillManual} laissé(s) en résiduel${r.rejected ? `, ${r.rejected} refusé(s)` : ""} → ${auditPath}`
|
|
2727
|
+
: `✓ ${r.applied} verdict(s) replayed from the ledger, ${r.stillManual} left residual${r.rejected ? `, ${r.rejected} refused` : ""} → ${auditPath}`,
|
|
2728
|
+
);
|
|
2729
|
+
// Named, never merely counted: these are exactly the criteria a refresh pass has to
|
|
2730
|
+
// adjudicate, and a silent count is what let a half-covered grid read as complete.
|
|
2731
|
+
if (rp.stale.length)
|
|
2732
|
+
console.error(
|
|
2733
|
+
lang === "fr"
|
|
2734
|
+
? `⚠ ${rp.stale.length} verdict(s) périmé(s) (l'évidence a changé) : ${rp.stale.join(", ")}`
|
|
2735
|
+
: `⚠ ${rp.stale.length} stale verdict(s) (the evidence changed): ${rp.stale.join(", ")}`,
|
|
2736
|
+
);
|
|
2737
|
+
if (rp.missing.length)
|
|
2738
|
+
console.error(
|
|
2739
|
+
lang === "fr"
|
|
2740
|
+
? `⚠ ${rp.missing.length} critère(s) absent(s) du registre : ${rp.missing.join(", ")}`
|
|
2741
|
+
: `⚠ ${rp.missing.length} criterion(ia) absent from the ledger: ${rp.missing.join(", ")}`,
|
|
2742
|
+
);
|
|
2743
|
+
if (rp.obsolete.length)
|
|
2744
|
+
console.error(
|
|
2745
|
+
lang === "fr"
|
|
2746
|
+
? `ℹ ${rp.obsolete.length} entrée(s) obsolète(s) (le moteur les décide désormais) : ${rp.obsolete.join(", ")}`
|
|
2747
|
+
: `ℹ ${rp.obsolete.length} obsolete entry(ies) (the engine now decides them): ${rp.obsolete.join(", ")}`,
|
|
2748
|
+
);
|
|
2749
|
+
}
|
|
2750
|
+
return 0;
|
|
2751
|
+
}
|
|
2752
|
+
|
|
2620
2753
|
/** `verify --apply <adjudication.json> --in <audit.json> --out <dir>` — fold an AI
|
|
2621
2754
|
* adjudication of the manual criteria back into the audit, fail-closed, then rewrite the
|
|
2622
2755
|
* audit JSON so `report`/`prd` re-render with the adjudicated statuses. */
|
|
@@ -2637,8 +2770,19 @@ function applyAdjudicationFile(p: ParsedArgs, adj: AdjudicationFile, lang: Lang)
|
|
|
2637
2770
|
console.error(`ultra11y verify: --in file not found or not valid JSON: ${inFlag}.`);
|
|
2638
2771
|
return 2;
|
|
2639
2772
|
}
|
|
2640
|
-
|
|
2641
|
-
|
|
2773
|
+
// A verdicts-only file (`ADJUDICATE.verdicts.json`) carries decisions without the evidence
|
|
2774
|
+
// they were made against. Put it back from the audit before folding: the worklist is a pure
|
|
2775
|
+
// function of the audit, so this reconstructs the very anchors the citation gate checks
|
|
2776
|
+
// against — the fold below is the same fold, with the same refusals.
|
|
2777
|
+
const cwd = typeof p.flags.cwd === "string" ? (p.flags.cwd as string) : undefined;
|
|
2778
|
+
hydrateAdjudication(adj, audit, { cwd });
|
|
2779
|
+
const strict = p.flags.strict === true;
|
|
2780
|
+
const r = applyAdjudication(audit, adj, { cwd, strict });
|
|
2781
|
+
// STRICT: one refusal discards the file, nothing is persisted — the historical contract.
|
|
2782
|
+
// PARTIAL (default): the refusals are reported, the criteria they condemn stay to assess,
|
|
2783
|
+
// and everything that proved itself is kept. Only a fold that landed NOTHING is an error;
|
|
2784
|
+
// otherwise a single bad verdict would again cost the whole run.
|
|
2785
|
+
if (!r.ok && (strict || r.applied + r.stillManual === 0)) {
|
|
2642
2786
|
if (p.flags.json) console.log(JSON.stringify(r, null, 2));
|
|
2643
2787
|
else {
|
|
2644
2788
|
console.error(lang === "fr" ? `✗ Adjudication rejetée (${r.issues.length} problème(s)) :` : `✗ Adjudication rejected (${r.issues.length} issue(s)):`);
|
|
@@ -2646,11 +2790,35 @@ function applyAdjudicationFile(p: ParsedArgs, adj: AdjudicationFile, lang: Lang)
|
|
|
2646
2790
|
}
|
|
2647
2791
|
return 1;
|
|
2648
2792
|
}
|
|
2793
|
+
if (r.rejected > 0 && !p.flags.json) {
|
|
2794
|
+
console.error(
|
|
2795
|
+
lang === "fr"
|
|
2796
|
+
? `⚠ ${r.rejected} critère(s) refusé(s) par la porte — ils restent à évaluer, avec le motif du refus :`
|
|
2797
|
+
: `⚠ ${r.rejected} criterion(ia) refused by the gate — they stay to assess, carrying the refusal:`,
|
|
2798
|
+
);
|
|
2799
|
+
for (const i of r.issues) console.error(` ✗ ${i}`);
|
|
2800
|
+
}
|
|
2649
2801
|
// Persist the updated audit so report/prd re-render with the adjudicated statuses.
|
|
2650
2802
|
const out = typeof p.flags.out === "string" ? (p.flags.out as string) : ".";
|
|
2651
2803
|
mkdirSync(out, { recursive: true });
|
|
2652
2804
|
const auditPath = join(out, "audit-latest.json");
|
|
2653
2805
|
writeFileSync(auditPath, JSON.stringify(r.audit, null, 2) + "\n");
|
|
2806
|
+
// Record what landed, so the next run does not have to pay a model to learn it again. Only
|
|
2807
|
+
// the ACCEPTED verdicts go in: a refused one in the ledger would be laundered back on the
|
|
2808
|
+
// next replay, which is the whole thing the gate exists to prevent.
|
|
2809
|
+
const ledgerOut = ledgerTarget(p, adj.standard);
|
|
2810
|
+
if (ledgerOut) {
|
|
2811
|
+
const refused = new Set(r.rejectedCriteria);
|
|
2812
|
+
const accepted = new Set(adj.items.map((it) => it.criteriaId).filter((id) => !refused.has(id)));
|
|
2813
|
+
const fresh = entriesFrom(adj, accepted, r.audit.date);
|
|
2814
|
+
writeLedger(ledgerOut, mergeLedger(readLedger(ledgerOut), adj.standard, fresh));
|
|
2815
|
+
if (!p.flags.json)
|
|
2816
|
+
console.log(
|
|
2817
|
+
lang === "fr"
|
|
2818
|
+
? `✓ ${fresh.length} verdict(s) enregistré(s) au registre → ${ledgerOut}`
|
|
2819
|
+
: `✓ ${fresh.length} verdict(s) recorded in the ledger → ${ledgerOut}`,
|
|
2820
|
+
);
|
|
2821
|
+
}
|
|
2654
2822
|
// Carry the numbers the fold produced. The failure payload has always described its own
|
|
2655
2823
|
// outcome (`issues`); the success one described only the mechanics, so a caller that gates
|
|
2656
2824
|
// on the adjudicated result — a CI step, an orchestrator's tool node — had to re-read and
|
|
@@ -2663,6 +2831,9 @@ function applyAdjudicationFile(p: ParsedArgs, adj: AdjudicationFile, lang: Lang)
|
|
|
2663
2831
|
auditPath,
|
|
2664
2832
|
applied: r.applied,
|
|
2665
2833
|
stillManual: r.stillManual,
|
|
2834
|
+
rejected: r.rejected,
|
|
2835
|
+
rejectedCriteria: r.rejectedCriteria,
|
|
2836
|
+
issues: r.issues,
|
|
2666
2837
|
conformancePct: r.audit.conformancePct,
|
|
2667
2838
|
findings: r.audit.findings.length,
|
|
2668
2839
|
grounding: r.grounding,
|
|
@@ -2674,8 +2845,8 @@ function applyAdjudicationFile(p: ParsedArgs, adj: AdjudicationFile, lang: Lang)
|
|
|
2674
2845
|
else
|
|
2675
2846
|
console.log(
|
|
2676
2847
|
lang === "fr"
|
|
2677
|
-
? `✓ ${r.applied} critère(s) adjugé(s), ${r.stillManual} laissé(s) en résiduel → ${auditPath}`
|
|
2678
|
-
: `✓ ${r.applied} criterion(ia) adjudicated, ${r.stillManual} left residual → ${auditPath}`,
|
|
2848
|
+
? `✓ ${r.applied} critère(s) adjugé(s), ${r.stillManual} laissé(s) en résiduel${r.rejected ? `, ${r.rejected} refusé(s)` : ""} → ${auditPath}`
|
|
2849
|
+
: `✓ ${r.applied} criterion(ia) adjudicated, ${r.stillManual} left residual${r.rejected ? `, ${r.rejected} refused` : ""} → ${auditPath}`,
|
|
2679
2850
|
);
|
|
2680
2851
|
return 0;
|
|
2681
2852
|
}
|
|
@@ -2734,16 +2905,18 @@ async function cmdJudge(p: ParsedArgs): Promise<number> {
|
|
|
2734
2905
|
}
|
|
2735
2906
|
const max = typeof p.flags.max === "string" ? Number(p.flags.max) : undefined;
|
|
2736
2907
|
const truncated = max !== undefined && Number.isFinite(max) && max > 0 && items.length > max;
|
|
2737
|
-
// `--max` bounds the spend
|
|
2738
|
-
//
|
|
2739
|
-
//
|
|
2740
|
-
|
|
2908
|
+
// `--max` bounds the spend. It used to be refused outright together with `--apply`, and
|
|
2909
|
+
// rightly so: the fold was all-or-nothing, so a bounded adjudication could only ever fail on
|
|
2910
|
+
// coverage — billing a full run to guarantee a failure. The fold is now per-verdict, so a
|
|
2911
|
+
// bounded run lands what it covered and leaves the rest to assess, which is precisely what
|
|
2912
|
+
// someone bounding spend is asking for. The pair is impossible only under `--strict`.
|
|
2913
|
+
if (truncated && p.flags.apply === true && p.flags.strict === true) {
|
|
2741
2914
|
console.error(
|
|
2742
2915
|
lang === "fr"
|
|
2743
|
-
? `ultra11y judge : --max ${max} ne couvre que ${max} des ${items.length} critères, et --apply exige une adjudication COMPLÈTE (le
|
|
2744
|
-
" Retirez --max pour tout adjuger, ou retirez --apply pour produire la worklist et l'appliquer plus tard."
|
|
2745
|
-
: `ultra11y judge: --max ${max} covers only ${max} of ${items.length} criteria, and --apply requires a COMPLETE adjudication (the
|
|
2746
|
-
" Drop --max to adjudicate everything, or drop --apply to produce the worklist and apply it later.",
|
|
2916
|
+
? `ultra11y judge : --max ${max} ne couvre que ${max} des ${items.length} critères, et --apply --strict exige une adjudication COMPLÈTE (le fold tout-ou-rien refuse une couverture partielle).\n` +
|
|
2917
|
+
" Retirez --strict pour appliquer ce qui est couvert, retirez --max pour tout adjuger, ou retirez --apply pour produire la worklist et l'appliquer plus tard."
|
|
2918
|
+
: `ultra11y judge: --max ${max} covers only ${max} of ${items.length} criteria, and --apply --strict requires a COMPLETE adjudication (the all-or-nothing fold refuses partial coverage).\n` +
|
|
2919
|
+
" Drop --strict to apply what is covered, drop --max to adjudicate everything, or drop --apply to produce the worklist and apply it later.",
|
|
2747
2920
|
);
|
|
2748
2921
|
return 2;
|
|
2749
2922
|
}
|
|
@@ -2799,20 +2972,45 @@ async function cmdJudge(p: ParsedArgs): Promise<number> {
|
|
|
2799
2972
|
// --apply runs the SAME gate an agent's verdicts go through. A truncated run cannot pass
|
|
2800
2973
|
// its coverage check, which is the correct outcome, not a bug.
|
|
2801
2974
|
const adj = JSON.parse(readText(w.todoPath)) as AdjudicationFile;
|
|
2802
|
-
const r = applyAdjudication(audit, adj);
|
|
2803
|
-
|
|
2975
|
+
const r = applyAdjudication(audit, adj, { strict: p.flags.strict === true });
|
|
2976
|
+
// Same policy as `verify --apply`: a batch the model got wrong costs its own criteria, not
|
|
2977
|
+
// the whole paid run. Only a fold that landed nothing at all is an error.
|
|
2978
|
+
if (!r.ok && (p.flags.strict === true || r.applied + r.stillManual === 0)) {
|
|
2804
2979
|
console.error(lang === "fr" ? `✗ Adjudication rejetée (${r.issues.length} problème(s)) :` : `✗ Adjudication rejected (${r.issues.length} issue(s)):`);
|
|
2805
2980
|
for (const i of r.issues.slice(0, 40)) console.error(` ✗ ${i}`);
|
|
2806
2981
|
if (r.issues.length > 40) console.error(` … +${r.issues.length - 40}`);
|
|
2807
2982
|
return 1;
|
|
2808
2983
|
}
|
|
2984
|
+
if (r.rejected > 0) {
|
|
2985
|
+
console.error(
|
|
2986
|
+
lang === "fr"
|
|
2987
|
+
? `⚠ ${r.rejected} critère(s) refusé(s) par la porte — ils restent à évaluer, avec le motif du refus :`
|
|
2988
|
+
: `⚠ ${r.rejected} criterion(ia) refused by the gate — they stay to assess, carrying the refusal:`,
|
|
2989
|
+
);
|
|
2990
|
+
for (const i of r.issues.slice(0, 40)) console.error(` ✗ ${i}`);
|
|
2991
|
+
if (r.issues.length > 40) console.error(` … +${r.issues.length - 40}`);
|
|
2992
|
+
}
|
|
2809
2993
|
mkdirSync(out, { recursive: true });
|
|
2810
2994
|
const auditPath = join(out, "audit-latest.json");
|
|
2811
2995
|
writeFileSync(auditPath, `${JSON.stringify(r.audit, null, 2)}\n`);
|
|
2996
|
+
// Same recording as `verify --apply`: what the model paid for, written down where the next
|
|
2997
|
+
// run can replay it for free.
|
|
2998
|
+
const ledgerOut = ledgerTarget(p, adj.standard);
|
|
2999
|
+
if (ledgerOut) {
|
|
3000
|
+
const refused = new Set(r.rejectedCriteria);
|
|
3001
|
+
const accepted = new Set(adj.items.map((it) => it.criteriaId).filter((id) => !refused.has(id)));
|
|
3002
|
+
const fresh = entriesFrom(adj, accepted, r.audit.date);
|
|
3003
|
+
writeLedger(ledgerOut, mergeLedger(readLedger(ledgerOut), adj.standard, fresh));
|
|
3004
|
+
console.log(
|
|
3005
|
+
lang === "fr"
|
|
3006
|
+
? `✓ ${fresh.length} verdict(s) enregistré(s) au registre → ${ledgerOut}`
|
|
3007
|
+
: `✓ ${fresh.length} verdict(s) recorded in the ledger → ${ledgerOut}`,
|
|
3008
|
+
);
|
|
3009
|
+
}
|
|
2812
3010
|
console.log(
|
|
2813
3011
|
lang === "fr"
|
|
2814
|
-
? `✓ ${r.applied} critère(s) adjugé(s), ${r.stillManual} laissé(s) en résiduel → ${auditPath}`
|
|
2815
|
-
: `✓ ${r.applied} criterion(ia) adjudicated, ${r.stillManual} left residual → ${auditPath}`,
|
|
3012
|
+
? `✓ ${r.applied} critère(s) adjugé(s), ${r.stillManual} laissé(s) en résiduel${r.rejected ? `, ${r.rejected} refusé(s)` : ""} → ${auditPath}`
|
|
3013
|
+
: `✓ ${r.applied} criterion(ia) adjudicated, ${r.stillManual} left residual${r.rejected ? `, ${r.rejected} refused` : ""} → ${auditPath}`,
|
|
2816
3014
|
);
|
|
2817
3015
|
return 0;
|
|
2818
3016
|
}
|