ultra11y 5.31.0 → 5.31.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.opencode/plugins/ultra11y.js +1 -1
- package/package.json +1 -1
- package/scripts/ultra11y.mjs +43 -5
- package/skills/review-a11y/SKILL.md +1 -1
- package/skills/review-a11y/scripts/ultra11y.mjs +43 -5
- package/skills/ultra11y/SKILL.md +1 -1
- package/skills/ultra11y/scripts/ultra11y.mjs +43 -5
- package/src/cli.ts +55 -3
- package/src/llm.ts +14 -1
- package/src/types.ts +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "ultra11y",
|
|
11
11
|
"source": "./",
|
|
12
12
|
"description": "Automatic accessibility review of the code under change before every commit, push and pull request, plus full WCAG 2.2 AA / RGAA audits and conformance deliverables on demand.",
|
|
13
|
-
"version": "5.31.
|
|
13
|
+
"version": "5.31.2",
|
|
14
14
|
"author": { "name": "maxgfr" },
|
|
15
15
|
"homepage": "https://github.com/maxgfr/ultra11y",
|
|
16
16
|
"repository": "https://github.com/maxgfr/ultra11y",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultra11y",
|
|
3
3
|
"description": "Accessibility for WCAG 2.2 AA and country standards (RGAA…): an automatic review of the code under change before every commit, push and pull request, plus full audits and conformance deliverables on demand.",
|
|
4
|
-
"version": "5.31.
|
|
4
|
+
"version": "5.31.2",
|
|
5
5
|
"author": { "name": "maxgfr" },
|
|
6
6
|
"homepage": "https://github.com/maxgfr/ultra11y",
|
|
7
7
|
"repository": "https://github.com/maxgfr/ultra11y",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultra11y",
|
|
3
3
|
"description": "Accessibility for WCAG 2.2 AA and country standards (RGAA…): an automatic review of the code under change before every commit, push and pull request, plus full audits and conformance deliverables on demand.",
|
|
4
|
-
"version": "5.31.
|
|
4
|
+
"version": "5.31.2",
|
|
5
5
|
"author": { "name": "maxgfr", "url": "https://github.com/maxgfr" },
|
|
6
6
|
"homepage": "https://github.com/maxgfr/ultra11y",
|
|
7
7
|
"repository": "https://github.com/maxgfr/ultra11y",
|
|
@@ -27,7 +27,7 @@ import { fileURLToPath } from "node:url";
|
|
|
27
27
|
// Ownership marker: `ultra11y install --opencode` refuses to overwrite a plugin file that
|
|
28
28
|
// does not carry it, so a user's own ultra11y.js is never clobbered.
|
|
29
29
|
const ULTRA11Y_OPENCODE_PLUGIN = "ULTRA11Y_OPENCODE_PLUGIN";
|
|
30
|
-
const ULTRA11Y_PLUGIN_VERSION = "5.31.
|
|
30
|
+
const ULTRA11Y_PLUGIN_VERSION = "5.31.2";
|
|
31
31
|
|
|
32
32
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
33
33
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultra11y",
|
|
3
|
-
"version": "5.31.
|
|
3
|
+
"version": "5.31.2",
|
|
4
4
|
"description": "Audit HTML/CSS/JSX against WCAG 2.2 AA and produce a dated compliance report, or author/review accessible markup (native-HTML-first). A skills.sh agent skill: an install-free static engine (bundled, no npm install) plus the agent's judgment, with check/verify gates against hallucinated non-conformities, and pluggable in-repo standards packs (RGAA France first; Section 508 & EN 301 549 next).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "pnpm@10.33.0",
|
package/scripts/ultra11y.mjs
CHANGED
|
@@ -6,7 +6,7 @@ import { join as join52, relative as relative5, resolve as resolve16, sep as sep
|
|
|
6
6
|
import { fileURLToPath as fileURLToPath5, pathToFileURL as pathToFileURL3 } from "url";
|
|
7
7
|
|
|
8
8
|
// src/types.ts
|
|
9
|
-
var VERSION = "5.31.
|
|
9
|
+
var VERSION = "5.31.2";
|
|
10
10
|
var SCHEMA_VERSION = 2;
|
|
11
11
|
var INAPPLICABLE_STATUS = "C";
|
|
12
12
|
|
|
@@ -60209,7 +60209,9 @@ async function judgeAll(batches, opts) {
|
|
|
60209
60209
|
Array.from({ length: Math.min(lanes, queue.length) }, async () => {
|
|
60210
60210
|
for (let b = queue.shift(); b !== void 0; b = queue.shift()) {
|
|
60211
60211
|
try {
|
|
60212
|
-
|
|
60212
|
+
const landed = await backend(b.items, b.prompt, opts);
|
|
60213
|
+
verdicts.push(...landed);
|
|
60214
|
+
if (landed.length) opts.onVerdicts?.(landed);
|
|
60213
60215
|
} catch (e) {
|
|
60214
60216
|
failures.push(e instanceof Error ? e.message : String(e));
|
|
60215
60217
|
}
|
|
@@ -68733,6 +68735,11 @@ function isCommand(s) {
|
|
|
68733
68735
|
return !!s && COMMANDS.includes(s);
|
|
68734
68736
|
}
|
|
68735
68737
|
var VALUE_FLAGS2 = /* @__PURE__ */ new Set([
|
|
68738
|
+
"runner",
|
|
68739
|
+
"grain",
|
|
68740
|
+
"max-budget-usd",
|
|
68741
|
+
"timeout",
|
|
68742
|
+
"concurrency",
|
|
68736
68743
|
"out",
|
|
68737
68744
|
"provider",
|
|
68738
68745
|
"grain",
|
|
@@ -70720,6 +70727,29 @@ async function cmdJudge(p) {
|
|
|
70720
70727
|
console.error(
|
|
70721
70728
|
lang === "fr" ? `ultra11y judge : ${items.length} crit\xE8re(s) en ${batches.length} lot(s), transport ${runner}, mod\xE8le ${model}\u2026` : `ultra11y judge: ${items.length} criterion(ia) in ${batches.length} batch(es), ${runner} transport, model ${model}\u2026`
|
|
70722
70729
|
);
|
|
70730
|
+
const outDir = typeof p.flags.out === "string" ? p.flags.out : ".";
|
|
70731
|
+
const cwd = typeof p.flags.cwd === "string" ? p.flags.cwd : void 0;
|
|
70732
|
+
const applying = p.flags.apply === true;
|
|
70733
|
+
const askedConcurrency = typeof p.flags.concurrency === "string" ? Number(p.flags.concurrency) : Number.NaN;
|
|
70734
|
+
const cliConcurrency = Number.isFinite(askedConcurrency) && askedConcurrency > 0 ? Math.min(askedConcurrency, 8) : 2;
|
|
70735
|
+
const CHECKPOINT_MS = 15e3;
|
|
70736
|
+
let lastCheckpoint = 0;
|
|
70737
|
+
const checkpoint = (force) => {
|
|
70738
|
+
if (!applying) return;
|
|
70739
|
+
const now = Date.now();
|
|
70740
|
+
if (!force && now - lastCheckpoint < CHECKPOINT_MS) return;
|
|
70741
|
+
lastCheckpoint = now;
|
|
70742
|
+
try {
|
|
70743
|
+
const partial = applyAdjudication(
|
|
70744
|
+
audit2,
|
|
70745
|
+
{ tool: "ultra11y", kind: "adjudication", schemaVersion: SCHEMA_VERSION, standard, auditDate: audit2.date, items },
|
|
70746
|
+
{ cwd }
|
|
70747
|
+
);
|
|
70748
|
+
if (partial.applied > 0) writeFileSync20(join52(outDir, "audit-latest.json"), `${JSON.stringify(partial.audit, null, 2)}
|
|
70749
|
+
`);
|
|
70750
|
+
} catch {
|
|
70751
|
+
}
|
|
70752
|
+
};
|
|
70723
70753
|
let spent = 0;
|
|
70724
70754
|
const maxBudgetUsd = typeof p.flags["max-budget-usd"] === "string" ? Number(p.flags["max-budget-usd"]) : void 0;
|
|
70725
70755
|
const timeoutMs = typeof p.flags.timeout === "string" ? Number(p.flags.timeout) * 1e3 : void 0;
|
|
@@ -70727,14 +70757,22 @@ async function cmdJudge(p) {
|
|
|
70727
70757
|
apiKey: key,
|
|
70728
70758
|
model,
|
|
70729
70759
|
backend: runner === "cli" ? judgeBatchCli : void 0,
|
|
70730
|
-
//
|
|
70731
|
-
//
|
|
70732
|
-
|
|
70760
|
+
// TWO local processes, not one, and the reason is measured rather than tuned: sequentially,
|
|
70761
|
+
// one criterion took ~75s on Haiku, so a 51-criterion sweep needs ~64 minutes and a CI job
|
|
70762
|
+
// that allows 45 is killed at 31. Two in flight brings it inside the ceiling; more than a
|
|
70763
|
+
// few is a different risk — each `claude` is its own Node process making its own API calls,
|
|
70764
|
+
// and four of them on a subscription token reach a rate limit faster than four HTTP
|
|
70765
|
+
// requests do. `--concurrency` overrides it for a runner with room.
|
|
70766
|
+
concurrency: runner === "cli" ? cliConcurrency : void 0,
|
|
70733
70767
|
maxBudgetUsd: Number.isFinite(maxBudgetUsd) ? maxBudgetUsd : void 0,
|
|
70734
70768
|
timeoutMs: Number.isFinite(timeoutMs) ? timeoutMs : void 0,
|
|
70735
70769
|
onCost: (usd) => {
|
|
70736
70770
|
spent += usd;
|
|
70737
70771
|
},
|
|
70772
|
+
onVerdicts: (landed) => {
|
|
70773
|
+
applyRawVerdicts(items, landed);
|
|
70774
|
+
checkpoint(false);
|
|
70775
|
+
},
|
|
70738
70776
|
onProgress: (done, total) => console.error(` ${done}/${total}`)
|
|
70739
70777
|
});
|
|
70740
70778
|
if (spent > 0) console.error(lang === "fr" ? `ultra11y judge : ${spent.toFixed(4)} $ d\xE9pens\xE9(s).` : `ultra11y judge: ${spent.toFixed(4)} spent.`);
|
|
@@ -5,7 +5,7 @@ when_to_use: "Invoke WITHOUT being asked whenever changed front-end code should
|
|
|
5
5
|
allowed-tools: Bash(node ${CLAUDE_SKILL_DIR}/scripts/ultra11y.mjs *), Read, Grep, Glob
|
|
6
6
|
license: MIT
|
|
7
7
|
metadata:
|
|
8
|
-
version: 5.31.
|
|
8
|
+
version: 5.31.2
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# review-a11y — accessibility review of changed code
|
|
@@ -6,7 +6,7 @@ import { join as join52, relative as relative5, resolve as resolve16, sep as sep
|
|
|
6
6
|
import { fileURLToPath as fileURLToPath5, pathToFileURL as pathToFileURL3 } from "url";
|
|
7
7
|
|
|
8
8
|
// src/types.ts
|
|
9
|
-
var VERSION = "5.31.
|
|
9
|
+
var VERSION = "5.31.2";
|
|
10
10
|
var SCHEMA_VERSION = 2;
|
|
11
11
|
var INAPPLICABLE_STATUS = "C";
|
|
12
12
|
|
|
@@ -60209,7 +60209,9 @@ async function judgeAll(batches, opts) {
|
|
|
60209
60209
|
Array.from({ length: Math.min(lanes, queue.length) }, async () => {
|
|
60210
60210
|
for (let b = queue.shift(); b !== void 0; b = queue.shift()) {
|
|
60211
60211
|
try {
|
|
60212
|
-
|
|
60212
|
+
const landed = await backend(b.items, b.prompt, opts);
|
|
60213
|
+
verdicts.push(...landed);
|
|
60214
|
+
if (landed.length) opts.onVerdicts?.(landed);
|
|
60213
60215
|
} catch (e) {
|
|
60214
60216
|
failures.push(e instanceof Error ? e.message : String(e));
|
|
60215
60217
|
}
|
|
@@ -68733,6 +68735,11 @@ function isCommand(s) {
|
|
|
68733
68735
|
return !!s && COMMANDS.includes(s);
|
|
68734
68736
|
}
|
|
68735
68737
|
var VALUE_FLAGS2 = /* @__PURE__ */ new Set([
|
|
68738
|
+
"runner",
|
|
68739
|
+
"grain",
|
|
68740
|
+
"max-budget-usd",
|
|
68741
|
+
"timeout",
|
|
68742
|
+
"concurrency",
|
|
68736
68743
|
"out",
|
|
68737
68744
|
"provider",
|
|
68738
68745
|
"grain",
|
|
@@ -70720,6 +70727,29 @@ async function cmdJudge(p) {
|
|
|
70720
70727
|
console.error(
|
|
70721
70728
|
lang === "fr" ? `ultra11y judge : ${items.length} crit\xE8re(s) en ${batches.length} lot(s), transport ${runner}, mod\xE8le ${model}\u2026` : `ultra11y judge: ${items.length} criterion(ia) in ${batches.length} batch(es), ${runner} transport, model ${model}\u2026`
|
|
70722
70729
|
);
|
|
70730
|
+
const outDir = typeof p.flags.out === "string" ? p.flags.out : ".";
|
|
70731
|
+
const cwd = typeof p.flags.cwd === "string" ? p.flags.cwd : void 0;
|
|
70732
|
+
const applying = p.flags.apply === true;
|
|
70733
|
+
const askedConcurrency = typeof p.flags.concurrency === "string" ? Number(p.flags.concurrency) : Number.NaN;
|
|
70734
|
+
const cliConcurrency = Number.isFinite(askedConcurrency) && askedConcurrency > 0 ? Math.min(askedConcurrency, 8) : 2;
|
|
70735
|
+
const CHECKPOINT_MS = 15e3;
|
|
70736
|
+
let lastCheckpoint = 0;
|
|
70737
|
+
const checkpoint = (force) => {
|
|
70738
|
+
if (!applying) return;
|
|
70739
|
+
const now = Date.now();
|
|
70740
|
+
if (!force && now - lastCheckpoint < CHECKPOINT_MS) return;
|
|
70741
|
+
lastCheckpoint = now;
|
|
70742
|
+
try {
|
|
70743
|
+
const partial = applyAdjudication(
|
|
70744
|
+
audit2,
|
|
70745
|
+
{ tool: "ultra11y", kind: "adjudication", schemaVersion: SCHEMA_VERSION, standard, auditDate: audit2.date, items },
|
|
70746
|
+
{ cwd }
|
|
70747
|
+
);
|
|
70748
|
+
if (partial.applied > 0) writeFileSync20(join52(outDir, "audit-latest.json"), `${JSON.stringify(partial.audit, null, 2)}
|
|
70749
|
+
`);
|
|
70750
|
+
} catch {
|
|
70751
|
+
}
|
|
70752
|
+
};
|
|
70723
70753
|
let spent = 0;
|
|
70724
70754
|
const maxBudgetUsd = typeof p.flags["max-budget-usd"] === "string" ? Number(p.flags["max-budget-usd"]) : void 0;
|
|
70725
70755
|
const timeoutMs = typeof p.flags.timeout === "string" ? Number(p.flags.timeout) * 1e3 : void 0;
|
|
@@ -70727,14 +70757,22 @@ async function cmdJudge(p) {
|
|
|
70727
70757
|
apiKey: key,
|
|
70728
70758
|
model,
|
|
70729
70759
|
backend: runner === "cli" ? judgeBatchCli : void 0,
|
|
70730
|
-
//
|
|
70731
|
-
//
|
|
70732
|
-
|
|
70760
|
+
// TWO local processes, not one, and the reason is measured rather than tuned: sequentially,
|
|
70761
|
+
// one criterion took ~75s on Haiku, so a 51-criterion sweep needs ~64 minutes and a CI job
|
|
70762
|
+
// that allows 45 is killed at 31. Two in flight brings it inside the ceiling; more than a
|
|
70763
|
+
// few is a different risk — each `claude` is its own Node process making its own API calls,
|
|
70764
|
+
// and four of them on a subscription token reach a rate limit faster than four HTTP
|
|
70765
|
+
// requests do. `--concurrency` overrides it for a runner with room.
|
|
70766
|
+
concurrency: runner === "cli" ? cliConcurrency : void 0,
|
|
70733
70767
|
maxBudgetUsd: Number.isFinite(maxBudgetUsd) ? maxBudgetUsd : void 0,
|
|
70734
70768
|
timeoutMs: Number.isFinite(timeoutMs) ? timeoutMs : void 0,
|
|
70735
70769
|
onCost: (usd) => {
|
|
70736
70770
|
spent += usd;
|
|
70737
70771
|
},
|
|
70772
|
+
onVerdicts: (landed) => {
|
|
70773
|
+
applyRawVerdicts(items, landed);
|
|
70774
|
+
checkpoint(false);
|
|
70775
|
+
},
|
|
70738
70776
|
onProgress: (done, total) => console.error(` ${done}/${total}`)
|
|
70739
70777
|
});
|
|
70740
70778
|
if (spent > 0) console.error(lang === "fr" ? `ultra11y judge : ${spent.toFixed(4)} $ d\xE9pens\xE9(s).` : `ultra11y judge: ${spent.toFixed(4)} spent.`);
|
package/skills/ultra11y/SKILL.md
CHANGED
|
@@ -4,7 +4,7 @@ description: "Use to AUDIT existing HTML/CSS/JSX against WCAG 2.2 AA accessibili
|
|
|
4
4
|
when_to_use: "Invoke when the user asks for an accessibility AUDIT or a formal deliverable: audit a repo, site or page against WCAG 2.2 AA or a country standard (RGAA, Section 508, EN 301 549); produce a dated conformance report, a criterion grid, a PRD backlog, or tickets (GitHub/GitLab/Jira); author accessible markup from scratch; wire the repo gate (init --hook/--ci) or a rendered-DOM capture pipeline. For a review of the code UNDER CHANGE (diff, branch, PR), use the `review-a11y` skill instead."
|
|
5
5
|
license: MIT
|
|
6
6
|
metadata:
|
|
7
|
-
version: 5.31.
|
|
7
|
+
version: 5.31.2
|
|
8
8
|
---
|
|
9
9
|
|
|
10
10
|
# ultra11y — audit WCAG 2.2 AA and write accessible markup
|
|
@@ -6,7 +6,7 @@ import { join as join52, relative as relative5, resolve as resolve16, sep as sep
|
|
|
6
6
|
import { fileURLToPath as fileURLToPath5, pathToFileURL as pathToFileURL3 } from "url";
|
|
7
7
|
|
|
8
8
|
// src/types.ts
|
|
9
|
-
var VERSION = "5.31.
|
|
9
|
+
var VERSION = "5.31.2";
|
|
10
10
|
var SCHEMA_VERSION = 2;
|
|
11
11
|
var INAPPLICABLE_STATUS = "C";
|
|
12
12
|
|
|
@@ -60209,7 +60209,9 @@ async function judgeAll(batches, opts) {
|
|
|
60209
60209
|
Array.from({ length: Math.min(lanes, queue.length) }, async () => {
|
|
60210
60210
|
for (let b = queue.shift(); b !== void 0; b = queue.shift()) {
|
|
60211
60211
|
try {
|
|
60212
|
-
|
|
60212
|
+
const landed = await backend(b.items, b.prompt, opts);
|
|
60213
|
+
verdicts.push(...landed);
|
|
60214
|
+
if (landed.length) opts.onVerdicts?.(landed);
|
|
60213
60215
|
} catch (e) {
|
|
60214
60216
|
failures.push(e instanceof Error ? e.message : String(e));
|
|
60215
60217
|
}
|
|
@@ -68733,6 +68735,11 @@ function isCommand(s) {
|
|
|
68733
68735
|
return !!s && COMMANDS.includes(s);
|
|
68734
68736
|
}
|
|
68735
68737
|
var VALUE_FLAGS2 = /* @__PURE__ */ new Set([
|
|
68738
|
+
"runner",
|
|
68739
|
+
"grain",
|
|
68740
|
+
"max-budget-usd",
|
|
68741
|
+
"timeout",
|
|
68742
|
+
"concurrency",
|
|
68736
68743
|
"out",
|
|
68737
68744
|
"provider",
|
|
68738
68745
|
"grain",
|
|
@@ -70720,6 +70727,29 @@ async function cmdJudge(p) {
|
|
|
70720
70727
|
console.error(
|
|
70721
70728
|
lang === "fr" ? `ultra11y judge : ${items.length} crit\xE8re(s) en ${batches.length} lot(s), transport ${runner}, mod\xE8le ${model}\u2026` : `ultra11y judge: ${items.length} criterion(ia) in ${batches.length} batch(es), ${runner} transport, model ${model}\u2026`
|
|
70722
70729
|
);
|
|
70730
|
+
const outDir = typeof p.flags.out === "string" ? p.flags.out : ".";
|
|
70731
|
+
const cwd = typeof p.flags.cwd === "string" ? p.flags.cwd : void 0;
|
|
70732
|
+
const applying = p.flags.apply === true;
|
|
70733
|
+
const askedConcurrency = typeof p.flags.concurrency === "string" ? Number(p.flags.concurrency) : Number.NaN;
|
|
70734
|
+
const cliConcurrency = Number.isFinite(askedConcurrency) && askedConcurrency > 0 ? Math.min(askedConcurrency, 8) : 2;
|
|
70735
|
+
const CHECKPOINT_MS = 15e3;
|
|
70736
|
+
let lastCheckpoint = 0;
|
|
70737
|
+
const checkpoint = (force) => {
|
|
70738
|
+
if (!applying) return;
|
|
70739
|
+
const now = Date.now();
|
|
70740
|
+
if (!force && now - lastCheckpoint < CHECKPOINT_MS) return;
|
|
70741
|
+
lastCheckpoint = now;
|
|
70742
|
+
try {
|
|
70743
|
+
const partial = applyAdjudication(
|
|
70744
|
+
audit2,
|
|
70745
|
+
{ tool: "ultra11y", kind: "adjudication", schemaVersion: SCHEMA_VERSION, standard, auditDate: audit2.date, items },
|
|
70746
|
+
{ cwd }
|
|
70747
|
+
);
|
|
70748
|
+
if (partial.applied > 0) writeFileSync20(join52(outDir, "audit-latest.json"), `${JSON.stringify(partial.audit, null, 2)}
|
|
70749
|
+
`);
|
|
70750
|
+
} catch {
|
|
70751
|
+
}
|
|
70752
|
+
};
|
|
70723
70753
|
let spent = 0;
|
|
70724
70754
|
const maxBudgetUsd = typeof p.flags["max-budget-usd"] === "string" ? Number(p.flags["max-budget-usd"]) : void 0;
|
|
70725
70755
|
const timeoutMs = typeof p.flags.timeout === "string" ? Number(p.flags.timeout) * 1e3 : void 0;
|
|
@@ -70727,14 +70757,22 @@ async function cmdJudge(p) {
|
|
|
70727
70757
|
apiKey: key,
|
|
70728
70758
|
model,
|
|
70729
70759
|
backend: runner === "cli" ? judgeBatchCli : void 0,
|
|
70730
|
-
//
|
|
70731
|
-
//
|
|
70732
|
-
|
|
70760
|
+
// TWO local processes, not one, and the reason is measured rather than tuned: sequentially,
|
|
70761
|
+
// one criterion took ~75s on Haiku, so a 51-criterion sweep needs ~64 minutes and a CI job
|
|
70762
|
+
// that allows 45 is killed at 31. Two in flight brings it inside the ceiling; more than a
|
|
70763
|
+
// few is a different risk — each `claude` is its own Node process making its own API calls,
|
|
70764
|
+
// and four of them on a subscription token reach a rate limit faster than four HTTP
|
|
70765
|
+
// requests do. `--concurrency` overrides it for a runner with room.
|
|
70766
|
+
concurrency: runner === "cli" ? cliConcurrency : void 0,
|
|
70733
70767
|
maxBudgetUsd: Number.isFinite(maxBudgetUsd) ? maxBudgetUsd : void 0,
|
|
70734
70768
|
timeoutMs: Number.isFinite(timeoutMs) ? timeoutMs : void 0,
|
|
70735
70769
|
onCost: (usd) => {
|
|
70736
70770
|
spent += usd;
|
|
70737
70771
|
},
|
|
70772
|
+
onVerdicts: (landed) => {
|
|
70773
|
+
applyRawVerdicts(items, landed);
|
|
70774
|
+
checkpoint(false);
|
|
70775
|
+
},
|
|
70738
70776
|
onProgress: (done, total) => console.error(` ${done}/${total}`)
|
|
70739
70777
|
});
|
|
70740
70778
|
if (spent > 0) console.error(lang === "fr" ? `ultra11y judge : ${spent.toFixed(4)} $ d\xE9pens\xE9(s).` : `ultra11y judge: ${spent.toFixed(4)} spent.`);
|
package/src/cli.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { realpathSync, writeFileSync, mkdirSync, existsSync, readFileSync, appen
|
|
|
2
2
|
import { join, relative, resolve, sep, dirname } from "node:path";
|
|
3
3
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4
4
|
import {
|
|
5
|
+
SCHEMA_VERSION,
|
|
5
6
|
VERSION,
|
|
6
7
|
type Lang,
|
|
7
8
|
type AuditResult,
|
|
@@ -542,6 +543,11 @@ function isCommand(s: string | undefined): s is Command {
|
|
|
542
543
|
}
|
|
543
544
|
|
|
544
545
|
const VALUE_FLAGS = new Set([
|
|
546
|
+
"runner",
|
|
547
|
+
"grain",
|
|
548
|
+
"max-budget-usd",
|
|
549
|
+
"timeout",
|
|
550
|
+
"concurrency",
|
|
545
551
|
"out",
|
|
546
552
|
"provider",
|
|
547
553
|
"grain",
|
|
@@ -3273,6 +3279,44 @@ async function cmdJudge(p: ParsedArgs): Promise<number> {
|
|
|
3273
3279
|
: `ultra11y judge: ${items.length} criterion(ia) in ${batches.length} batch(es), ${runner} transport, model ${model}…`,
|
|
3274
3280
|
);
|
|
3275
3281
|
|
|
3282
|
+
const outDir = typeof p.flags.out === "string" ? (p.flags.out as string) : ".";
|
|
3283
|
+
const cwd = typeof p.flags.cwd === "string" ? (p.flags.cwd as string) : undefined;
|
|
3284
|
+
const applying = p.flags.apply === true;
|
|
3285
|
+
const askedConcurrency = typeof p.flags.concurrency === "string" ? Number(p.flags.concurrency) : Number.NaN;
|
|
3286
|
+
const cliConcurrency = Number.isFinite(askedConcurrency) && askedConcurrency > 0 ? Math.min(askedConcurrency, 8) : 2;
|
|
3287
|
+
|
|
3288
|
+
// CHECKPOINTS, because a killed run used to lose everything it had already paid for.
|
|
3289
|
+
//
|
|
3290
|
+
// Measured: a per-criterion pass reached 31 of 51 criteria in 40 minutes, the CI job hit its
|
|
3291
|
+
// 45-minute ceiling, the process was killed — and the audit came back with all 51 still to
|
|
3292
|
+
// assess. The fold only ran at the END, so 31 rulings were bought and thrown away. Isolating
|
|
3293
|
+
// the model call per criterion buys nothing while the WRITE is still one all-or-nothing
|
|
3294
|
+
// operation, which is exactly what the per-criterion grain was sold as fixing.
|
|
3295
|
+
//
|
|
3296
|
+
// Time-throttled rather than every batch: the fold re-grounds every citation against real
|
|
3297
|
+
// source, and doing that 51 times over would spend on bookkeeping what the grain saves.
|
|
3298
|
+
const CHECKPOINT_MS = 15_000;
|
|
3299
|
+
let lastCheckpoint = 0;
|
|
3300
|
+
const checkpoint = (force: boolean): void => {
|
|
3301
|
+
if (!applying) return;
|
|
3302
|
+
const now = Date.now();
|
|
3303
|
+
if (!force && now - lastCheckpoint < CHECKPOINT_MS) return;
|
|
3304
|
+
lastCheckpoint = now;
|
|
3305
|
+
try {
|
|
3306
|
+
const partial = applyAdjudication(
|
|
3307
|
+
audit,
|
|
3308
|
+
{ tool: "ultra11y", kind: "adjudication", schemaVersion: SCHEMA_VERSION, standard, auditDate: audit.date, items },
|
|
3309
|
+
{ cwd },
|
|
3310
|
+
);
|
|
3311
|
+
// A partial sweep cannot satisfy the COVERAGE check, and that is correct rather than a
|
|
3312
|
+
// problem: the per-verdict fold still lands every valid verdict, and the criteria nobody
|
|
3313
|
+
// has reached yet stay to assess. Only a checkpoint that landed nothing is worth skipping.
|
|
3314
|
+
if (partial.applied > 0) writeFileSync(join(outDir, "audit-latest.json"), `${JSON.stringify(partial.audit, null, 2)}\n`);
|
|
3315
|
+
} catch {
|
|
3316
|
+
// A checkpoint is a safety net, never a reason to lose the run it protects.
|
|
3317
|
+
}
|
|
3318
|
+
};
|
|
3319
|
+
|
|
3276
3320
|
let spent = 0;
|
|
3277
3321
|
// A dollar ceiling and a wall clock — never a turn budget. `--max-turns` is not a flag of
|
|
3278
3322
|
// the Claude Code CLI, and the CLI swallows unknown flags without a word, so passing one
|
|
@@ -3283,14 +3327,22 @@ async function cmdJudge(p: ParsedArgs): Promise<number> {
|
|
|
3283
3327
|
apiKey: key,
|
|
3284
3328
|
model,
|
|
3285
3329
|
backend: runner === "cli" ? judgeBatchCli : undefined,
|
|
3286
|
-
//
|
|
3287
|
-
//
|
|
3288
|
-
|
|
3330
|
+
// TWO local processes, not one, and the reason is measured rather than tuned: sequentially,
|
|
3331
|
+
// one criterion took ~75s on Haiku, so a 51-criterion sweep needs ~64 minutes and a CI job
|
|
3332
|
+
// that allows 45 is killed at 31. Two in flight brings it inside the ceiling; more than a
|
|
3333
|
+
// few is a different risk — each `claude` is its own Node process making its own API calls,
|
|
3334
|
+
// and four of them on a subscription token reach a rate limit faster than four HTTP
|
|
3335
|
+
// requests do. `--concurrency` overrides it for a runner with room.
|
|
3336
|
+
concurrency: runner === "cli" ? cliConcurrency : undefined,
|
|
3289
3337
|
maxBudgetUsd: Number.isFinite(maxBudgetUsd) ? maxBudgetUsd : undefined,
|
|
3290
3338
|
timeoutMs: Number.isFinite(timeoutMs) ? timeoutMs : undefined,
|
|
3291
3339
|
onCost: (usd) => {
|
|
3292
3340
|
spent += usd;
|
|
3293
3341
|
},
|
|
3342
|
+
onVerdicts: (landed) => {
|
|
3343
|
+
applyRawVerdicts(items, landed);
|
|
3344
|
+
checkpoint(false);
|
|
3345
|
+
},
|
|
3294
3346
|
onProgress: (done, total) => console.error(` ${done}/${total}`),
|
|
3295
3347
|
});
|
|
3296
3348
|
// What it cost, from the run itself rather than from a dashboard read a day later.
|
package/src/llm.ts
CHANGED
|
@@ -74,6 +74,15 @@ export interface LlmOptions {
|
|
|
74
74
|
/** Called with each invocation's real cost, so a run can report what it spent instead of
|
|
75
75
|
* leaving the reader to find it in a provider dashboard. */
|
|
76
76
|
onCost?: (usd: number) => void;
|
|
77
|
+
/** Called with each batch's verdicts AS THEY LAND, so a caller can persist them before the
|
|
78
|
+
* run is over.
|
|
79
|
+
*
|
|
80
|
+
* Measured, and it cost a whole paid run: a per-criterion pass reached 31 of 51 criteria in
|
|
81
|
+
* 40 minutes, the CI job hit its 45-minute ceiling, the process was killed — and because
|
|
82
|
+
* the fold only ran at the END, all 31 verdicts went with it. The audit came back with 51
|
|
83
|
+
* criteria still to assess, having paid for 31. Isolating the model CALL per criterion is
|
|
84
|
+
* worth nothing if the write is still one all-or-nothing operation at the end. */
|
|
85
|
+
onVerdicts?: (verdicts: RawVerdict[]) => void;
|
|
77
86
|
/** Backoff before retry N (1-based). Injected so a test can exercise the retry path
|
|
78
87
|
* without waiting out the real curve. */
|
|
79
88
|
backoffMs?: (attempt: number) => number;
|
|
@@ -270,7 +279,11 @@ export async function judgeAll(
|
|
|
270
279
|
Array.from({ length: Math.min(lanes, queue.length) }, async () => {
|
|
271
280
|
for (let b = queue.shift(); b !== undefined; b = queue.shift()) {
|
|
272
281
|
try {
|
|
273
|
-
|
|
282
|
+
const landed = await backend(b.items, b.prompt, opts);
|
|
283
|
+
verdicts.push(...landed);
|
|
284
|
+
// Handed over BEFORE the next batch starts, so a run that dies mid-sweep leaves
|
|
285
|
+
// behind what it had already ruled on rather than nothing at all.
|
|
286
|
+
if (landed.length) opts.onVerdicts?.(landed);
|
|
274
287
|
} catch (e) {
|
|
275
288
|
failures.push(e instanceof Error ? e.message : String(e));
|
|
276
289
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Single source of truth for shared types + the embedded version. sync-version.mjs
|
|
2
2
|
// rewrites VERSION at release time (kept in lockstep with package.json + SKILL.md).
|
|
3
|
-
export const VERSION = "5.31.
|
|
3
|
+
export const VERSION = "5.31.2";
|
|
4
4
|
export const SCHEMA_VERSION = 2;
|
|
5
5
|
|
|
6
6
|
// Lang = the UI FRAME's language — the `L` localization tables each module (report,
|