taste-lint 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/corpus/copywriting.jsonl +4 -4
- package/data/rule-drafts/interaction/interaction-layout-shift.yaml +3 -3
- package/data/rules/copywriting/copywriting-antithesis-contrast.yaml +43 -0
- package/data/rules/copywriting/copywriting-antithesis-frame.yaml +4 -2
- package/data/rules/copywriting/copywriting-artificial-balance.yaml +4 -1
- package/data/rules/copywriting/copywriting-boilerplate-offer.yaml +1 -1
- package/data/rules/copywriting/copywriting-canned-phrasing.yaml +1 -1
- package/data/rules/copywriting/copywriting-claim-without-evidence.yaml +10 -1
- package/data/rules/copywriting/copywriting-copula-avoidance.yaml +1 -1
- package/data/rules/copywriting/copywriting-dead-link.yaml +28 -0
- package/data/rules/copywriting/copywriting-em-dash-in-ui.yaml +23 -0
- package/data/rules/copywriting/copywriting-excessive-hedging.yaml +1 -1
- package/data/rules/copywriting/copywriting-external-arrow-internal-link.yaml +26 -0
- package/data/rules/copywriting/copywriting-fabricated-specificity.yaml +6 -3
- package/data/rules/copywriting/copywriting-generic-framing.yaml +12 -6
- package/data/rules/copywriting/copywriting-generic-language.yaml +12 -4
- package/data/rules/copywriting/copywriting-generic-validation.yaml +1 -1
- package/data/rules/copywriting/copywriting-machine-prose.yaml +4 -1
- package/data/rules/copywriting/copywriting-mechanism-over-outcome.yaml +4 -1
- package/data/rules/copywriting/copywriting-meta-commentary.yaml +1 -1
- package/data/rules/copywriting/copywriting-missing-reasoning-link.yaml +4 -1
- package/data/rules/copywriting/copywriting-over-explanation.yaml +4 -1
- package/data/rules/copywriting/copywriting-participle-tail.yaml +1 -1
- package/data/rules/copywriting/copywriting-prompt-restatement.yaml +5 -2
- package/data/rules/copywriting/copywriting-punctuation-tics.yaml +4 -1
- package/data/rules/copywriting/copywriting-reader-first.yaml +4 -1
- package/data/rules/copywriting/copywriting-repetitive-transitions.yaml +1 -1
- package/data/rules/copywriting/copywriting-semantic-links.yaml +2 -2
- package/data/rules/copywriting/copywriting-slogan-headings.yaml +27 -0
- package/data/rules/copywriting/copywriting-structural-symmetry.yaml +4 -1
- package/data/rules/copywriting/copywriting-tee-up.yaml +4 -2
- package/data/rules/copywriting/copywriting-unexplained-action.yaml +5 -2
- package/data/rules/copywriting/copywriting-unnatural-polish.yaml +8 -5
- package/data/rules/copywriting/copywriting-unverifiable-proof.yaml +1 -1
- package/data/rules/copywriting/copywriting-vague-error.yaml +2 -0
- package/data/rules/craft/craft-atmosphere-gradient.yaml +1 -1
- package/data/rules/craft/craft-decoration-no-role.yaml +2 -1
- package/data/rules/craft/craft-decorative-ordinals.yaml +26 -0
- package/data/rules/craft/craft-faux-product-chrome.yaml +4 -1
- package/data/rules/schema/rule.schema.json +7 -1
- package/dist/cli.js +565 -97
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +34 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +275 -17
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/data/rules/copywriting/copywriting-idea-repetition.yaml +0 -34
package/dist/cli.js
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
import { Command, CommanderError, Option } from "commander";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import readline from "node:readline";
|
|
6
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
5
7
|
import os from "node:os";
|
|
6
8
|
import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
7
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
8
9
|
import { parse } from "yaml";
|
|
9
10
|
import { fromMarkdown } from "mdast-util-from-markdown";
|
|
10
11
|
import { gfmFromMarkdown } from "mdast-util-gfm";
|
|
@@ -14,7 +15,7 @@ import { mdxFromMarkdown } from "mdast-util-mdx";
|
|
|
14
15
|
import { mdxjs } from "micromark-extension-mdxjs";
|
|
15
16
|
import { styleText } from "node:util";
|
|
16
17
|
//#region packages/cli/package.json
|
|
17
|
-
var version = "0.
|
|
18
|
+
var version = "0.4.0";
|
|
18
19
|
//#endregion
|
|
19
20
|
//#region src/lib/stats.ts
|
|
20
21
|
const makePRNG = (seed) => {
|
|
@@ -345,7 +346,9 @@ const CONTEXT_KEYS = [
|
|
|
345
346
|
"docType",
|
|
346
347
|
"element",
|
|
347
348
|
"role",
|
|
348
|
-
"neighbours"
|
|
349
|
+
"neighbours",
|
|
350
|
+
"earlier",
|
|
351
|
+
"brief"
|
|
349
352
|
];
|
|
350
353
|
//#endregion
|
|
351
354
|
//#region src/eval/corpus.ts
|
|
@@ -416,6 +419,7 @@ const loadCorpus = (corpusDir, rules, options = {}) => {
|
|
|
416
419
|
].includes(labelSource)) throw new Error(`Invalid corpus line ${where}: labelSource`);
|
|
417
420
|
if (labelSource === "ai" && (typeof r.labelModel !== "string" || !r.labelModel.trim() || typeof r.labelPromptHash !== "string" || !/^[a-f0-9]{64}$/.test(r.labelPromptHash))) throw new Error(`Invalid corpus line ${where}: AI labels require model and prompt hash`);
|
|
418
421
|
if (!isRecord$1(r.source) || typeof r.source.repo !== "string" || typeof r.source.path !== "string" || r.source.id !== void 0 && (typeof r.source.id !== "string" || !r.source.id.trim())) throw new Error(`Invalid corpus line ${where}: source needs repo and path`);
|
|
422
|
+
if (r.note !== void 0 && typeof r.note !== "string") throw new TypeError(`Invalid corpus line ${where}: note must be text`);
|
|
419
423
|
if (r.split !== void 0 && r.split !== "dev" && r.split !== "holdout") throw new Error(`Invalid corpus line ${where}: split must be dev or holdout`);
|
|
420
424
|
if (r.classes !== void 0 && (!Array.isArray(r.classes) || r.classes.some((c) => typeof c !== "string"))) throw new Error(`Invalid corpus line ${where}: classes must be strings`);
|
|
421
425
|
if (r.context !== void 0) {
|
|
@@ -442,6 +446,7 @@ const loadCorpus = (corpusDir, rules, options = {}) => {
|
|
|
442
446
|
labelSource,
|
|
443
447
|
labels,
|
|
444
448
|
neighbours: r.neighbours,
|
|
449
|
+
note: r.note,
|
|
445
450
|
source: r.source,
|
|
446
451
|
split: r.split ?? splitFor(JSON.stringify([r.source.repo, r.source.path])),
|
|
447
452
|
text: r.text
|
|
@@ -552,6 +557,218 @@ const pairedOutcomes = (items, rule, probabilities) => {
|
|
|
552
557
|
return contrast;
|
|
553
558
|
};
|
|
554
559
|
//#endregion
|
|
560
|
+
//#region src/lib/errors.ts
|
|
561
|
+
var InputError = class extends Error {
|
|
562
|
+
code;
|
|
563
|
+
details;
|
|
564
|
+
constructor(code, message, details = {}) {
|
|
565
|
+
super(message);
|
|
566
|
+
this.code = code;
|
|
567
|
+
this.details = details;
|
|
568
|
+
this.name = "InputError";
|
|
569
|
+
}
|
|
570
|
+
};
|
|
571
|
+
//#endregion
|
|
572
|
+
//#region src/scan/storage.ts
|
|
573
|
+
const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
574
|
+
const object$2 = (value) => !!value && typeof value === "object" && !Array.isArray(value);
|
|
575
|
+
const readJson = (file) => {
|
|
576
|
+
try {
|
|
577
|
+
return JSON.parse(fs.readFileSync(file, "utf-8"));
|
|
578
|
+
} catch {
|
|
579
|
+
throw new InputError("INVALID_SCAN_INPUT", `Cannot read JSON input: ${file}`);
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
const writeJson = (file, value) => {
|
|
583
|
+
fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
|
|
584
|
+
const temp = `${file}.${randomUUID()}.tmp`;
|
|
585
|
+
try {
|
|
586
|
+
fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
|
|
587
|
+
fs.renameSync(temp, file);
|
|
588
|
+
} finally {
|
|
589
|
+
fs.rmSync(temp, { force: true });
|
|
590
|
+
}
|
|
591
|
+
};
|
|
592
|
+
//#endregion
|
|
593
|
+
//#region src/eval/label.ts
|
|
594
|
+
const readSamples = (file) => {
|
|
595
|
+
const raw = readJson(file);
|
|
596
|
+
if (!object$2(raw) || raw.version !== 1 && raw.version !== 2 || !Array.isArray(raw.samples)) throw new InputError("INVALID_LABELS", `Expected a version 1 or 2 sample file from lint --samples: ${file}`);
|
|
597
|
+
return raw;
|
|
598
|
+
};
|
|
599
|
+
const show = (sample, index, total, first) => {
|
|
600
|
+
const { context, neighbours, text } = sample.item;
|
|
601
|
+
const lines = [
|
|
602
|
+
"",
|
|
603
|
+
`[${index}/${total}] ${sample.ruleId}${context?.docType ? ` · ${context.docType}` : ""}${context?.role ? ` · ${context.role}` : ""}`,
|
|
604
|
+
`Q: ${sample.criterion}`
|
|
605
|
+
];
|
|
606
|
+
const criteria = sample.rubric?.criteria;
|
|
607
|
+
if (first && criteria) for (const [key, side] of [["yes", criteria.true], ["no", criteria.false]]) {
|
|
608
|
+
if (!side) continue;
|
|
609
|
+
lines.push(` ${key}: ${side.what ?? ""}`);
|
|
610
|
+
for (const example of side.examples ?? []) lines.push(` e.g. ${example}`);
|
|
611
|
+
}
|
|
612
|
+
if (context?.headingAbove) lines.push(`Under heading: ${context.headingAbove}`);
|
|
613
|
+
if (neighbours?.prev) lines.push(`Before: ${neighbours.prev.text}`);
|
|
614
|
+
lines.push("Text:", ` ${text.split("\n").join("\n ")}`);
|
|
615
|
+
if (neighbours?.next) lines.push(`After: ${neighbours.next.text}`);
|
|
616
|
+
return `${lines.join("\n")}\n`;
|
|
617
|
+
};
|
|
618
|
+
/** Walk unlabelled samples, saving after every answer so quitting loses nothing. */
|
|
619
|
+
const labelSession = async (file, options) => {
|
|
620
|
+
const data = readSamples(file);
|
|
621
|
+
if (data.annotation !== void 0) throw new InputError("INVALID_LABELS", "This file carries AI annotation. Hand-label a fresh lint --samples file.");
|
|
622
|
+
const now = options.now ?? Date.now;
|
|
623
|
+
const deadline = options.minutes === void 0 ? Infinity : now() + options.minutes * 6e4;
|
|
624
|
+
const pending = data.samples.filter((s) => s.label === null && !s.unsure);
|
|
625
|
+
const rl = readline.createInterface({
|
|
626
|
+
input: options.input,
|
|
627
|
+
terminal: false
|
|
628
|
+
});
|
|
629
|
+
const lines = rl[Symbol.asyncIterator]();
|
|
630
|
+
const ask = async (prompt) => {
|
|
631
|
+
options.output.write(prompt);
|
|
632
|
+
const next = await lines.next();
|
|
633
|
+
return next.done ? void 0 : String(next.value).trim();
|
|
634
|
+
};
|
|
635
|
+
const seenRules = /* @__PURE__ */ new Set();
|
|
636
|
+
let labelled = 0;
|
|
637
|
+
let unsure = 0;
|
|
638
|
+
let stopped = "done";
|
|
639
|
+
try {
|
|
640
|
+
for (const [index, sample] of pending.entries()) {
|
|
641
|
+
if (now() >= deadline) {
|
|
642
|
+
stopped = "timebox";
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
options.output.write(show(sample, index + 1, pending.length, !seenRules.has(sample.ruleId)));
|
|
646
|
+
seenRules.add(sample.ruleId);
|
|
647
|
+
let answer;
|
|
648
|
+
do
|
|
649
|
+
answer = (await ask("[y] violation [n] fine [u] unsure [s] skip [q] quit > "))?.toLowerCase();
|
|
650
|
+
while (answer !== void 0 && ![
|
|
651
|
+
"y",
|
|
652
|
+
"n",
|
|
653
|
+
"u",
|
|
654
|
+
"s",
|
|
655
|
+
"q"
|
|
656
|
+
].includes(answer));
|
|
657
|
+
if (answer === void 0 || answer === "q") {
|
|
658
|
+
stopped = "quit";
|
|
659
|
+
break;
|
|
660
|
+
}
|
|
661
|
+
if (answer === "s") continue;
|
|
662
|
+
const note = await ask(answer === "u" ? "What would you need to decide? > " : "Note (enter to skip) > ");
|
|
663
|
+
if (answer === "u") {
|
|
664
|
+
sample.unsure = true;
|
|
665
|
+
unsure += 1;
|
|
666
|
+
} else {
|
|
667
|
+
sample.label = answer === "y";
|
|
668
|
+
labelled += 1;
|
|
669
|
+
}
|
|
670
|
+
if (note) sample.note = note;
|
|
671
|
+
writeJson(file, data);
|
|
672
|
+
if (note === void 0) {
|
|
673
|
+
stopped = "quit";
|
|
674
|
+
break;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
} finally {
|
|
678
|
+
rl.close();
|
|
679
|
+
}
|
|
680
|
+
return {
|
|
681
|
+
labelled,
|
|
682
|
+
remaining: data.samples.filter((s) => s.label === null && !s.unsure).length,
|
|
683
|
+
stopped,
|
|
684
|
+
unsure
|
|
685
|
+
};
|
|
686
|
+
};
|
|
687
|
+
/** Group hand labels by rule: unsure answers and notes are rubric gaps. */
|
|
688
|
+
const labelGaps = (files) => {
|
|
689
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
690
|
+
for (const file of files) for (const sample of readSamples(file).samples) {
|
|
691
|
+
if (sample.label === null && !sample.unsure) continue;
|
|
692
|
+
const gaps = byRule.get(sample.ruleId) ?? {
|
|
693
|
+
fine: 0,
|
|
694
|
+
notes: [],
|
|
695
|
+
ruleId: sample.ruleId,
|
|
696
|
+
unsure: 0,
|
|
697
|
+
violations: 0
|
|
698
|
+
};
|
|
699
|
+
if (sample.unsure) gaps.unsure += 1;
|
|
700
|
+
else if (sample.label) gaps.violations += 1;
|
|
701
|
+
else gaps.fine += 1;
|
|
702
|
+
if (sample.note) gaps.notes.push({
|
|
703
|
+
label: sample.unsure ? "unsure" : Boolean(sample.label),
|
|
704
|
+
note: sample.note,
|
|
705
|
+
text: sample.item.text
|
|
706
|
+
});
|
|
707
|
+
byRule.set(sample.ruleId, gaps);
|
|
708
|
+
}
|
|
709
|
+
return [...byRule.values()].toSorted((a, b) => b.unsure - a.unsure || b.notes.length - a.notes.length || a.ruleId.localeCompare(b.ruleId));
|
|
710
|
+
};
|
|
711
|
+
const renderGaps = (gaps) => {
|
|
712
|
+
if (!gaps.length) return "No hand labels yet. Run taste-lint eval label <samples.json>.\n";
|
|
713
|
+
const out = [];
|
|
714
|
+
for (const g of gaps) {
|
|
715
|
+
out.push(`${g.ruleId}: ${g.violations} violation, ${g.fine} fine, ${g.unsure} unsure`);
|
|
716
|
+
if (g.unsure) out.push(" Unsure answers mean the rubric cannot settle these; sharpen question.criteria before tuning.");
|
|
717
|
+
for (const n of g.notes) {
|
|
718
|
+
const label = n.label === "unsure" ? "unsure" : n.label ? "yes" : "no";
|
|
719
|
+
out.push(` [${label}] ${n.note}`, ` ${n.text.replaceAll("\n", " ").slice(0, 100)}`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
return `${out.join("\n")}\n`;
|
|
723
|
+
};
|
|
724
|
+
/** How often a second labeller (usually an AI) matches your hand labels, per rule. */
|
|
725
|
+
const labelAgreement = (handFile, otherFile, floor = .8) => {
|
|
726
|
+
const hand = new Map(readSamples(handFile).samples.filter((s) => s.label !== null).map((s) => [s.id, s]));
|
|
727
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
728
|
+
for (const other of readSamples(otherFile).samples) {
|
|
729
|
+
const mine = hand.get(other.id);
|
|
730
|
+
if (!mine || other.label === null || mine.ruleId !== other.ruleId) continue;
|
|
731
|
+
const tally = byRule.get(other.ruleId) ?? {
|
|
732
|
+
agree: 0,
|
|
733
|
+
n: 0
|
|
734
|
+
};
|
|
735
|
+
tally.n += 1;
|
|
736
|
+
tally.agree += mine.label === other.label ? 1 : 0;
|
|
737
|
+
byRule.set(other.ruleId, tally);
|
|
738
|
+
}
|
|
739
|
+
return [...byRule.entries()].map(([ruleId, { n, agree }]) => {
|
|
740
|
+
const ci = wilson(agree, n);
|
|
741
|
+
return {
|
|
742
|
+
agree,
|
|
743
|
+
ci,
|
|
744
|
+
n,
|
|
745
|
+
ruleId,
|
|
746
|
+
trusted: ci[0] >= floor
|
|
747
|
+
};
|
|
748
|
+
}).toSorted((a, b) => a.ruleId.localeCompare(b.ruleId));
|
|
749
|
+
};
|
|
750
|
+
const pct$1 = (x) => `${Math.round(x * 100)}%`;
|
|
751
|
+
const renderAgreement = (rows, floor) => {
|
|
752
|
+
if (!rows.length) return "No samples are labelled in both files.\n";
|
|
753
|
+
return `${rows.map((r) => `${r.ruleId}: ${r.agree}/${r.n} agree (${pct$1(r.agree / r.n)}, 95% CI ${pct$1(r.ci[0])}-${pct$1(r.ci[1])}) ${r.trusted ? "trusted" : `below ${pct$1(floor)}: label this rule by hand`}`).join("\n")}\n`;
|
|
754
|
+
};
|
|
755
|
+
const renderDisagreements = (evals, items, rules) => {
|
|
756
|
+
const byId = new Map(items.map((item) => [item.id, item]));
|
|
757
|
+
const out = [];
|
|
758
|
+
for (const e of evals) {
|
|
759
|
+
const rule = rules.find((r) => r.id === e.ruleId);
|
|
760
|
+
if (!rule) continue;
|
|
761
|
+
const wrong = e.pairs.filter((p) => p.probability >= rule.thresholds.review !== p.label);
|
|
762
|
+
if (!wrong.length) continue;
|
|
763
|
+
out.push(`${e.ruleId}: ${wrong.length} of ${e.pairs.length} disagree (${path.relative(process.cwd(), rule.file)})`);
|
|
764
|
+
for (const p of wrong) {
|
|
765
|
+
const item = byId.get(p.id);
|
|
766
|
+
out.push(` ${p.label ? "labelled violation, judge missed" : "labelled fine, judge flagged"} p=${p.probability.toFixed(2)} [${item?.labelSource ?? "?"}/${item?.split ?? "?"}] ${p.id}`, ` ${(item?.text ?? "").replaceAll("\n", " ").slice(0, 160)}`, ...item?.note ? [` note: ${item.note}`] : []);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return out.length ? `\nDisagreements (review threshold):\n${out.join("\n")}\n` : "\nNo disagreements at the review threshold.\n";
|
|
770
|
+
};
|
|
771
|
+
//#endregion
|
|
555
772
|
//#region src/extract/tailwind.ts
|
|
556
773
|
const ROOT_PX = 16;
|
|
557
774
|
const FONT_SIZES = {
|
|
@@ -843,18 +1060,6 @@ const compareScale = (classes, theme = {}) => {
|
|
|
843
1060
|
};
|
|
844
1061
|
};
|
|
845
1062
|
//#endregion
|
|
846
|
-
//#region src/lib/errors.ts
|
|
847
|
-
var InputError = class extends Error {
|
|
848
|
-
code;
|
|
849
|
-
details;
|
|
850
|
-
constructor(code, message, details = {}) {
|
|
851
|
-
super(message);
|
|
852
|
-
this.code = code;
|
|
853
|
-
this.details = details;
|
|
854
|
-
this.name = "InputError";
|
|
855
|
-
}
|
|
856
|
-
};
|
|
857
|
-
//#endregion
|
|
858
1063
|
//#region src/lib/glob.ts
|
|
859
1064
|
const DEFAULT_EXCLUDE = [
|
|
860
1065
|
"**/node_modules/**",
|
|
@@ -1056,18 +1261,18 @@ const invalid = (key, expected) => {
|
|
|
1056
1261
|
field: key
|
|
1057
1262
|
});
|
|
1058
1263
|
};
|
|
1264
|
+
const object$1 = (value, key) => {
|
|
1265
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return invalid(key, "an object");
|
|
1266
|
+
return value;
|
|
1267
|
+
};
|
|
1268
|
+
const keys = (value, allowed, prefix) => {
|
|
1269
|
+
for (const key of Object.keys(value)) if (!allowed.includes(key)) invalid(`${prefix}${key}`, `one of ${allowed.join(", ")}`);
|
|
1270
|
+
};
|
|
1271
|
+
const strings$1 = (value, key) => {
|
|
1272
|
+
if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) invalid(key, "an array of strings");
|
|
1273
|
+
};
|
|
1059
1274
|
function validateConfig(raw) {
|
|
1060
|
-
const
|
|
1061
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return invalid(key, "an object");
|
|
1062
|
-
return value;
|
|
1063
|
-
};
|
|
1064
|
-
const keys = (value, allowed, prefix) => {
|
|
1065
|
-
for (const key of Object.keys(value)) if (!allowed.includes(key)) invalid(`${prefix}${key}`, `one of ${allowed.join(", ")}`);
|
|
1066
|
-
};
|
|
1067
|
-
const strings = (value, key) => {
|
|
1068
|
-
if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) invalid(key, "an array of strings");
|
|
1069
|
-
};
|
|
1070
|
-
const value = object(raw, "config");
|
|
1275
|
+
const value = object$1(raw, "config");
|
|
1071
1276
|
keys(value, [
|
|
1072
1277
|
"$schema",
|
|
1073
1278
|
"exclude",
|
|
@@ -1078,28 +1283,28 @@ function validateConfig(raw) {
|
|
|
1078
1283
|
"rules"
|
|
1079
1284
|
], "");
|
|
1080
1285
|
if (value.$schema !== void 0 && typeof value.$schema !== "string") invalid("$schema", "a string");
|
|
1081
|
-
if (value.exclude !== void 0) strings(value.exclude, "exclude");
|
|
1082
|
-
if (value.rules !== void 0) strings(value.rules, "rules");
|
|
1286
|
+
if (value.exclude !== void 0) strings$1(value.exclude, "exclude");
|
|
1287
|
+
if (value.rules !== void 0) strings$1(value.rules, "rules");
|
|
1083
1288
|
if (value.smartQuotesAtBuild !== void 0 && typeof value.smartQuotesAtBuild !== "boolean") invalid("smartQuotesAtBuild", "a boolean");
|
|
1084
1289
|
if (value.docTypes !== void 0) {
|
|
1085
1290
|
if (!Array.isArray(value.docTypes)) invalid("docTypes", "an array");
|
|
1086
1291
|
for (const [i, entry] of value.docTypes.entries()) {
|
|
1087
|
-
const row = object(entry, `docTypes[${i}]`);
|
|
1292
|
+
const row = object$1(entry, `docTypes[${i}]`);
|
|
1088
1293
|
keys(row, ["glob", "type"], `docTypes[${i}].`);
|
|
1089
1294
|
if (typeof row.glob !== "string") invalid(`docTypes[${i}].glob`, "a string");
|
|
1090
1295
|
if (!DOC_TYPES.includes(row.type)) invalid(`docTypes[${i}].type`, DOC_TYPES.join(", "));
|
|
1091
1296
|
}
|
|
1092
1297
|
}
|
|
1093
1298
|
if (value.components !== void 0) {
|
|
1094
|
-
const c = object(value.components, "components");
|
|
1299
|
+
const c = object$1(value.components, "components");
|
|
1095
1300
|
keys(c, ["skip", "unwrap"], "components.");
|
|
1096
|
-
for (const key of ["skip", "unwrap"]) if (c[key] !== void 0) strings(c[key], `components.${key}`);
|
|
1301
|
+
for (const key of ["skip", "unwrap"]) if (c[key] !== void 0) strings$1(c[key], `components.${key}`);
|
|
1097
1302
|
}
|
|
1098
1303
|
if (value.tailwind !== void 0) {
|
|
1099
|
-
const t = object(value.tailwind, "tailwind");
|
|
1304
|
+
const t = object$1(value.tailwind, "tailwind");
|
|
1100
1305
|
keys(t, ["theme"], "tailwind.");
|
|
1101
1306
|
if (t.theme !== void 0) {
|
|
1102
|
-
const theme = object(t.theme, "tailwind.theme");
|
|
1307
|
+
const theme = object$1(t.theme, "tailwind.theme");
|
|
1103
1308
|
for (const [key, val] of Object.entries(theme)) if (typeof val !== "string") invalid(`tailwind.theme.${key}`, "a string");
|
|
1104
1309
|
}
|
|
1105
1310
|
}
|
|
@@ -1486,6 +1691,8 @@ const buildState = (unit, rules) => {
|
|
|
1486
1691
|
if (keys.has("docType")) lines.push(`DOC TYPE: ${unit.context.docType}`);
|
|
1487
1692
|
if (keys.has("element") && unit.context.element) lines.push(`ELEMENT: ${unit.context.element}${unit.context.attr ? ` (${unit.context.attr})` : ""}`);
|
|
1488
1693
|
if (keys.has("role")) lines.push(`ROLE: ${unit.context.role}`);
|
|
1694
|
+
if (keys.has("brief") && unit.context.brief) lines.push(`BRIEF: ${clip("BRIEF: ", unit.context.brief)}`);
|
|
1695
|
+
if (keys.has("earlier") && unit.context.earlier) lines.push(`EARLIER: ${clip("EARLIER: ", unit.context.earlier)}`);
|
|
1489
1696
|
if (keys.has("section") && unit.context.section) lines.push(`SECTION: ${clip("SECTION: ", unit.context.section)}`);
|
|
1490
1697
|
return {
|
|
1491
1698
|
state: lines.join("\n"),
|
|
@@ -2085,7 +2292,8 @@ const preconditions = (file, v) => {
|
|
|
2085
2292
|
"notInCode",
|
|
2086
2293
|
"docType",
|
|
2087
2294
|
"role",
|
|
2088
|
-
"smartQuotesAtBuild"
|
|
2295
|
+
"smartQuotesAtBuild",
|
|
2296
|
+
"minWords"
|
|
2089
2297
|
]);
|
|
2090
2298
|
const out = {};
|
|
2091
2299
|
if (raw.notInCode !== void 0) {
|
|
@@ -2098,6 +2306,10 @@ const preconditions = (file, v) => {
|
|
|
2098
2306
|
if (raw.smartQuotesAtBuild !== false) fail(file, "preconditions.smartQuotesAtBuild may only be false");
|
|
2099
2307
|
out.smartQuotesAtBuild = false;
|
|
2100
2308
|
}
|
|
2309
|
+
if (raw.minWords !== void 0) {
|
|
2310
|
+
if (!Number.isInteger(raw.minWords) || raw.minWords < 1) fail(file, "preconditions.minWords must be a positive integer");
|
|
2311
|
+
out.minWords = raw.minWords;
|
|
2312
|
+
}
|
|
2101
2313
|
return out;
|
|
2102
2314
|
};
|
|
2103
2315
|
const INCLUDE_BY_KIND = {
|
|
@@ -2193,6 +2405,7 @@ const missingEvidence = (unit, rule) => {
|
|
|
2193
2405
|
};
|
|
2194
2406
|
//#endregion
|
|
2195
2407
|
//#region src/map/plan.ts
|
|
2408
|
+
const WORD$1 = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
|
|
2196
2409
|
const inScope = (rule, unit) => unit.kind === "element" || matchesAny(unit.file, rule.scope.include) && !matchesAny(unit.file, rule.scope.exclude);
|
|
2197
2410
|
const passesPreconditions = (rule, unit, config) => {
|
|
2198
2411
|
const p = rule.preconditions;
|
|
@@ -2200,6 +2413,7 @@ const passesPreconditions = (rule, unit, config) => {
|
|
|
2200
2413
|
if (p.notInCode && unit.inCode) return false;
|
|
2201
2414
|
if (p.docType && !p.docType.includes(unit.context.docType)) return false;
|
|
2202
2415
|
if (p.role && !p.role.includes(unit.context.role)) return false;
|
|
2416
|
+
if (p.minWords && (unit.text.match(WORD$1)?.length ?? 0) < p.minWords) return false;
|
|
2203
2417
|
if (p.smartQuotesAtBuild === false && config.smartQuotesAtBuild && (unit.kind === "paragraph" || unit.kind === "heading") && /\.(md|mdx)$/.test(unit.file)) return false;
|
|
2204
2418
|
return true;
|
|
2205
2419
|
};
|
|
@@ -3183,6 +3397,110 @@ const MOTION_SOURCE_RULES = [codeRule({
|
|
|
3183
3397
|
unit: ["source"]
|
|
3184
3398
|
}, "src/rules/code/motion-source.ts")];
|
|
3185
3399
|
//#endregion
|
|
3400
|
+
//#region src/rules/code/prose.ts
|
|
3401
|
+
const PROSE_ROLES = [
|
|
3402
|
+
"body",
|
|
3403
|
+
"list-item",
|
|
3404
|
+
"cell",
|
|
3405
|
+
"caption",
|
|
3406
|
+
"literal-copy",
|
|
3407
|
+
"unknown"
|
|
3408
|
+
];
|
|
3409
|
+
const echoesEarlier = (unit) => unit.context.earlier ? hit(`shares content words with ${unit.context.earlier.split("\n").length} earlier sentence(s)`) : none;
|
|
3410
|
+
const hasBrief = (unit) => unit.context.brief ? hit("checked against the brief") : none;
|
|
3411
|
+
const PROSE_RULES = [codeRule({
|
|
3412
|
+
categoryId: "machine-prose",
|
|
3413
|
+
check: echoesEarlier,
|
|
3414
|
+
hint: "Cut the restatement or replace it with the evidence it was standing in for.",
|
|
3415
|
+
id: "copywriting-idea-repetition",
|
|
3416
|
+
preconditions: {
|
|
3417
|
+
minWords: 5,
|
|
3418
|
+
notInCode: true,
|
|
3419
|
+
role: PROSE_ROLES
|
|
3420
|
+
},
|
|
3421
|
+
question: {
|
|
3422
|
+
context: ["docType", "earlier"],
|
|
3423
|
+
criteria: {
|
|
3424
|
+
false: {
|
|
3425
|
+
examples: [
|
|
3426
|
+
"EARLIER: Northwind keeps both versions side by side. TEXT: Open the conflict and pick the lines you want from each version.",
|
|
3427
|
+
"EARLIER: Relay unreachable. Check your connection and try again. TEXT: Disk full. Free up space on this laptop and Northwind will resume.",
|
|
3428
|
+
"EARLIER: Speed matters to users. TEXT: A 400ms dropdown loses a click one time in five in our logs."
|
|
3429
|
+
],
|
|
3430
|
+
what: "TEXT adds a fact, example, step or consequence, or is a parallel entry for a different case."
|
|
3431
|
+
},
|
|
3432
|
+
true: {
|
|
3433
|
+
examples: ["EARLIER: When two people edit the same file, Northwind keeps both versions side by side. TEXT: Conflicting edits are never overwritten: you get both versions, side by side.", "EARLIER: It costs $8 per seat per month, with a 14-day free trial. TEXT: Just $8 per seat per month, and your first 14 days are free."],
|
|
3434
|
+
what: "TEXT says again what an EARLIER sentence already said, with nothing new."
|
|
3435
|
+
}
|
|
3436
|
+
},
|
|
3437
|
+
instructions: "EARLIER lists sentences said before TEXT in the same document that share words with it. Does TEXT restate one of them without adding a new fact, example, step or consequence? Return false for parallel entries that share a structure but cover different cases (one error message, FAQ answer or step each), for a deliberate one-sentence summary closing a long section, and for a fact repeated where the reader acts on it (the price on the plan's button)."
|
|
3438
|
+
},
|
|
3439
|
+
source: {
|
|
3440
|
+
line: 1,
|
|
3441
|
+
path: "https://typesafe-parallel-judgment-lab.every-4573.chatgpt.site/downloads/typesafe-lab-source.zip (scripts/run-ai-checker.mjs)",
|
|
3442
|
+
repo: "every.to",
|
|
3443
|
+
ruleId: "idea_repetition"
|
|
3444
|
+
},
|
|
3445
|
+
thresholds: {
|
|
3446
|
+
act: .75,
|
|
3447
|
+
review: .4
|
|
3448
|
+
},
|
|
3449
|
+
title: "Repeats the same idea without adding new evidence",
|
|
3450
|
+
unit: ["paragraph", "jsx-text"]
|
|
3451
|
+
}, "src/rules/code/prose.ts"), codeRule({
|
|
3452
|
+
categoryId: "evidence-over-claims",
|
|
3453
|
+
check: hasBrief,
|
|
3454
|
+
hint: "Cut the setting, mode, policy or comparison, or add it to the brief if the product really does it.",
|
|
3455
|
+
id: "copywriting-invented-capability",
|
|
3456
|
+
preconditions: { notInCode: true },
|
|
3457
|
+
question: {
|
|
3458
|
+
context: [
|
|
3459
|
+
"docType",
|
|
3460
|
+
"element",
|
|
3461
|
+
"brief"
|
|
3462
|
+
],
|
|
3463
|
+
criteria: {
|
|
3464
|
+
false: {
|
|
3465
|
+
examples: [
|
|
3466
|
+
"BRIEF: keeps both versions when edits conflict. TEXT: Keep both versions",
|
|
3467
|
+
"BRIEF: 14-day free trial, no card required. TEXT: Start your free trial",
|
|
3468
|
+
"BRIEF: a desktop sync app. TEXT: Email address",
|
|
3469
|
+
"BRIEF: 14-day free trial, no card required; $8 per seat per month. TEXT: 14 days. No card. $8 per seat after that.",
|
|
3470
|
+
"BRIEF: keeps both versions when edits conflict. TEXT: Nobody’s work gets thrown away, even when two people edit the same file offline."
|
|
3471
|
+
],
|
|
3472
|
+
what: "Every claim in TEXT rewords, summarises or directly follows from the brief, or TEXT is ordinary interface chrome or an instruction to the reader."
|
|
3473
|
+
},
|
|
3474
|
+
true: {
|
|
3475
|
+
examples: [
|
|
3476
|
+
"BRIEF: local network first, falls back to an encrypted relay. TEXT: Relay only",
|
|
3477
|
+
"BRIEF: 14-day free trial, no card required. TEXT: Cancel anytime during the trial",
|
|
3478
|
+
"BRIEF: keeps both versions side by side. TEXT: Most sync tools pick a winner and silently overwrite the other."
|
|
3479
|
+
],
|
|
3480
|
+
what: "TEXT offers or asserts a capability, setting, mode, role, policy, integration or comparison the brief never states."
|
|
3481
|
+
}
|
|
3482
|
+
},
|
|
3483
|
+
instructions: "BRIEF is everything known about the product. Does TEXT offer or assert a capability, setting, mode, role, policy, integration, page, or claim about other products that BRIEF neither states nor directly implies? Return false for rewording, shortening or drawing a direct consequence from BRIEF, for ordinary interface chrome every app has (Save, Cancel, Email, Password, Settings, Sign in), and for instructions to the reader."
|
|
3484
|
+
},
|
|
3485
|
+
source: {
|
|
3486
|
+
line: 10,
|
|
3487
|
+
path: "skills/ui-design/rules/slop-invented-behaviour.md",
|
|
3488
|
+
repo: "mblode/agent-skills",
|
|
3489
|
+
ruleId: "slop-invented-behaviour"
|
|
3490
|
+
},
|
|
3491
|
+
thresholds: {
|
|
3492
|
+
act: .75,
|
|
3493
|
+
review: .4
|
|
3494
|
+
},
|
|
3495
|
+
title: "States a capability the brief never gave",
|
|
3496
|
+
unit: [
|
|
3497
|
+
"paragraph",
|
|
3498
|
+
"heading",
|
|
3499
|
+
"jsx-text",
|
|
3500
|
+
"attr-string"
|
|
3501
|
+
]
|
|
3502
|
+
}, "src/rules/code/prose.ts")];
|
|
3503
|
+
//#endregion
|
|
3186
3504
|
//#region src/rules/code/skill-classes.ts
|
|
3187
3505
|
const make = (id, title, categoryId, source, check, hint) => codeRule({
|
|
3188
3506
|
categoryId,
|
|
@@ -3227,6 +3545,67 @@ const SKILL_CLASS_RULES = [
|
|
|
3227
3545
|
}, "Prefer transform/opacity for frequent motion; measure deliberate container resizing before changing it.")
|
|
3228
3546
|
];
|
|
3229
3547
|
//#endregion
|
|
3548
|
+
//#region src/rules/code/slop-source.ts
|
|
3549
|
+
const EYEBROW = /<(?:p|span|div)\b[^>]*className="[^"]*\b(?:uppercase|text-xs|text-\[(?:9|10|11|12)px\])[^"]*"[^>]*>(?:\s*<span[^>]*\/>)?[^<{]{2,80}<\/(?:p|span|div)>\s*<h[1-3]\b/gu;
|
|
3550
|
+
const FACT = /\b\d+[- ]days?\b|\bno (?:credit )?card\b|\bcancel any ?time\b|\$\d+(?:\.\d\d)?(?= ?(?:\/|per|a) ?(?:seat|user|member|month))/giu;
|
|
3551
|
+
const MAX_EYEBROWS = 2;
|
|
3552
|
+
const MAX_REPEATS = 3;
|
|
3553
|
+
const eyebrows = (unit) => {
|
|
3554
|
+
const found = [...unit.text.matchAll(EYEBROW)];
|
|
3555
|
+
if (found.length <= MAX_EYEBROWS) return none;
|
|
3556
|
+
return {
|
|
3557
|
+
...hit(`${found.length} headings carry an eyebrow label`),
|
|
3558
|
+
offset: found[MAX_EYEBROWS]?.index
|
|
3559
|
+
};
|
|
3560
|
+
};
|
|
3561
|
+
const factPadding = (unit) => {
|
|
3562
|
+
const counts = /* @__PURE__ */ new Map();
|
|
3563
|
+
for (const m of unit.text.matchAll(FACT)) {
|
|
3564
|
+
const key = m[0].toLowerCase().replaceAll(/[- ]/gu, " ");
|
|
3565
|
+
const seen = counts.get(key) ?? {
|
|
3566
|
+
first: m.index,
|
|
3567
|
+
n: 0
|
|
3568
|
+
};
|
|
3569
|
+
counts.set(key, {
|
|
3570
|
+
...seen,
|
|
3571
|
+
n: seen.n + 1
|
|
3572
|
+
});
|
|
3573
|
+
}
|
|
3574
|
+
const [top] = [...counts].toSorted((a, b) => b[1].n - a[1].n);
|
|
3575
|
+
if (!top || top[1].n <= MAX_REPEATS) return none;
|
|
3576
|
+
return {
|
|
3577
|
+
...hit(`"${top[0]}" appears ${top[1].n} times`),
|
|
3578
|
+
offset: top[1].first
|
|
3579
|
+
};
|
|
3580
|
+
};
|
|
3581
|
+
const source = (ruleId) => ({
|
|
3582
|
+
line: 10,
|
|
3583
|
+
path: `skills/ui-design/rules/${ruleId}.md`,
|
|
3584
|
+
repo: "mblode/agent-skills",
|
|
3585
|
+
ruleId
|
|
3586
|
+
});
|
|
3587
|
+
const SLOP_SOURCE_RULES = [codeRule({
|
|
3588
|
+
categoryId: "visual-hierarchy",
|
|
3589
|
+
check: eyebrows,
|
|
3590
|
+
hint: "Keep an eyebrow on at most two sections, where it names something the heading cannot. Delete the rest.",
|
|
3591
|
+
id: "craft-eyebrow-overuse",
|
|
3592
|
+
scope: { include: ["**/*.tsx", "**/*.jsx"] },
|
|
3593
|
+
source: source("slop-eyebrow-overuse"),
|
|
3594
|
+
status: "review-only",
|
|
3595
|
+
title: "An eyebrow over nearly every heading",
|
|
3596
|
+
unit: ["source"]
|
|
3597
|
+
}, "src/rules/code/slop-source.ts"), codeRule({
|
|
3598
|
+
categoryId: "evidence-over-claims",
|
|
3599
|
+
check: factPadding,
|
|
3600
|
+
hint: "Say each fact once, where the reader needs it: the price on the plan, the trial on the button. Cut the sections that exist only to repeat them.",
|
|
3601
|
+
id: "copywriting-fact-padding",
|
|
3602
|
+
scope: { include: ["**/*.tsx", "**/*.jsx"] },
|
|
3603
|
+
source: source("slop-fact-padding"),
|
|
3604
|
+
status: "review-only",
|
|
3605
|
+
title: "The same few facts restated to fill the page",
|
|
3606
|
+
unit: ["source"]
|
|
3607
|
+
}, "src/rules/code/slop-source.ts")];
|
|
3608
|
+
//#endregion
|
|
3230
3609
|
//#region src/rules/code/typography.ts
|
|
3231
3610
|
const MODULE = "src/rules/code/typography.ts";
|
|
3232
3611
|
const AUDIT = "skills/typography-audit/rules";
|
|
@@ -3476,7 +3855,9 @@ const CODE_RULES = [
|
|
|
3476
3855
|
...DOCUMENT_RULES,
|
|
3477
3856
|
...IMAGE_RULES,
|
|
3478
3857
|
...SKILL_CLASS_RULES,
|
|
3479
|
-
...MOTION_SOURCE_RULES
|
|
3858
|
+
...MOTION_SOURCE_RULES,
|
|
3859
|
+
...SLOP_SOURCE_RULES,
|
|
3860
|
+
...PROSE_RULES
|
|
3480
3861
|
];
|
|
3481
3862
|
//#endregion
|
|
3482
3863
|
//#region src/rules/load.ts
|
|
@@ -3776,7 +4157,11 @@ const runEval = async (options, ctx = {}) => {
|
|
|
3776
4157
|
knownRuleIds
|
|
3777
4158
|
});
|
|
3778
4159
|
if (options.split && options.split !== "all") items = items.filter((i) => i.split === options.split);
|
|
3779
|
-
if (
|
|
4160
|
+
if (options.sources) {
|
|
4161
|
+
const sources = new Set(options.sources);
|
|
4162
|
+
items = items.filter((i) => sources.has(i.labelSource));
|
|
4163
|
+
}
|
|
4164
|
+
if (!items.length) throw new Error("No corpus items in the selected split and label sources.");
|
|
3780
4165
|
const model = options.model ?? "jev-latest";
|
|
3781
4166
|
const resultsDir = options.resultsDir ?? defaultResultsDir();
|
|
3782
4167
|
const cache = new AnswerCache(path.join(resultsDir, "cache"), !options.noCache);
|
|
@@ -3793,6 +4178,7 @@ const runEval = async (options, ctx = {}) => {
|
|
|
3793
4178
|
model,
|
|
3794
4179
|
recorder
|
|
3795
4180
|
});
|
|
4181
|
+
if (unknowns.some((u) => u.reason === "auth")) throw new ProviderError("auth", 401);
|
|
3796
4182
|
const labelSources = {};
|
|
3797
4183
|
for (const item of items) labelSources[item.labelSource] = (labelSources[item.labelSource] ?? 0) + 1;
|
|
3798
4184
|
const evals = evaluateRules(items, rules, probabilities, unknowns, skipped);
|
|
@@ -3812,41 +4198,56 @@ const runEval = async (options, ctx = {}) => {
|
|
|
3812
4198
|
exitCode,
|
|
3813
4199
|
labelSources,
|
|
3814
4200
|
referenceCoverage: corpusCoverage(items, rules),
|
|
3815
|
-
report: (options.check ? `Regression check: ${exitCode === 0 ? "PASS" : exitCode === 1 ? "FAIL (reference disagreement)" : "INCOMPLETE (missing evaluations)"}. Uses the review threshold.\n` : "") + (labelSources.ai ? `AI-labeled reference: ${labelSources.ai} items. Metrics measure agreement with AI labels, not human judgments.\n` : "") + renderEval(evals, usage, Boolean(options.dryRun), pendingRequests),
|
|
4201
|
+
report: (options.check ? `Regression check: ${exitCode === 0 ? "PASS" : exitCode === 1 ? "FAIL (reference disagreement)" : "INCOMPLETE (missing evaluations)"}. Uses the review threshold.\n` : "") + (labelSources.ai ? `AI-labeled reference: ${labelSources.ai} items. Metrics measure agreement with AI labels, not human judgments.\n` : "") + renderEval(evals, usage, Boolean(options.dryRun), pendingRequests) + (options.disagreements && !options.dryRun ? renderDisagreements(evals, items, rules) : ""),
|
|
3816
4202
|
rules: evals,
|
|
3817
4203
|
usage
|
|
3818
4204
|
};
|
|
3819
4205
|
};
|
|
3820
4206
|
//#endregion
|
|
3821
|
-
//#region src/
|
|
3822
|
-
const
|
|
3823
|
-
const
|
|
3824
|
-
const
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
4207
|
+
//#region src/reduce/suppress.ts
|
|
4208
|
+
const PATTERN = /(?:taste-lint-ignore|ui-audit-ignore)\s*:\s*([a-z0-9-]+)/giu;
|
|
4209
|
+
const suppressionsFor = (sourceLines, line) => {
|
|
4210
|
+
const ids = /* @__PURE__ */ new Set();
|
|
4211
|
+
for (const n of [line - 2, line - 1]) {
|
|
4212
|
+
const text = sourceLines[n];
|
|
4213
|
+
if (!text) continue;
|
|
4214
|
+
for (const m of text.matchAll(PATTERN)) ids.add(m[1].toLowerCase());
|
|
3829
4215
|
}
|
|
4216
|
+
return ids;
|
|
3830
4217
|
};
|
|
3831
|
-
const
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
4218
|
+
const applySuppressions = (findings, rules, sources) => {
|
|
4219
|
+
const byId = new Map(rules.map((r) => [r.id, r]));
|
|
4220
|
+
return findings.map((f) => {
|
|
4221
|
+
const lines = sources.get(f.file);
|
|
4222
|
+
if (!lines) return f;
|
|
4223
|
+
const ids = suppressionsFor(lines, f.line);
|
|
4224
|
+
const portedId = byId.get(f.ruleId)?.source.ruleId?.toLowerCase();
|
|
4225
|
+
if (ids.has(f.ruleId) || portedId && ids.has(portedId) || ids.has("all")) return {
|
|
4226
|
+
...f,
|
|
4227
|
+
suppressed: true
|
|
4228
|
+
};
|
|
4229
|
+
return f;
|
|
4230
|
+
});
|
|
3840
4231
|
};
|
|
3841
4232
|
//#endregion
|
|
3842
4233
|
//#region src/scan/samples.ts
|
|
3843
|
-
const
|
|
4234
|
+
const STRATA = [
|
|
4235
|
+
"ignored",
|
|
4236
|
+
"disagree",
|
|
4237
|
+
"review",
|
|
4238
|
+
"high",
|
|
4239
|
+
"negative"
|
|
4240
|
+
];
|
|
4241
|
+
const makeSamples = (units, rules, answers, negatives, perStratum = 3, sourceRepo = "scan", sources = /* @__PURE__ */ new Map()) => {
|
|
3844
4242
|
const buckets = /* @__PURE__ */ new Map();
|
|
3845
4243
|
const seen = /* @__PURE__ */ new Set();
|
|
3846
4244
|
const semanticRules = rules.filter((rule) => rule.question);
|
|
3847
4245
|
for (const unit of units) for (const rule of semanticRules) {
|
|
3848
|
-
const
|
|
4246
|
+
const answered = answers.get(unit.id)?.[rule.id];
|
|
4247
|
+
const probability = answered ?? (negatives.get(unit.id)?.has(rule.id) ? 0 : void 0);
|
|
3849
4248
|
if (probability === void 0) continue;
|
|
4249
|
+
const ignores = suppressionsFor(sources.get(unit.file) ?? [], unit.line);
|
|
4250
|
+
const stratum = ignores.has(rule.id) || ignores.has("all") || ignores.has(rule.source.ruleId?.toLowerCase() ?? rule.id) ? "ignored" : answered !== void 0 && rule.mechanical && answered < rule.thresholds.review ? "disagree" : probability < rule.thresholds.review ? "negative" : probability < rule.thresholds.act ? "review" : "high";
|
|
3850
4251
|
const { docType, role, headingAbove } = unit.context;
|
|
3851
4252
|
const context = {
|
|
3852
4253
|
docType,
|
|
@@ -3893,28 +4294,29 @@ const makeSamples = (units, rules, answers, negatives, perStratum = 3, sourceRep
|
|
|
3893
4294
|
rubric,
|
|
3894
4295
|
ruleId: rule.id
|
|
3895
4296
|
};
|
|
3896
|
-
const bucket = `${rule.id}:${docType}
|
|
4297
|
+
const bucket = `${STRATA.indexOf(stratum)}:${rule.id}:${docType}`;
|
|
3897
4298
|
const list = buckets.get(bucket) ?? [];
|
|
3898
4299
|
list.push(sample);
|
|
3899
4300
|
buckets.set(bucket, list);
|
|
3900
4301
|
}
|
|
3901
|
-
return [...buckets.
|
|
4302
|
+
return [...buckets.entries()].toSorted(([a], [b]) => a.localeCompare(b)).flatMap(([, list]) => list.toSorted((a, b) => a.id.localeCompare(b.id)).slice(0, perStratum));
|
|
3902
4303
|
};
|
|
3903
4304
|
const labelsToCorpus = (file, destination, rulesDir) => {
|
|
3904
4305
|
const raw = readJson(file);
|
|
3905
|
-
if (!object$
|
|
4306
|
+
if (!object$2(raw) || raw.version !== 1 && raw.version !== 2 || !Array.isArray(raw.samples)) throw new InputError("INVALID_LABELS", "Expected a version 1 or 2 blind sample file.");
|
|
3906
4307
|
if (raw.completed !== void 0 && raw.completed !== raw.samples.length) throw new InputError("INVALID_LABELS", "Labeling is incomplete. Review every sample before importing.");
|
|
3907
4308
|
const annotation = raw.annotation;
|
|
3908
|
-
const ai = object$
|
|
4309
|
+
const ai = object$2(annotation) && annotation.source === "ai";
|
|
3909
4310
|
if (annotation !== void 0 && (!ai || typeof annotation.model !== "string" || !annotation.model.trim() || typeof annotation.promptHash !== "string" || !/^[a-f0-9]{64}$/.test(annotation.promptHash))) throw new InputError("INVALID_LABELS", "AI annotation requires a model and prompt hash.");
|
|
3910
4311
|
const ids = /* @__PURE__ */ new Set();
|
|
3911
4312
|
const items = [];
|
|
3912
4313
|
for (const row of raw.samples) {
|
|
3913
|
-
if (!object$
|
|
4314
|
+
if (!object$2(row) || typeof row.id !== "string" || ids.has(row.id) || typeof row.ruleId !== "string" || raw.version === 2 && (!object$2(row.rubric) || row.rubric.type !== "noul" || typeof row.criterion !== "string" || row.rubric.instructions !== row.criterion) || !object$2(row.item) || row.item.id !== row.id || row.label !== null && typeof row.label !== "boolean") throw new InputError("INVALID_LABELS", "Samples require unique IDs and boolean or null labels.");
|
|
3914
4315
|
ids.add(row.id);
|
|
3915
4316
|
if (row.label === null) continue;
|
|
3916
4317
|
items.push({
|
|
3917
4318
|
...row.item,
|
|
4319
|
+
...typeof row.note === "string" && row.note.trim() ? { note: row.note.trim() } : {},
|
|
3918
4320
|
labelSource: ai ? "ai" : "hand",
|
|
3919
4321
|
...ai ? {
|
|
3920
4322
|
labelModel: annotation.model,
|
|
@@ -3939,15 +4341,27 @@ const labelsToCorpus = (file, destination, rulesDir) => {
|
|
|
3939
4341
|
};
|
|
3940
4342
|
//#endregion
|
|
3941
4343
|
//#region src/commands/eval.ts
|
|
4344
|
+
const LABEL_SOURCES = [
|
|
4345
|
+
"hand",
|
|
4346
|
+
"ai",
|
|
4347
|
+
"manifest",
|
|
4348
|
+
"manifest-weak",
|
|
4349
|
+
"sweep"
|
|
4350
|
+
];
|
|
4351
|
+
const parseSources = (value) => value?.split(",").map((s) => s.trim()).filter(Boolean).map((s) => {
|
|
4352
|
+
if (!LABEL_SOURCES.includes(s)) throw new Error(`--source must be one or more of ${LABEL_SOURCES.join(", ")}`);
|
|
4353
|
+
return s;
|
|
4354
|
+
});
|
|
3942
4355
|
function registerEvalCommand(program) {
|
|
3943
4356
|
const command = program.command("eval").enablePositionalOptions().description("Run every rule over its labelled corpus and report precision, recall and calibration").option("--corpus <path>", "Corpus directory (default: packaged data/corpus)").option("--rules <path>", "Rules directory").addOption(new Option("--output <format>", "Output format").choices(["text", "json"]).default("text")).option("--only <ids>", "Comma-separated rule ids").addOption(new Option("--split <name>", "Evaluation split").choices([
|
|
3944
4357
|
"dev",
|
|
3945
4358
|
"holdout",
|
|
3946
4359
|
"all"
|
|
3947
|
-
]).default("all")).option("--include-weak", "Include manifest-weak labels").option("--check", "Fail on reference disagreement; incomplete evaluation exits 2").option("--dry-run", "Plan without calling Jev").option("--no-cache", "Ignore cached answers").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
4360
|
+
]).default("all")).option("--include-weak", "Include manifest-weak labels").option("--source <sources>", "Only these label sources, e.g. hand to measure agreement with you").option("--disagreements", "List items where the judge and the label disagree, with text and notes").option("--check", "Fail on reference disagreement; incomplete evaluation exits 2").option("--dry-run", "Plan without calling Jev").option("--no-cache", "Ignore cached answers").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
3948
4361
|
const result = await runEval({
|
|
3949
4362
|
check: options.check,
|
|
3950
4363
|
corpusDir: options.corpus,
|
|
4364
|
+
disagreements: options.disagreements,
|
|
3951
4365
|
dryRun: options.dryRun,
|
|
3952
4366
|
includeWeak: options.includeWeak,
|
|
3953
4367
|
model: options.model,
|
|
@@ -3955,6 +4369,7 @@ function registerEvalCommand(program) {
|
|
|
3955
4369
|
only: options.only?.split(",").map((s) => s.trim()).filter(Boolean),
|
|
3956
4370
|
resultsDir: options.resultsDir,
|
|
3957
4371
|
rulesDir: options.rules,
|
|
4372
|
+
sources: parseSources(options.source),
|
|
3958
4373
|
split: options.split
|
|
3959
4374
|
});
|
|
3960
4375
|
process.stdout.write(options.output === "json" ? `${JSON.stringify(result, null, 2)}\n` : result.report);
|
|
@@ -3973,8 +4388,77 @@ function registerEvalCommand(program) {
|
|
|
3973
4388
|
command.command("labels <samples>").description("Convert a labelled sample file from lint --samples into a corpus JSONL").requiredOption("--out <file>", "New corpus JSONL file").option("--rules <path>", "Rules directory used to validate labels").action((file, options) => {
|
|
3974
4389
|
process.stdout.write(`Exported ${labelsToCorpus(file, options.out, options.rules)} labels.\n`);
|
|
3975
4390
|
});
|
|
4391
|
+
command.command("label <samples>").description("Label a lint --samples file yourself, one sample at a time; saves after every answer").option("--minutes <n>", "Stop after this many minutes", "120").action(async (file, options) => {
|
|
4392
|
+
const minutes = Number(options.minutes);
|
|
4393
|
+
if (!(Number.isFinite(minutes) && minutes > 0)) throw new Error("--minutes must be a positive number");
|
|
4394
|
+
const result = await labelSession(file, {
|
|
4395
|
+
input: process.stdin,
|
|
4396
|
+
minutes,
|
|
4397
|
+
output: process.stdout
|
|
4398
|
+
});
|
|
4399
|
+
process.stdout.write(`\n${result.labelled} labelled, ${result.unsure} unsure, ${result.remaining} left${result.stopped === "timebox" ? " (timebox reached)" : ""}.\nNext: taste-lint eval gaps ${file}, then taste-lint eval labels ${file} --out data/corpus/<name>.jsonl\n`);
|
|
4400
|
+
});
|
|
4401
|
+
command.command("gaps <samples...>").description("Group hand labels by rule: unsure answers and notes show where the rubric needs sharpening").action((files) => {
|
|
4402
|
+
process.stdout.write(renderGaps(labelGaps(files)));
|
|
4403
|
+
});
|
|
4404
|
+
command.command("agreement <hand> <other>").description("Compare another labeller's sample file (usually AI) with your hand labels, per rule").option("--floor <p>", "Agreement lower bound needed to trust the other labeller", "0.8").action((hand, other, options) => {
|
|
4405
|
+
const floor = Number(options.floor);
|
|
4406
|
+
if (!(Number.isFinite(floor) && floor > 0 && floor < 1)) throw new Error("--floor must be a number between 0 and 1 (exclusive)");
|
|
4407
|
+
const rows = labelAgreement(hand, other, floor);
|
|
4408
|
+
process.stdout.write(renderAgreement(rows, floor));
|
|
4409
|
+
process.exitCode = rows.every((r) => r.trusted) ? 0 : 1;
|
|
4410
|
+
});
|
|
3976
4411
|
}
|
|
3977
4412
|
//#endregion
|
|
4413
|
+
//#region src/extract/earlier.ts
|
|
4414
|
+
const NOT_PROSE = /* @__PURE__ */ new Set([
|
|
4415
|
+
"heading",
|
|
4416
|
+
"button",
|
|
4417
|
+
"link",
|
|
4418
|
+
"label",
|
|
4419
|
+
"placeholder",
|
|
4420
|
+
"aria"
|
|
4421
|
+
]);
|
|
4422
|
+
const STOP = new Set("about above after again also always another because been before being below between both came come could does doing done down each even every from have having here into just like made make many more most much must never only other over same should since some such than that their them then there these they this those through very want were what when where which while will with would your yours you're we're it's that's there's isn't don't can't won't".split(" "));
|
|
4423
|
+
const SENTENCE = /[^.!?\n]+[.!?]*/gu;
|
|
4424
|
+
const WORD = /[\p{L}\p{N}$][\p{L}\p{N}$'’.-]*[\p{L}\p{N}]|[\p{L}\p{N}]/gu;
|
|
4425
|
+
const MIN_SHARED = 3;
|
|
4426
|
+
const MAX_EVIDENCE = 8;
|
|
4427
|
+
const tokens = (sentence) => {
|
|
4428
|
+
const out = /* @__PURE__ */ new Set();
|
|
4429
|
+
for (const [raw] of sentence.toLowerCase().matchAll(WORD)) {
|
|
4430
|
+
const word = raw.replaceAll("’", "'");
|
|
4431
|
+
if (/\d/u.test(word)) out.add(word);
|
|
4432
|
+
else if (word.length >= 4 && !STOP.has(word)) out.add(word.length > 4 ? word.replace(/(?:'s|s)$/u, "") : word);
|
|
4433
|
+
}
|
|
4434
|
+
return out;
|
|
4435
|
+
};
|
|
4436
|
+
const isProse = (unit) => (unit.kind === "paragraph" || unit.kind === "jsx-text") && !unit.inCode && !NOT_PROSE.has(unit.context.role);
|
|
4437
|
+
const attachEarlier = (units) => {
|
|
4438
|
+
const prose = units.filter(isProse).toSorted((a, b) => a.sourceStart - b.sourceStart);
|
|
4439
|
+
const split = prose.map((unit) => [...unit.text.matchAll(SENTENCE)].map(([s]) => s.trim()).filter(Boolean).map((text) => ({
|
|
4440
|
+
text,
|
|
4441
|
+
words: tokens(text)
|
|
4442
|
+
})));
|
|
4443
|
+
const spread = /* @__PURE__ */ new Map();
|
|
4444
|
+
for (const sentences of split) for (const word of new Set(sentences.flatMap((s) => [...s.words]))) spread.set(word, (spread.get(word) ?? 0) + 1);
|
|
4445
|
+
const subject = (word) => prose.length >= 6 && (spread.get(word) ?? 0) * 3 > prose.length;
|
|
4446
|
+
const seen = [];
|
|
4447
|
+
const index = /* @__PURE__ */ new Map();
|
|
4448
|
+
for (const [u, unit] of prose.entries()) {
|
|
4449
|
+
const matched = /* @__PURE__ */ new Map();
|
|
4450
|
+
for (const sentence of split[u] ?? []) {
|
|
4451
|
+
const words = [...sentence.words].filter((w) => !subject(w));
|
|
4452
|
+
const shared = /* @__PURE__ */ new Map();
|
|
4453
|
+
for (const word of words) for (const id of index.get(word) ?? []) shared.set(id, (shared.get(id) ?? 0) + 1);
|
|
4454
|
+
for (const [id, n] of shared) if (n >= MIN_SHARED || n >= 2 && n * 2 >= words.length) matched.set(id, Math.max(matched.get(id) ?? 0, n));
|
|
4455
|
+
const id = seen.push(sentence.text) - 1;
|
|
4456
|
+
for (const word of words) index.set(word, [...index.get(word) ?? [], id]);
|
|
4457
|
+
}
|
|
4458
|
+
if (matched.size > 0) unit.context.earlier = [...matched].toSorted((a, b) => b[1] - a[1] || a[0] - b[0]).slice(0, MAX_EVIDENCE).toSorted((a, b) => a[0] - b[0]).map(([id]) => seen[id]).join("\n");
|
|
4459
|
+
}
|
|
4460
|
+
};
|
|
4461
|
+
//#endregion
|
|
3978
4462
|
//#region src/lib/frontmatter.ts
|
|
3979
4463
|
const stripBom = (text) => text.codePointAt(0) === 65279 ? ` ${text.slice(1)}` : text;
|
|
3980
4464
|
const scalar = (value) => {
|
|
@@ -4763,6 +5247,7 @@ const extractSource = (config, file, source, repository) => {
|
|
|
4763
5247
|
config,
|
|
4764
5248
|
docType
|
|
4765
5249
|
}) : [];
|
|
5250
|
+
attachEarlier(units);
|
|
4766
5251
|
const whole = makeUnit(file, new LineIndex(source), {
|
|
4767
5252
|
context: {
|
|
4768
5253
|
docType,
|
|
@@ -5345,32 +5830,6 @@ const buildScorecard = (units, rules, findings, unknowns, silent) => {
|
|
|
5345
5830
|
};
|
|
5346
5831
|
};
|
|
5347
5832
|
//#endregion
|
|
5348
|
-
//#region src/reduce/suppress.ts
|
|
5349
|
-
const PATTERN = /(?:taste-lint-ignore|ui-audit-ignore)\s*:\s*([a-z0-9-]+)/giu;
|
|
5350
|
-
const suppressionsFor = (sourceLines, line) => {
|
|
5351
|
-
const ids = /* @__PURE__ */ new Set();
|
|
5352
|
-
for (const n of [line - 2, line - 1]) {
|
|
5353
|
-
const text = sourceLines[n];
|
|
5354
|
-
if (!text) continue;
|
|
5355
|
-
for (const m of text.matchAll(PATTERN)) ids.add(m[1].toLowerCase());
|
|
5356
|
-
}
|
|
5357
|
-
return ids;
|
|
5358
|
-
};
|
|
5359
|
-
const applySuppressions = (findings, rules, sources) => {
|
|
5360
|
-
const byId = new Map(rules.map((r) => [r.id, r]));
|
|
5361
|
-
return findings.map((f) => {
|
|
5362
|
-
const lines = sources.get(f.file);
|
|
5363
|
-
if (!lines) return f;
|
|
5364
|
-
const ids = suppressionsFor(lines, f.line);
|
|
5365
|
-
const portedId = byId.get(f.ruleId)?.source.ruleId?.toLowerCase();
|
|
5366
|
-
if (ids.has(f.ruleId) || portedId && ids.has(portedId) || ids.has("all")) return {
|
|
5367
|
-
...f,
|
|
5368
|
-
suppressed: true
|
|
5369
|
-
};
|
|
5370
|
-
return f;
|
|
5371
|
-
});
|
|
5372
|
-
};
|
|
5373
|
-
//#endregion
|
|
5374
5833
|
//#region src/scan/git.ts
|
|
5375
5834
|
const git = (root, args) => {
|
|
5376
5835
|
try {
|
|
@@ -5547,6 +6006,10 @@ const runLint = async (options = {}, ctx = {}) => {
|
|
|
5547
6006
|
sources.set(file, source.split("\n"));
|
|
5548
6007
|
}
|
|
5549
6008
|
if (options.extraUnits) units.push(...options.extraUnits);
|
|
6009
|
+
if (options.brief) {
|
|
6010
|
+
const brief = fs.readFileSync(path.resolve(options.brief), "utf-8").trim();
|
|
6011
|
+
for (const unit of units) unit.context.brief = brief;
|
|
6012
|
+
}
|
|
5550
6013
|
const scope = {
|
|
5551
6014
|
byDocType: {},
|
|
5552
6015
|
diagnostics: scan.messages,
|
|
@@ -5609,7 +6072,7 @@ const runLint = async (options = {}, ctx = {}) => {
|
|
|
5609
6072
|
onProgress: ctx.onProgress,
|
|
5610
6073
|
recorder
|
|
5611
6074
|
});
|
|
5612
|
-
ctx.onEvidence?.(units, judgement.answers, judgement.negatives, rules);
|
|
6075
|
+
ctx.onEvidence?.(units, judgement.answers, judgement.negatives, rules, sources);
|
|
5613
6076
|
const status = options.dryRun ? "dry-run" : judgement.usage.errors > 0 || parseFailures.size > 0 || units.length === 0 ? "incomplete" : "complete";
|
|
5614
6077
|
const jf = jevFindings(judgement.jobs, judgement.answers);
|
|
5615
6078
|
let findings = [...judgement.mechanical, ...jf.findings];
|
|
@@ -5863,7 +6326,7 @@ const renderTty = (result, options = {}) => {
|
|
|
5863
6326
|
//#region src/commands/lint.ts
|
|
5864
6327
|
const quote = (value) => `'${value.replaceAll("'", "'\"'\"'")}'`;
|
|
5865
6328
|
function registerLintCommand(program) {
|
|
5866
|
-
program.command("lint").description("Lint files (or a rendered page) against the taste rules").argument("[paths...]", "Files or directories to lint, relative to --root").option("--root <path>", "Project root for globs and config", process.cwd()).option("--only <ids>", "Comma-separated rule ids").option("--exclude <globs>", "Comma-separated globs to skip (added to taste-lint.config.json exclude)").option("--dry-run", "Plan and estimate cost without calling Jev").option("--print-requests", "With --dry-run and --output tty, print request JSONL to stderr").addOption(new Option("--profile <name>", "Scope: product (tsx, jsx, css), writing (md, mdx), instructions (AGENTS.md, skills, plans) or all").choices(PROFILE_NAMES)).option("--since <ref>", "Report only findings on lines changed since this Git revision").option("--samples <file>", "Write blind labelling samples for the Jev rules that ran, without their predicted scores").option("--no-cache", "Ignore cached answers (new answers are still recorded)").addOption(new Option("--fail-on <severity>", "Lowest severity that fails the run").choices(SEVERITIES).default("minor")).option("--fix", "Apply deterministic fixes for act-band findings").addOption(new Option("--output <format>", "Output format").choices([
|
|
6329
|
+
program.command("lint").description("Lint files (or a rendered page) against the taste rules").argument("[paths...]", "Files or directories to lint, relative to --root").option("--root <path>", "Project root for globs and config", process.cwd()).option("--only <ids>", "Comma-separated rule ids").option("--exclude <globs>", "Comma-separated globs to skip (added to taste-lint.config.json exclude)").option("--dry-run", "Plan and estimate cost without calling Jev").option("--print-requests", "With --dry-run and --output tty, print request JSONL to stderr").addOption(new Option("--profile <name>", "Scope: product (tsx, jsx, css), writing (md, mdx), instructions (AGENTS.md, skills, plans) or all").choices(PROFILE_NAMES)).option("--since <ref>", "Report only findings on lines changed since this Git revision").option("--brief <file>", "What the product is and does; copy that states anything else is flagged").option("--samples <file>", "Write blind labelling samples for the Jev rules that ran, without their predicted scores").option("--no-cache", "Ignore cached answers (new answers are still recorded)").addOption(new Option("--fail-on <severity>", "Lowest severity that fails the run").choices(SEVERITIES).default("minor")).option("--fix", "Apply deterministic fixes for act-band findings").addOption(new Option("--output <format>", "Output format").choices([
|
|
5867
6330
|
"tty",
|
|
5868
6331
|
"json",
|
|
5869
6332
|
"sarif"
|
|
@@ -5881,6 +6344,7 @@ function registerLintCommand(program) {
|
|
|
5881
6344
|
let lastProgress = 0;
|
|
5882
6345
|
let samples = [];
|
|
5883
6346
|
const result = await runLint({
|
|
6347
|
+
brief: options.brief,
|
|
5884
6348
|
dryRun: options.dryRun,
|
|
5885
6349
|
exclude: options.exclude?.split(",").map((s) => s.trim()).filter(Boolean),
|
|
5886
6350
|
extraUnits,
|
|
@@ -5896,8 +6360,8 @@ function registerLintCommand(program) {
|
|
|
5896
6360
|
since: options.since,
|
|
5897
6361
|
targets
|
|
5898
6362
|
}, {
|
|
5899
|
-
onEvidence: options.samples ? (units, answers, negatives, rules) => {
|
|
5900
|
-
samples = makeSamples(units, rules, answers, negatives, 3, path.basename(config.root));
|
|
6363
|
+
onEvidence: options.samples ? (units, answers, negatives, rules, sources) => {
|
|
6364
|
+
samples = makeSamples(units, rules, answers, negatives, 3, path.basename(config.root), sources);
|
|
5901
6365
|
} : void 0,
|
|
5902
6366
|
onProgress: (progress) => {
|
|
5903
6367
|
if (!(options.progress || process.stderr.isTTY)) return;
|
|
@@ -5929,10 +6393,11 @@ function registerLintCommand(program) {
|
|
|
5929
6393
|
if (options.url) retryArgs.push("--url", options.url, "--selector", options.selector);
|
|
5930
6394
|
if (options.profile) retryArgs.push("--profile", options.profile);
|
|
5931
6395
|
if (options.since) retryArgs.push("--since", options.since);
|
|
6396
|
+
if (options.brief) retryArgs.push("--brief", options.brief);
|
|
5932
6397
|
retryArgs.push("--", ...targets);
|
|
5933
6398
|
const rerun = result.status === "incomplete" ? retryArgs.map(quote).join(" ") : void 0;
|
|
5934
6399
|
if (options.samples) writeJson(options.samples, {
|
|
5935
|
-
instructions: "Blind review: set label true for a violation, false for acceptable, or leave null when uncertain. No predicted probabilities are included.",
|
|
6400
|
+
instructions: "Blind review: set label true for a violation, false for acceptable, or leave null when uncertain. No predicted probabilities are included. Likely failures come first; label them yourself with taste-lint eval label.",
|
|
5936
6401
|
samples,
|
|
5937
6402
|
version: 2
|
|
5938
6403
|
});
|
|
@@ -5994,6 +6459,7 @@ const promotionEvidence = (items, ruleId, pairs, threshold, floor, minItems) =>
|
|
|
5994
6459
|
};
|
|
5995
6460
|
//#endregion
|
|
5996
6461
|
//#region src/eval/tune.ts
|
|
6462
|
+
const fromSources = (items, sources) => sources ? items.filter((item) => sources.includes(item.labelSource)) : items;
|
|
5997
6463
|
const THRESHOLDS = Array.from({ length: 10 }, (_, i) => Math.round((.5 + i * .05) * 100) / 100);
|
|
5998
6464
|
const decideThreshold = (pairs, currentAct, floor, minItems, review = 0) => {
|
|
5999
6465
|
const n = pairs.length;
|
|
@@ -6047,7 +6513,7 @@ const runTune = async (options, ctx = {}) => {
|
|
|
6047
6513
|
only: options.only
|
|
6048
6514
|
}).filter((r) => r.tier !== "mechanical");
|
|
6049
6515
|
const knownRuleIds = new Set(loadRules(rulesDir, { allowDraft: true }).map((r) => r.id));
|
|
6050
|
-
const allItems = loadCorpus(resolveCorpusDir(options.corpusDir), rules, { knownRuleIds });
|
|
6516
|
+
const allItems = fromSources(loadCorpus(resolveCorpusDir(options.corpusDir), rules, { knownRuleIds }), options.sources);
|
|
6051
6517
|
const items = allItems.filter((item) => item.split === "dev");
|
|
6052
6518
|
const model = options.model ?? "jev-latest";
|
|
6053
6519
|
const resultsDir = options.resultsDir ?? defaultResultsDir();
|
|
@@ -6122,7 +6588,7 @@ const runTuneAb = async (options, ctx = {}) => {
|
|
|
6122
6588
|
};
|
|
6123
6589
|
const variant = validateRule(merged, options.variantFile, rule.id);
|
|
6124
6590
|
const knownRuleIds = new Set(loadRules(rulesDir, { allowDraft: true }).map((r) => r.id));
|
|
6125
|
-
const items = loadCorpus(resolveCorpusDir(options.corpusDir), [rule], { knownRuleIds }).filter((i) => i.split === "dev" && rule.id in i.labels);
|
|
6591
|
+
const items = fromSources(loadCorpus(resolveCorpusDir(options.corpusDir), [rule], { knownRuleIds }), options.sources).filter((i) => i.split === "dev" && rule.id in i.labels);
|
|
6126
6592
|
const model = options.model ?? "jev-latest";
|
|
6127
6593
|
const resultsDir = options.resultsDir ?? defaultResultsDir();
|
|
6128
6594
|
const cache = new AnswerCache(path.join(resultsDir, "cache"));
|
|
@@ -6168,7 +6634,7 @@ const ruleToRaw = (rule) => {
|
|
|
6168
6634
|
//#endregion
|
|
6169
6635
|
//#region src/commands/tune.ts
|
|
6170
6636
|
function registerTuneCommand(program) {
|
|
6171
|
-
program.command("tune").description("Pick act thresholds per rule from the dev split; --write updates data/rules/tuning.json").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--only <ids>", "Comma-separated rule ids").option("--floor <p>", "Required lower bound on act-band precision", "0.8").option("--min-items <n>", "Labelled dev items a rule needs before a decision", "10").option("--write", "Write tuning.json").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
6637
|
+
program.command("tune").description("Pick act thresholds per rule from the dev split; --write updates data/rules/tuning.json").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--only <ids>", "Comma-separated rule ids").option("--floor <p>", "Required lower bound on act-band precision", "0.8").option("--min-items <n>", "Labelled dev items a rule needs before a decision", "10").option("--write", "Write tuning.json").option("--source <sources>", "Only these label sources; tune on hand to fit the judge to you").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
6172
6638
|
const result = await runTune({
|
|
6173
6639
|
corpusDir: options.corpus,
|
|
6174
6640
|
floor: Number(options.floor),
|
|
@@ -6177,17 +6643,19 @@ function registerTuneCommand(program) {
|
|
|
6177
6643
|
only: options.only?.split(",").map((s) => s.trim()).filter(Boolean),
|
|
6178
6644
|
resultsDir: options.resultsDir,
|
|
6179
6645
|
rulesDir: options.rules,
|
|
6646
|
+
sources: parseSources(options.source),
|
|
6180
6647
|
write: options.write
|
|
6181
6648
|
});
|
|
6182
6649
|
process.stdout.write(result.report);
|
|
6183
6650
|
process.exitCode = result.exitCode;
|
|
6184
|
-
}).command("ab").description("Compare two question wordings for one rule over the dev split (McNemar)").requiredOption("--rule <id>", "Rule id").requiredOption("--variant <file>", "YAML file with a replacement question").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
6651
|
+
}).command("ab").description("Compare two question wordings for one rule over the dev split (McNemar)").requiredOption("--rule <id>", "Rule id").requiredOption("--variant <file>", "YAML file with a replacement question").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--source <sources>", "Only these label sources; tune on hand to fit the judge to you").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
|
|
6185
6652
|
const result = await runTuneAb({
|
|
6186
6653
|
corpusDir: options.corpus,
|
|
6187
6654
|
model: options.model,
|
|
6188
6655
|
resultsDir: options.resultsDir,
|
|
6189
6656
|
ruleId: options.rule,
|
|
6190
6657
|
rulesDir: options.rules,
|
|
6658
|
+
sources: parseSources(options.source),
|
|
6191
6659
|
variantFile: options.variant
|
|
6192
6660
|
});
|
|
6193
6661
|
process.stdout.write(result.report);
|