taste-lint 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/data/corpus/copywriting.jsonl +4 -4
  2. package/data/rule-drafts/interaction/interaction-layout-shift.yaml +3 -3
  3. package/data/rules/copywriting/copywriting-antithesis-contrast.yaml +43 -0
  4. package/data/rules/copywriting/copywriting-antithesis-frame.yaml +4 -2
  5. package/data/rules/copywriting/copywriting-artificial-balance.yaml +4 -1
  6. package/data/rules/copywriting/copywriting-boilerplate-offer.yaml +1 -1
  7. package/data/rules/copywriting/copywriting-canned-phrasing.yaml +1 -1
  8. package/data/rules/copywriting/copywriting-claim-without-evidence.yaml +10 -1
  9. package/data/rules/copywriting/copywriting-copula-avoidance.yaml +1 -1
  10. package/data/rules/copywriting/copywriting-dead-link.yaml +28 -0
  11. package/data/rules/copywriting/copywriting-em-dash-in-ui.yaml +23 -0
  12. package/data/rules/copywriting/copywriting-excessive-hedging.yaml +1 -1
  13. package/data/rules/copywriting/copywriting-external-arrow-internal-link.yaml +26 -0
  14. package/data/rules/copywriting/copywriting-fabricated-specificity.yaml +6 -3
  15. package/data/rules/copywriting/copywriting-generic-framing.yaml +12 -6
  16. package/data/rules/copywriting/copywriting-generic-language.yaml +12 -4
  17. package/data/rules/copywriting/copywriting-generic-validation.yaml +1 -1
  18. package/data/rules/copywriting/copywriting-machine-prose.yaml +4 -1
  19. package/data/rules/copywriting/copywriting-mechanism-over-outcome.yaml +4 -1
  20. package/data/rules/copywriting/copywriting-meta-commentary.yaml +1 -1
  21. package/data/rules/copywriting/copywriting-missing-reasoning-link.yaml +4 -1
  22. package/data/rules/copywriting/copywriting-over-explanation.yaml +4 -1
  23. package/data/rules/copywriting/copywriting-participle-tail.yaml +1 -1
  24. package/data/rules/copywriting/copywriting-prompt-restatement.yaml +5 -2
  25. package/data/rules/copywriting/copywriting-punctuation-tics.yaml +4 -1
  26. package/data/rules/copywriting/copywriting-reader-first.yaml +4 -1
  27. package/data/rules/copywriting/copywriting-repetitive-transitions.yaml +1 -1
  28. package/data/rules/copywriting/copywriting-semantic-links.yaml +2 -2
  29. package/data/rules/copywriting/copywriting-slogan-headings.yaml +27 -0
  30. package/data/rules/copywriting/copywriting-structural-symmetry.yaml +4 -1
  31. package/data/rules/copywriting/copywriting-tee-up.yaml +4 -2
  32. package/data/rules/copywriting/copywriting-unexplained-action.yaml +5 -2
  33. package/data/rules/copywriting/copywriting-unnatural-polish.yaml +8 -5
  34. package/data/rules/copywriting/copywriting-unverifiable-proof.yaml +1 -1
  35. package/data/rules/copywriting/copywriting-vague-error.yaml +2 -0
  36. package/data/rules/craft/craft-atmosphere-gradient.yaml +1 -1
  37. package/data/rules/craft/craft-decoration-no-role.yaml +2 -1
  38. package/data/rules/craft/craft-decorative-ordinals.yaml +26 -0
  39. package/data/rules/craft/craft-faux-product-chrome.yaml +4 -1
  40. package/data/rules/schema/rule.schema.json +7 -1
  41. package/dist/cli.js +565 -97
  42. package/dist/cli.js.map +1 -1
  43. package/dist/index.d.ts +34 -18
  44. package/dist/index.d.ts.map +1 -1
  45. package/dist/index.js +275 -17
  46. package/dist/index.js.map +1 -1
  47. package/package.json +2 -2
  48. package/data/rules/copywriting/copywriting-idea-repetition.yaml +0 -34
package/dist/cli.js CHANGED
@@ -2,9 +2,10 @@
2
2
  import { Command, CommanderError, Option } from "commander";
3
3
  import fs from "node:fs";
4
4
  import path from "node:path";
5
+ import readline from "node:readline";
6
+ import { createHash, randomUUID } from "node:crypto";
5
7
  import os from "node:os";
6
8
  import { execFileSync, spawn, spawnSync } from "node:child_process";
7
- import { createHash, randomUUID } from "node:crypto";
8
9
  import { parse } from "yaml";
9
10
  import { fromMarkdown } from "mdast-util-from-markdown";
10
11
  import { gfmFromMarkdown } from "mdast-util-gfm";
@@ -14,7 +15,7 @@ import { mdxFromMarkdown } from "mdast-util-mdx";
14
15
  import { mdxjs } from "micromark-extension-mdxjs";
15
16
  import { styleText } from "node:util";
16
17
  //#region packages/cli/package.json
17
- var version = "0.3.0";
18
+ var version = "0.4.0";
18
19
  //#endregion
19
20
  //#region src/lib/stats.ts
20
21
  const makePRNG = (seed) => {
@@ -345,7 +346,9 @@ const CONTEXT_KEYS = [
345
346
  "docType",
346
347
  "element",
347
348
  "role",
348
- "neighbours"
349
+ "neighbours",
350
+ "earlier",
351
+ "brief"
349
352
  ];
350
353
  //#endregion
351
354
  //#region src/eval/corpus.ts
@@ -416,6 +419,7 @@ const loadCorpus = (corpusDir, rules, options = {}) => {
416
419
  ].includes(labelSource)) throw new Error(`Invalid corpus line ${where}: labelSource`);
417
420
  if (labelSource === "ai" && (typeof r.labelModel !== "string" || !r.labelModel.trim() || typeof r.labelPromptHash !== "string" || !/^[a-f0-9]{64}$/.test(r.labelPromptHash))) throw new Error(`Invalid corpus line ${where}: AI labels require model and prompt hash`);
418
421
  if (!isRecord$1(r.source) || typeof r.source.repo !== "string" || typeof r.source.path !== "string" || r.source.id !== void 0 && (typeof r.source.id !== "string" || !r.source.id.trim())) throw new Error(`Invalid corpus line ${where}: source needs repo and path`);
422
+ if (r.note !== void 0 && typeof r.note !== "string") throw new TypeError(`Invalid corpus line ${where}: note must be text`);
419
423
  if (r.split !== void 0 && r.split !== "dev" && r.split !== "holdout") throw new Error(`Invalid corpus line ${where}: split must be dev or holdout`);
420
424
  if (r.classes !== void 0 && (!Array.isArray(r.classes) || r.classes.some((c) => typeof c !== "string"))) throw new Error(`Invalid corpus line ${where}: classes must be strings`);
421
425
  if (r.context !== void 0) {
@@ -442,6 +446,7 @@ const loadCorpus = (corpusDir, rules, options = {}) => {
442
446
  labelSource,
443
447
  labels,
444
448
  neighbours: r.neighbours,
449
+ note: r.note,
445
450
  source: r.source,
446
451
  split: r.split ?? splitFor(JSON.stringify([r.source.repo, r.source.path])),
447
452
  text: r.text
@@ -552,6 +557,218 @@ const pairedOutcomes = (items, rule, probabilities) => {
552
557
  return contrast;
553
558
  };
554
559
  //#endregion
560
+ //#region src/lib/errors.ts
561
+ var InputError = class extends Error {
562
+ code;
563
+ details;
564
+ constructor(code, message, details = {}) {
565
+ super(message);
566
+ this.code = code;
567
+ this.details = details;
568
+ this.name = "InputError";
569
+ }
570
+ };
571
+ //#endregion
572
+ //#region src/scan/storage.ts
573
+ const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
574
+ const object$2 = (value) => !!value && typeof value === "object" && !Array.isArray(value);
575
+ const readJson = (file) => {
576
+ try {
577
+ return JSON.parse(fs.readFileSync(file, "utf-8"));
578
+ } catch {
579
+ throw new InputError("INVALID_SCAN_INPUT", `Cannot read JSON input: ${file}`);
580
+ }
581
+ };
582
+ const writeJson = (file, value) => {
583
+ fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
584
+ const temp = `${file}.${randomUUID()}.tmp`;
585
+ try {
586
+ fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
587
+ fs.renameSync(temp, file);
588
+ } finally {
589
+ fs.rmSync(temp, { force: true });
590
+ }
591
+ };
592
+ //#endregion
593
+ //#region src/eval/label.ts
594
+ const readSamples = (file) => {
595
+ const raw = readJson(file);
596
+ if (!object$2(raw) || raw.version !== 1 && raw.version !== 2 || !Array.isArray(raw.samples)) throw new InputError("INVALID_LABELS", `Expected a version 1 or 2 sample file from lint --samples: ${file}`);
597
+ return raw;
598
+ };
599
+ const show = (sample, index, total, first) => {
600
+ const { context, neighbours, text } = sample.item;
601
+ const lines = [
602
+ "",
603
+ `[${index}/${total}] ${sample.ruleId}${context?.docType ? ` · ${context.docType}` : ""}${context?.role ? ` · ${context.role}` : ""}`,
604
+ `Q: ${sample.criterion}`
605
+ ];
606
+ const criteria = sample.rubric?.criteria;
607
+ if (first && criteria) for (const [key, side] of [["yes", criteria.true], ["no", criteria.false]]) {
608
+ if (!side) continue;
609
+ lines.push(` ${key}: ${side.what ?? ""}`);
610
+ for (const example of side.examples ?? []) lines.push(` e.g. ${example}`);
611
+ }
612
+ if (context?.headingAbove) lines.push(`Under heading: ${context.headingAbove}`);
613
+ if (neighbours?.prev) lines.push(`Before: ${neighbours.prev.text}`);
614
+ lines.push("Text:", ` ${text.split("\n").join("\n ")}`);
615
+ if (neighbours?.next) lines.push(`After: ${neighbours.next.text}`);
616
+ return `${lines.join("\n")}\n`;
617
+ };
618
+ /** Walk unlabelled samples, saving after every answer so quitting loses nothing. */
619
+ const labelSession = async (file, options) => {
620
+ const data = readSamples(file);
621
+ if (data.annotation !== void 0) throw new InputError("INVALID_LABELS", "This file carries AI annotation. Hand-label a fresh lint --samples file.");
622
+ const now = options.now ?? Date.now;
623
+ const deadline = options.minutes === void 0 ? Infinity : now() + options.minutes * 6e4;
624
+ const pending = data.samples.filter((s) => s.label === null && !s.unsure);
625
+ const rl = readline.createInterface({
626
+ input: options.input,
627
+ terminal: false
628
+ });
629
+ const lines = rl[Symbol.asyncIterator]();
630
+ const ask = async (prompt) => {
631
+ options.output.write(prompt);
632
+ const next = await lines.next();
633
+ return next.done ? void 0 : String(next.value).trim();
634
+ };
635
+ const seenRules = /* @__PURE__ */ new Set();
636
+ let labelled = 0;
637
+ let unsure = 0;
638
+ let stopped = "done";
639
+ try {
640
+ for (const [index, sample] of pending.entries()) {
641
+ if (now() >= deadline) {
642
+ stopped = "timebox";
643
+ break;
644
+ }
645
+ options.output.write(show(sample, index + 1, pending.length, !seenRules.has(sample.ruleId)));
646
+ seenRules.add(sample.ruleId);
647
+ let answer;
648
+ do
649
+ answer = (await ask("[y] violation [n] fine [u] unsure [s] skip [q] quit > "))?.toLowerCase();
650
+ while (answer !== void 0 && ![
651
+ "y",
652
+ "n",
653
+ "u",
654
+ "s",
655
+ "q"
656
+ ].includes(answer));
657
+ if (answer === void 0 || answer === "q") {
658
+ stopped = "quit";
659
+ break;
660
+ }
661
+ if (answer === "s") continue;
662
+ const note = await ask(answer === "u" ? "What would you need to decide? > " : "Note (enter to skip) > ");
663
+ if (answer === "u") {
664
+ sample.unsure = true;
665
+ unsure += 1;
666
+ } else {
667
+ sample.label = answer === "y";
668
+ labelled += 1;
669
+ }
670
+ if (note) sample.note = note;
671
+ writeJson(file, data);
672
+ if (note === void 0) {
673
+ stopped = "quit";
674
+ break;
675
+ }
676
+ }
677
+ } finally {
678
+ rl.close();
679
+ }
680
+ return {
681
+ labelled,
682
+ remaining: data.samples.filter((s) => s.label === null && !s.unsure).length,
683
+ stopped,
684
+ unsure
685
+ };
686
+ };
687
+ /** Group hand labels by rule: unsure answers and notes are rubric gaps. */
688
+ const labelGaps = (files) => {
689
+ const byRule = /* @__PURE__ */ new Map();
690
+ for (const file of files) for (const sample of readSamples(file).samples) {
691
+ if (sample.label === null && !sample.unsure) continue;
692
+ const gaps = byRule.get(sample.ruleId) ?? {
693
+ fine: 0,
694
+ notes: [],
695
+ ruleId: sample.ruleId,
696
+ unsure: 0,
697
+ violations: 0
698
+ };
699
+ if (sample.unsure) gaps.unsure += 1;
700
+ else if (sample.label) gaps.violations += 1;
701
+ else gaps.fine += 1;
702
+ if (sample.note) gaps.notes.push({
703
+ label: sample.unsure ? "unsure" : Boolean(sample.label),
704
+ note: sample.note,
705
+ text: sample.item.text
706
+ });
707
+ byRule.set(sample.ruleId, gaps);
708
+ }
709
+ return [...byRule.values()].toSorted((a, b) => b.unsure - a.unsure || b.notes.length - a.notes.length || a.ruleId.localeCompare(b.ruleId));
710
+ };
711
+ const renderGaps = (gaps) => {
712
+ if (!gaps.length) return "No hand labels yet. Run taste-lint eval label <samples.json>.\n";
713
+ const out = [];
714
+ for (const g of gaps) {
715
+ out.push(`${g.ruleId}: ${g.violations} violation, ${g.fine} fine, ${g.unsure} unsure`);
716
+ if (g.unsure) out.push(" Unsure answers mean the rubric cannot settle these; sharpen question.criteria before tuning.");
717
+ for (const n of g.notes) {
718
+ const label = n.label === "unsure" ? "unsure" : n.label ? "yes" : "no";
719
+ out.push(` [${label}] ${n.note}`, ` ${n.text.replaceAll("\n", " ").slice(0, 100)}`);
720
+ }
721
+ }
722
+ return `${out.join("\n")}\n`;
723
+ };
724
+ /** How often a second labeller (usually an AI) matches your hand labels, per rule. */
725
+ const labelAgreement = (handFile, otherFile, floor = .8) => {
726
+ const hand = new Map(readSamples(handFile).samples.filter((s) => s.label !== null).map((s) => [s.id, s]));
727
+ const byRule = /* @__PURE__ */ new Map();
728
+ for (const other of readSamples(otherFile).samples) {
729
+ const mine = hand.get(other.id);
730
+ if (!mine || other.label === null || mine.ruleId !== other.ruleId) continue;
731
+ const tally = byRule.get(other.ruleId) ?? {
732
+ agree: 0,
733
+ n: 0
734
+ };
735
+ tally.n += 1;
736
+ tally.agree += mine.label === other.label ? 1 : 0;
737
+ byRule.set(other.ruleId, tally);
738
+ }
739
+ return [...byRule.entries()].map(([ruleId, { n, agree }]) => {
740
+ const ci = wilson(agree, n);
741
+ return {
742
+ agree,
743
+ ci,
744
+ n,
745
+ ruleId,
746
+ trusted: ci[0] >= floor
747
+ };
748
+ }).toSorted((a, b) => a.ruleId.localeCompare(b.ruleId));
749
+ };
750
+ const pct$1 = (x) => `${Math.round(x * 100)}%`;
751
+ const renderAgreement = (rows, floor) => {
752
+ if (!rows.length) return "No samples are labelled in both files.\n";
753
+ return `${rows.map((r) => `${r.ruleId}: ${r.agree}/${r.n} agree (${pct$1(r.agree / r.n)}, 95% CI ${pct$1(r.ci[0])}-${pct$1(r.ci[1])}) ${r.trusted ? "trusted" : `below ${pct$1(floor)}: label this rule by hand`}`).join("\n")}\n`;
754
+ };
755
+ const renderDisagreements = (evals, items, rules) => {
756
+ const byId = new Map(items.map((item) => [item.id, item]));
757
+ const out = [];
758
+ for (const e of evals) {
759
+ const rule = rules.find((r) => r.id === e.ruleId);
760
+ if (!rule) continue;
761
+ const wrong = e.pairs.filter((p) => p.probability >= rule.thresholds.review !== p.label);
762
+ if (!wrong.length) continue;
763
+ out.push(`${e.ruleId}: ${wrong.length} of ${e.pairs.length} disagree (${path.relative(process.cwd(), rule.file)})`);
764
+ for (const p of wrong) {
765
+ const item = byId.get(p.id);
766
+ out.push(` ${p.label ? "labelled violation, judge missed" : "labelled fine, judge flagged"} p=${p.probability.toFixed(2)} [${item?.labelSource ?? "?"}/${item?.split ?? "?"}] ${p.id}`, ` ${(item?.text ?? "").replaceAll("\n", " ").slice(0, 160)}`, ...item?.note ? [` note: ${item.note}`] : []);
767
+ }
768
+ }
769
+ return out.length ? `\nDisagreements (review threshold):\n${out.join("\n")}\n` : "\nNo disagreements at the review threshold.\n";
770
+ };
771
+ //#endregion
555
772
  //#region src/extract/tailwind.ts
556
773
  const ROOT_PX = 16;
557
774
  const FONT_SIZES = {
@@ -843,18 +1060,6 @@ const compareScale = (classes, theme = {}) => {
843
1060
  };
844
1061
  };
845
1062
  //#endregion
846
- //#region src/lib/errors.ts
847
- var InputError = class extends Error {
848
- code;
849
- details;
850
- constructor(code, message, details = {}) {
851
- super(message);
852
- this.code = code;
853
- this.details = details;
854
- this.name = "InputError";
855
- }
856
- };
857
- //#endregion
858
1063
  //#region src/lib/glob.ts
859
1064
  const DEFAULT_EXCLUDE = [
860
1065
  "**/node_modules/**",
@@ -1056,18 +1261,18 @@ const invalid = (key, expected) => {
1056
1261
  field: key
1057
1262
  });
1058
1263
  };
1264
+ const object$1 = (value, key) => {
1265
+ if (!value || typeof value !== "object" || Array.isArray(value)) return invalid(key, "an object");
1266
+ return value;
1267
+ };
1268
+ const keys = (value, allowed, prefix) => {
1269
+ for (const key of Object.keys(value)) if (!allowed.includes(key)) invalid(`${prefix}${key}`, `one of ${allowed.join(", ")}`);
1270
+ };
1271
+ const strings$1 = (value, key) => {
1272
+ if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) invalid(key, "an array of strings");
1273
+ };
1059
1274
  function validateConfig(raw) {
1060
- const object = (value, key) => {
1061
- if (!value || typeof value !== "object" || Array.isArray(value)) return invalid(key, "an object");
1062
- return value;
1063
- };
1064
- const keys = (value, allowed, prefix) => {
1065
- for (const key of Object.keys(value)) if (!allowed.includes(key)) invalid(`${prefix}${key}`, `one of ${allowed.join(", ")}`);
1066
- };
1067
- const strings = (value, key) => {
1068
- if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) invalid(key, "an array of strings");
1069
- };
1070
- const value = object(raw, "config");
1275
+ const value = object$1(raw, "config");
1071
1276
  keys(value, [
1072
1277
  "$schema",
1073
1278
  "exclude",
@@ -1078,28 +1283,28 @@ function validateConfig(raw) {
1078
1283
  "rules"
1079
1284
  ], "");
1080
1285
  if (value.$schema !== void 0 && typeof value.$schema !== "string") invalid("$schema", "a string");
1081
- if (value.exclude !== void 0) strings(value.exclude, "exclude");
1082
- if (value.rules !== void 0) strings(value.rules, "rules");
1286
+ if (value.exclude !== void 0) strings$1(value.exclude, "exclude");
1287
+ if (value.rules !== void 0) strings$1(value.rules, "rules");
1083
1288
  if (value.smartQuotesAtBuild !== void 0 && typeof value.smartQuotesAtBuild !== "boolean") invalid("smartQuotesAtBuild", "a boolean");
1084
1289
  if (value.docTypes !== void 0) {
1085
1290
  if (!Array.isArray(value.docTypes)) invalid("docTypes", "an array");
1086
1291
  for (const [i, entry] of value.docTypes.entries()) {
1087
- const row = object(entry, `docTypes[${i}]`);
1292
+ const row = object$1(entry, `docTypes[${i}]`);
1088
1293
  keys(row, ["glob", "type"], `docTypes[${i}].`);
1089
1294
  if (typeof row.glob !== "string") invalid(`docTypes[${i}].glob`, "a string");
1090
1295
  if (!DOC_TYPES.includes(row.type)) invalid(`docTypes[${i}].type`, DOC_TYPES.join(", "));
1091
1296
  }
1092
1297
  }
1093
1298
  if (value.components !== void 0) {
1094
- const c = object(value.components, "components");
1299
+ const c = object$1(value.components, "components");
1095
1300
  keys(c, ["skip", "unwrap"], "components.");
1096
- for (const key of ["skip", "unwrap"]) if (c[key] !== void 0) strings(c[key], `components.${key}`);
1301
+ for (const key of ["skip", "unwrap"]) if (c[key] !== void 0) strings$1(c[key], `components.${key}`);
1097
1302
  }
1098
1303
  if (value.tailwind !== void 0) {
1099
- const t = object(value.tailwind, "tailwind");
1304
+ const t = object$1(value.tailwind, "tailwind");
1100
1305
  keys(t, ["theme"], "tailwind.");
1101
1306
  if (t.theme !== void 0) {
1102
- const theme = object(t.theme, "tailwind.theme");
1307
+ const theme = object$1(t.theme, "tailwind.theme");
1103
1308
  for (const [key, val] of Object.entries(theme)) if (typeof val !== "string") invalid(`tailwind.theme.${key}`, "a string");
1104
1309
  }
1105
1310
  }
@@ -1486,6 +1691,8 @@ const buildState = (unit, rules) => {
1486
1691
  if (keys.has("docType")) lines.push(`DOC TYPE: ${unit.context.docType}`);
1487
1692
  if (keys.has("element") && unit.context.element) lines.push(`ELEMENT: ${unit.context.element}${unit.context.attr ? ` (${unit.context.attr})` : ""}`);
1488
1693
  if (keys.has("role")) lines.push(`ROLE: ${unit.context.role}`);
1694
+ if (keys.has("brief") && unit.context.brief) lines.push(`BRIEF: ${clip("BRIEF: ", unit.context.brief)}`);
1695
+ if (keys.has("earlier") && unit.context.earlier) lines.push(`EARLIER: ${clip("EARLIER: ", unit.context.earlier)}`);
1489
1696
  if (keys.has("section") && unit.context.section) lines.push(`SECTION: ${clip("SECTION: ", unit.context.section)}`);
1490
1697
  return {
1491
1698
  state: lines.join("\n"),
@@ -2085,7 +2292,8 @@ const preconditions = (file, v) => {
2085
2292
  "notInCode",
2086
2293
  "docType",
2087
2294
  "role",
2088
- "smartQuotesAtBuild"
2295
+ "smartQuotesAtBuild",
2296
+ "minWords"
2089
2297
  ]);
2090
2298
  const out = {};
2091
2299
  if (raw.notInCode !== void 0) {
@@ -2098,6 +2306,10 @@ const preconditions = (file, v) => {
2098
2306
  if (raw.smartQuotesAtBuild !== false) fail(file, "preconditions.smartQuotesAtBuild may only be false");
2099
2307
  out.smartQuotesAtBuild = false;
2100
2308
  }
2309
+ if (raw.minWords !== void 0) {
2310
+ if (!Number.isInteger(raw.minWords) || raw.minWords < 1) fail(file, "preconditions.minWords must be a positive integer");
2311
+ out.minWords = raw.minWords;
2312
+ }
2101
2313
  return out;
2102
2314
  };
2103
2315
  const INCLUDE_BY_KIND = {
@@ -2193,6 +2405,7 @@ const missingEvidence = (unit, rule) => {
2193
2405
  };
2194
2406
  //#endregion
2195
2407
  //#region src/map/plan.ts
2408
+ const WORD$1 = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu;
2196
2409
  const inScope = (rule, unit) => unit.kind === "element" || matchesAny(unit.file, rule.scope.include) && !matchesAny(unit.file, rule.scope.exclude);
2197
2410
  const passesPreconditions = (rule, unit, config) => {
2198
2411
  const p = rule.preconditions;
@@ -2200,6 +2413,7 @@ const passesPreconditions = (rule, unit, config) => {
2200
2413
  if (p.notInCode && unit.inCode) return false;
2201
2414
  if (p.docType && !p.docType.includes(unit.context.docType)) return false;
2202
2415
  if (p.role && !p.role.includes(unit.context.role)) return false;
2416
+ if (p.minWords && (unit.text.match(WORD$1)?.length ?? 0) < p.minWords) return false;
2203
2417
  if (p.smartQuotesAtBuild === false && config.smartQuotesAtBuild && (unit.kind === "paragraph" || unit.kind === "heading") && /\.(md|mdx)$/.test(unit.file)) return false;
2204
2418
  return true;
2205
2419
  };
@@ -3183,6 +3397,110 @@ const MOTION_SOURCE_RULES = [codeRule({
3183
3397
  unit: ["source"]
3184
3398
  }, "src/rules/code/motion-source.ts")];
3185
3399
  //#endregion
3400
+ //#region src/rules/code/prose.ts
3401
+ const PROSE_ROLES = [
3402
+ "body",
3403
+ "list-item",
3404
+ "cell",
3405
+ "caption",
3406
+ "literal-copy",
3407
+ "unknown"
3408
+ ];
3409
+ const echoesEarlier = (unit) => unit.context.earlier ? hit(`shares content words with ${unit.context.earlier.split("\n").length} earlier sentence(s)`) : none;
3410
+ const hasBrief = (unit) => unit.context.brief ? hit("checked against the brief") : none;
3411
+ const PROSE_RULES = [codeRule({
3412
+ categoryId: "machine-prose",
3413
+ check: echoesEarlier,
3414
+ hint: "Cut the restatement or replace it with the evidence it was standing in for.",
3415
+ id: "copywriting-idea-repetition",
3416
+ preconditions: {
3417
+ minWords: 5,
3418
+ notInCode: true,
3419
+ role: PROSE_ROLES
3420
+ },
3421
+ question: {
3422
+ context: ["docType", "earlier"],
3423
+ criteria: {
3424
+ false: {
3425
+ examples: [
3426
+ "EARLIER: Northwind keeps both versions side by side. TEXT: Open the conflict and pick the lines you want from each version.",
3427
+ "EARLIER: Relay unreachable. Check your connection and try again. TEXT: Disk full. Free up space on this laptop and Northwind will resume.",
3428
+ "EARLIER: Speed matters to users. TEXT: A 400ms dropdown loses a click one time in five in our logs."
3429
+ ],
3430
+ what: "TEXT adds a fact, example, step or consequence, or is a parallel entry for a different case."
3431
+ },
3432
+ true: {
3433
+ examples: ["EARLIER: When two people edit the same file, Northwind keeps both versions side by side. TEXT: Conflicting edits are never overwritten: you get both versions, side by side.", "EARLIER: It costs $8 per seat per month, with a 14-day free trial. TEXT: Just $8 per seat per month, and your first 14 days are free."],
3434
+ what: "TEXT says again what an EARLIER sentence already said, with nothing new."
3435
+ }
3436
+ },
3437
+ instructions: "EARLIER lists sentences said before TEXT in the same document that share words with it. Does TEXT restate one of them without adding a new fact, example, step or consequence? Return false for parallel entries that share a structure but cover different cases (one error message, FAQ answer or step each), for a deliberate one-sentence summary closing a long section, and for a fact repeated where the reader acts on it (the price on the plan's button)."
3438
+ },
3439
+ source: {
3440
+ line: 1,
3441
+ path: "https://typesafe-parallel-judgment-lab.every-4573.chatgpt.site/downloads/typesafe-lab-source.zip (scripts/run-ai-checker.mjs)",
3442
+ repo: "every.to",
3443
+ ruleId: "idea_repetition"
3444
+ },
3445
+ thresholds: {
3446
+ act: .75,
3447
+ review: .4
3448
+ },
3449
+ title: "Repeats the same idea without adding new evidence",
3450
+ unit: ["paragraph", "jsx-text"]
3451
+ }, "src/rules/code/prose.ts"), codeRule({
3452
+ categoryId: "evidence-over-claims",
3453
+ check: hasBrief,
3454
+ hint: "Cut the setting, mode, policy or comparison, or add it to the brief if the product really does it.",
3455
+ id: "copywriting-invented-capability",
3456
+ preconditions: { notInCode: true },
3457
+ question: {
3458
+ context: [
3459
+ "docType",
3460
+ "element",
3461
+ "brief"
3462
+ ],
3463
+ criteria: {
3464
+ false: {
3465
+ examples: [
3466
+ "BRIEF: keeps both versions when edits conflict. TEXT: Keep both versions",
3467
+ "BRIEF: 14-day free trial, no card required. TEXT: Start your free trial",
3468
+ "BRIEF: a desktop sync app. TEXT: Email address",
3469
+ "BRIEF: 14-day free trial, no card required; $8 per seat per month. TEXT: 14 days. No card. $8 per seat after that.",
3470
+ "BRIEF: keeps both versions when edits conflict. TEXT: Nobody’s work gets thrown away, even when two people edit the same file offline."
3471
+ ],
3472
+ what: "Every claim in TEXT rewords, summarises or directly follows from the brief, or TEXT is ordinary interface chrome or an instruction to the reader."
3473
+ },
3474
+ true: {
3475
+ examples: [
3476
+ "BRIEF: local network first, falls back to an encrypted relay. TEXT: Relay only",
3477
+ "BRIEF: 14-day free trial, no card required. TEXT: Cancel anytime during the trial",
3478
+ "BRIEF: keeps both versions side by side. TEXT: Most sync tools pick a winner and silently overwrite the other."
3479
+ ],
3480
+ what: "TEXT offers or asserts a capability, setting, mode, role, policy, integration or comparison the brief never states."
3481
+ }
3482
+ },
3483
+ instructions: "BRIEF is everything known about the product. Does TEXT offer or assert a capability, setting, mode, role, policy, integration, page, or claim about other products that BRIEF neither states nor directly implies? Return false for rewording, shortening or drawing a direct consequence from BRIEF, for ordinary interface chrome every app has (Save, Cancel, Email, Password, Settings, Sign in), and for instructions to the reader."
3484
+ },
3485
+ source: {
3486
+ line: 10,
3487
+ path: "skills/ui-design/rules/slop-invented-behaviour.md",
3488
+ repo: "mblode/agent-skills",
3489
+ ruleId: "slop-invented-behaviour"
3490
+ },
3491
+ thresholds: {
3492
+ act: .75,
3493
+ review: .4
3494
+ },
3495
+ title: "States a capability the brief never gave",
3496
+ unit: [
3497
+ "paragraph",
3498
+ "heading",
3499
+ "jsx-text",
3500
+ "attr-string"
3501
+ ]
3502
+ }, "src/rules/code/prose.ts")];
3503
+ //#endregion
3186
3504
  //#region src/rules/code/skill-classes.ts
3187
3505
  const make = (id, title, categoryId, source, check, hint) => codeRule({
3188
3506
  categoryId,
@@ -3227,6 +3545,67 @@ const SKILL_CLASS_RULES = [
3227
3545
  }, "Prefer transform/opacity for frequent motion; measure deliberate container resizing before changing it.")
3228
3546
  ];
3229
3547
  //#endregion
3548
+ //#region src/rules/code/slop-source.ts
3549
+ const EYEBROW = /<(?:p|span|div)\b[^>]*className="[^"]*\b(?:uppercase|text-xs|text-\[(?:9|10|11|12)px\])[^"]*"[^>]*>(?:\s*<span[^>]*\/>)?[^<{]{2,80}<\/(?:p|span|div)>\s*<h[1-3]\b/gu;
3550
+ const FACT = /\b\d+[- ]days?\b|\bno (?:credit )?card\b|\bcancel any ?time\b|\$\d+(?:\.\d\d)?(?= ?(?:\/|per|a) ?(?:seat|user|member|month))/giu;
3551
+ const MAX_EYEBROWS = 2;
3552
+ const MAX_REPEATS = 3;
3553
+ const eyebrows = (unit) => {
3554
+ const found = [...unit.text.matchAll(EYEBROW)];
3555
+ if (found.length <= MAX_EYEBROWS) return none;
3556
+ return {
3557
+ ...hit(`${found.length} headings carry an eyebrow label`),
3558
+ offset: found[MAX_EYEBROWS]?.index
3559
+ };
3560
+ };
3561
+ const factPadding = (unit) => {
3562
+ const counts = /* @__PURE__ */ new Map();
3563
+ for (const m of unit.text.matchAll(FACT)) {
3564
+ const key = m[0].toLowerCase().replaceAll(/[- ]/gu, " ");
3565
+ const seen = counts.get(key) ?? {
3566
+ first: m.index,
3567
+ n: 0
3568
+ };
3569
+ counts.set(key, {
3570
+ ...seen,
3571
+ n: seen.n + 1
3572
+ });
3573
+ }
3574
+ const [top] = [...counts].toSorted((a, b) => b[1].n - a[1].n);
3575
+ if (!top || top[1].n <= MAX_REPEATS) return none;
3576
+ return {
3577
+ ...hit(`"${top[0]}" appears ${top[1].n} times`),
3578
+ offset: top[1].first
3579
+ };
3580
+ };
3581
+ const source = (ruleId) => ({
3582
+ line: 10,
3583
+ path: `skills/ui-design/rules/${ruleId}.md`,
3584
+ repo: "mblode/agent-skills",
3585
+ ruleId
3586
+ });
3587
+ const SLOP_SOURCE_RULES = [codeRule({
3588
+ categoryId: "visual-hierarchy",
3589
+ check: eyebrows,
3590
+ hint: "Keep an eyebrow on at most two sections, where it names something the heading cannot. Delete the rest.",
3591
+ id: "craft-eyebrow-overuse",
3592
+ scope: { include: ["**/*.tsx", "**/*.jsx"] },
3593
+ source: source("slop-eyebrow-overuse"),
3594
+ status: "review-only",
3595
+ title: "An eyebrow over nearly every heading",
3596
+ unit: ["source"]
3597
+ }, "src/rules/code/slop-source.ts"), codeRule({
3598
+ categoryId: "evidence-over-claims",
3599
+ check: factPadding,
3600
+ hint: "Say each fact once, where the reader needs it: the price on the plan, the trial on the button. Cut the sections that exist only to repeat them.",
3601
+ id: "copywriting-fact-padding",
3602
+ scope: { include: ["**/*.tsx", "**/*.jsx"] },
3603
+ source: source("slop-fact-padding"),
3604
+ status: "review-only",
3605
+ title: "The same few facts restated to fill the page",
3606
+ unit: ["source"]
3607
+ }, "src/rules/code/slop-source.ts")];
3608
+ //#endregion
3230
3609
  //#region src/rules/code/typography.ts
3231
3610
  const MODULE = "src/rules/code/typography.ts";
3232
3611
  const AUDIT = "skills/typography-audit/rules";
@@ -3476,7 +3855,9 @@ const CODE_RULES = [
3476
3855
  ...DOCUMENT_RULES,
3477
3856
  ...IMAGE_RULES,
3478
3857
  ...SKILL_CLASS_RULES,
3479
- ...MOTION_SOURCE_RULES
3858
+ ...MOTION_SOURCE_RULES,
3859
+ ...SLOP_SOURCE_RULES,
3860
+ ...PROSE_RULES
3480
3861
  ];
3481
3862
  //#endregion
3482
3863
  //#region src/rules/load.ts
@@ -3776,7 +4157,11 @@ const runEval = async (options, ctx = {}) => {
3776
4157
  knownRuleIds
3777
4158
  });
3778
4159
  if (options.split && options.split !== "all") items = items.filter((i) => i.split === options.split);
3779
- if (!items.length) throw new Error("No corpus items in the selected split.");
4160
+ if (options.sources) {
4161
+ const sources = new Set(options.sources);
4162
+ items = items.filter((i) => sources.has(i.labelSource));
4163
+ }
4164
+ if (!items.length) throw new Error("No corpus items in the selected split and label sources.");
3780
4165
  const model = options.model ?? "jev-latest";
3781
4166
  const resultsDir = options.resultsDir ?? defaultResultsDir();
3782
4167
  const cache = new AnswerCache(path.join(resultsDir, "cache"), !options.noCache);
@@ -3793,6 +4178,7 @@ const runEval = async (options, ctx = {}) => {
3793
4178
  model,
3794
4179
  recorder
3795
4180
  });
4181
+ if (unknowns.some((u) => u.reason === "auth")) throw new ProviderError("auth", 401);
3796
4182
  const labelSources = {};
3797
4183
  for (const item of items) labelSources[item.labelSource] = (labelSources[item.labelSource] ?? 0) + 1;
3798
4184
  const evals = evaluateRules(items, rules, probabilities, unknowns, skipped);
@@ -3812,41 +4198,56 @@ const runEval = async (options, ctx = {}) => {
3812
4198
  exitCode,
3813
4199
  labelSources,
3814
4200
  referenceCoverage: corpusCoverage(items, rules),
3815
- report: (options.check ? `Regression check: ${exitCode === 0 ? "PASS" : exitCode === 1 ? "FAIL (reference disagreement)" : "INCOMPLETE (missing evaluations)"}. Uses the review threshold.\n` : "") + (labelSources.ai ? `AI-labeled reference: ${labelSources.ai} items. Metrics measure agreement with AI labels, not human judgments.\n` : "") + renderEval(evals, usage, Boolean(options.dryRun), pendingRequests),
4201
+ report: (options.check ? `Regression check: ${exitCode === 0 ? "PASS" : exitCode === 1 ? "FAIL (reference disagreement)" : "INCOMPLETE (missing evaluations)"}. Uses the review threshold.\n` : "") + (labelSources.ai ? `AI-labeled reference: ${labelSources.ai} items. Metrics measure agreement with AI labels, not human judgments.\n` : "") + renderEval(evals, usage, Boolean(options.dryRun), pendingRequests) + (options.disagreements && !options.dryRun ? renderDisagreements(evals, items, rules) : ""),
3816
4202
  rules: evals,
3817
4203
  usage
3818
4204
  };
3819
4205
  };
3820
4206
  //#endregion
3821
- //#region src/scan/storage.ts
3822
- const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
3823
- const object$1 = (value) => !!value && typeof value === "object" && !Array.isArray(value);
3824
- const readJson = (file) => {
3825
- try {
3826
- return JSON.parse(fs.readFileSync(file, "utf-8"));
3827
- } catch {
3828
- throw new InputError("INVALID_SCAN_INPUT", `Cannot read JSON input: ${file}`);
4207
+ //#region src/reduce/suppress.ts
4208
+ const PATTERN = /(?:taste-lint-ignore|ui-audit-ignore)\s*:\s*([a-z0-9-]+)/giu;
4209
+ const suppressionsFor = (sourceLines, line) => {
4210
+ const ids = /* @__PURE__ */ new Set();
4211
+ for (const n of [line - 2, line - 1]) {
4212
+ const text = sourceLines[n];
4213
+ if (!text) continue;
4214
+ for (const m of text.matchAll(PATTERN)) ids.add(m[1].toLowerCase());
3829
4215
  }
4216
+ return ids;
3830
4217
  };
3831
- const writeJson = (file, value) => {
3832
- fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
3833
- const temp = `${file}.${randomUUID()}.tmp`;
3834
- try {
3835
- fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
3836
- fs.renameSync(temp, file);
3837
- } finally {
3838
- fs.rmSync(temp, { force: true });
3839
- }
4218
+ const applySuppressions = (findings, rules, sources) => {
4219
+ const byId = new Map(rules.map((r) => [r.id, r]));
4220
+ return findings.map((f) => {
4221
+ const lines = sources.get(f.file);
4222
+ if (!lines) return f;
4223
+ const ids = suppressionsFor(lines, f.line);
4224
+ const portedId = byId.get(f.ruleId)?.source.ruleId?.toLowerCase();
4225
+ if (ids.has(f.ruleId) || portedId && ids.has(portedId) || ids.has("all")) return {
4226
+ ...f,
4227
+ suppressed: true
4228
+ };
4229
+ return f;
4230
+ });
3840
4231
  };
3841
4232
  //#endregion
3842
4233
  //#region src/scan/samples.ts
3843
- const makeSamples = (units, rules, answers, negatives, perStratum = 3, sourceRepo = "scan") => {
4234
+ const STRATA = [
4235
+ "ignored",
4236
+ "disagree",
4237
+ "review",
4238
+ "high",
4239
+ "negative"
4240
+ ];
4241
+ const makeSamples = (units, rules, answers, negatives, perStratum = 3, sourceRepo = "scan", sources = /* @__PURE__ */ new Map()) => {
3844
4242
  const buckets = /* @__PURE__ */ new Map();
3845
4243
  const seen = /* @__PURE__ */ new Set();
3846
4244
  const semanticRules = rules.filter((rule) => rule.question);
3847
4245
  for (const unit of units) for (const rule of semanticRules) {
3848
- const probability = answers.get(unit.id)?.[rule.id] ?? (negatives.get(unit.id)?.has(rule.id) ? 0 : void 0);
4246
+ const answered = answers.get(unit.id)?.[rule.id];
4247
+ const probability = answered ?? (negatives.get(unit.id)?.has(rule.id) ? 0 : void 0);
3849
4248
  if (probability === void 0) continue;
4249
+ const ignores = suppressionsFor(sources.get(unit.file) ?? [], unit.line);
4250
+ const stratum = ignores.has(rule.id) || ignores.has("all") || ignores.has(rule.source.ruleId?.toLowerCase() ?? rule.id) ? "ignored" : answered !== void 0 && rule.mechanical && answered < rule.thresholds.review ? "disagree" : probability < rule.thresholds.review ? "negative" : probability < rule.thresholds.act ? "review" : "high";
3850
4251
  const { docType, role, headingAbove } = unit.context;
3851
4252
  const context = {
3852
4253
  docType,
@@ -3893,28 +4294,29 @@ const makeSamples = (units, rules, answers, negatives, perStratum = 3, sourceRep
3893
4294
  rubric,
3894
4295
  ruleId: rule.id
3895
4296
  };
3896
- const bucket = `${rule.id}:${docType}:${probability < rule.thresholds.review ? "negative" : probability < rule.thresholds.act ? "review" : "high"}`;
4297
+ const bucket = `${STRATA.indexOf(stratum)}:${rule.id}:${docType}`;
3897
4298
  const list = buckets.get(bucket) ?? [];
3898
4299
  list.push(sample);
3899
4300
  buckets.set(bucket, list);
3900
4301
  }
3901
- return [...buckets.values()].flatMap((list) => list.toSorted((a, b) => a.id.localeCompare(b.id)).slice(0, perStratum)).toSorted((a, b) => a.id.localeCompare(b.id));
4302
+ return [...buckets.entries()].toSorted(([a], [b]) => a.localeCompare(b)).flatMap(([, list]) => list.toSorted((a, b) => a.id.localeCompare(b.id)).slice(0, perStratum));
3902
4303
  };
3903
4304
  const labelsToCorpus = (file, destination, rulesDir) => {
3904
4305
  const raw = readJson(file);
3905
- if (!object$1(raw) || raw.version !== 1 && raw.version !== 2 || !Array.isArray(raw.samples)) throw new InputError("INVALID_LABELS", "Expected a version 1 or 2 blind sample file.");
4306
+ if (!object$2(raw) || raw.version !== 1 && raw.version !== 2 || !Array.isArray(raw.samples)) throw new InputError("INVALID_LABELS", "Expected a version 1 or 2 blind sample file.");
3906
4307
  if (raw.completed !== void 0 && raw.completed !== raw.samples.length) throw new InputError("INVALID_LABELS", "Labeling is incomplete. Review every sample before importing.");
3907
4308
  const annotation = raw.annotation;
3908
- const ai = object$1(annotation) && annotation.source === "ai";
4309
+ const ai = object$2(annotation) && annotation.source === "ai";
3909
4310
  if (annotation !== void 0 && (!ai || typeof annotation.model !== "string" || !annotation.model.trim() || typeof annotation.promptHash !== "string" || !/^[a-f0-9]{64}$/.test(annotation.promptHash))) throw new InputError("INVALID_LABELS", "AI annotation requires a model and prompt hash.");
3910
4311
  const ids = /* @__PURE__ */ new Set();
3911
4312
  const items = [];
3912
4313
  for (const row of raw.samples) {
3913
- if (!object$1(row) || typeof row.id !== "string" || ids.has(row.id) || typeof row.ruleId !== "string" || raw.version === 2 && (!object$1(row.rubric) || row.rubric.type !== "noul" || typeof row.criterion !== "string" || row.rubric.instructions !== row.criterion) || !object$1(row.item) || row.item.id !== row.id || row.label !== null && typeof row.label !== "boolean") throw new InputError("INVALID_LABELS", "Samples require unique IDs and boolean or null labels.");
4314
+ if (!object$2(row) || typeof row.id !== "string" || ids.has(row.id) || typeof row.ruleId !== "string" || raw.version === 2 && (!object$2(row.rubric) || row.rubric.type !== "noul" || typeof row.criterion !== "string" || row.rubric.instructions !== row.criterion) || !object$2(row.item) || row.item.id !== row.id || row.label !== null && typeof row.label !== "boolean") throw new InputError("INVALID_LABELS", "Samples require unique IDs and boolean or null labels.");
3914
4315
  ids.add(row.id);
3915
4316
  if (row.label === null) continue;
3916
4317
  items.push({
3917
4318
  ...row.item,
4319
+ ...typeof row.note === "string" && row.note.trim() ? { note: row.note.trim() } : {},
3918
4320
  labelSource: ai ? "ai" : "hand",
3919
4321
  ...ai ? {
3920
4322
  labelModel: annotation.model,
@@ -3939,15 +4341,27 @@ const labelsToCorpus = (file, destination, rulesDir) => {
3939
4341
  };
3940
4342
  //#endregion
3941
4343
  //#region src/commands/eval.ts
4344
+ const LABEL_SOURCES = [
4345
+ "hand",
4346
+ "ai",
4347
+ "manifest",
4348
+ "manifest-weak",
4349
+ "sweep"
4350
+ ];
4351
+ const parseSources = (value) => value?.split(",").map((s) => s.trim()).filter(Boolean).map((s) => {
4352
+ if (!LABEL_SOURCES.includes(s)) throw new Error(`--source must be one or more of ${LABEL_SOURCES.join(", ")}`);
4353
+ return s;
4354
+ });
3942
4355
  function registerEvalCommand(program) {
3943
4356
  const command = program.command("eval").enablePositionalOptions().description("Run every rule over its labelled corpus and report precision, recall and calibration").option("--corpus <path>", "Corpus directory (default: packaged data/corpus)").option("--rules <path>", "Rules directory").addOption(new Option("--output <format>", "Output format").choices(["text", "json"]).default("text")).option("--only <ids>", "Comma-separated rule ids").addOption(new Option("--split <name>", "Evaluation split").choices([
3944
4357
  "dev",
3945
4358
  "holdout",
3946
4359
  "all"
3947
- ]).default("all")).option("--include-weak", "Include manifest-weak labels").option("--check", "Fail on reference disagreement; incomplete evaluation exits 2").option("--dry-run", "Plan without calling Jev").option("--no-cache", "Ignore cached answers").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
4360
+ ]).default("all")).option("--include-weak", "Include manifest-weak labels").option("--source <sources>", "Only these label sources, e.g. hand to measure agreement with you").option("--disagreements", "List items where the judge and the label disagree, with text and notes").option("--check", "Fail on reference disagreement; incomplete evaluation exits 2").option("--dry-run", "Plan without calling Jev").option("--no-cache", "Ignore cached answers").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
3948
4361
  const result = await runEval({
3949
4362
  check: options.check,
3950
4363
  corpusDir: options.corpus,
4364
+ disagreements: options.disagreements,
3951
4365
  dryRun: options.dryRun,
3952
4366
  includeWeak: options.includeWeak,
3953
4367
  model: options.model,
@@ -3955,6 +4369,7 @@ function registerEvalCommand(program) {
3955
4369
  only: options.only?.split(",").map((s) => s.trim()).filter(Boolean),
3956
4370
  resultsDir: options.resultsDir,
3957
4371
  rulesDir: options.rules,
4372
+ sources: parseSources(options.source),
3958
4373
  split: options.split
3959
4374
  });
3960
4375
  process.stdout.write(options.output === "json" ? `${JSON.stringify(result, null, 2)}\n` : result.report);
@@ -3973,8 +4388,77 @@ function registerEvalCommand(program) {
3973
4388
  command.command("labels <samples>").description("Convert a labelled sample file from lint --samples into a corpus JSONL").requiredOption("--out <file>", "New corpus JSONL file").option("--rules <path>", "Rules directory used to validate labels").action((file, options) => {
3974
4389
  process.stdout.write(`Exported ${labelsToCorpus(file, options.out, options.rules)} labels.\n`);
3975
4390
  });
4391
+ command.command("label <samples>").description("Label a lint --samples file yourself, one sample at a time; saves after every answer").option("--minutes <n>", "Stop after this many minutes", "120").action(async (file, options) => {
4392
+ const minutes = Number(options.minutes);
4393
+ if (!(Number.isFinite(minutes) && minutes > 0)) throw new Error("--minutes must be a positive number");
4394
+ const result = await labelSession(file, {
4395
+ input: process.stdin,
4396
+ minutes,
4397
+ output: process.stdout
4398
+ });
4399
+ process.stdout.write(`\n${result.labelled} labelled, ${result.unsure} unsure, ${result.remaining} left${result.stopped === "timebox" ? " (timebox reached)" : ""}.\nNext: taste-lint eval gaps ${file}, then taste-lint eval labels ${file} --out data/corpus/<name>.jsonl\n`);
4400
+ });
4401
+ command.command("gaps <samples...>").description("Group hand labels by rule: unsure answers and notes show where the rubric needs sharpening").action((files) => {
4402
+ process.stdout.write(renderGaps(labelGaps(files)));
4403
+ });
4404
+ command.command("agreement <hand> <other>").description("Compare another labeller's sample file (usually AI) with your hand labels, per rule").option("--floor <p>", "Agreement lower bound needed to trust the other labeller", "0.8").action((hand, other, options) => {
4405
+ const floor = Number(options.floor);
4406
+ if (!(Number.isFinite(floor) && floor > 0 && floor < 1)) throw new Error("--floor must be a number between 0 and 1 (exclusive)");
4407
+ const rows = labelAgreement(hand, other, floor);
4408
+ process.stdout.write(renderAgreement(rows, floor));
4409
+ process.exitCode = rows.every((r) => r.trusted) ? 0 : 1;
4410
+ });
3976
4411
  }
3977
4412
  //#endregion
4413
+ //#region src/extract/earlier.ts
4414
+ const NOT_PROSE = /* @__PURE__ */ new Set([
4415
+ "heading",
4416
+ "button",
4417
+ "link",
4418
+ "label",
4419
+ "placeholder",
4420
+ "aria"
4421
+ ]);
4422
+ const STOP = new Set("about above after again also always another because been before being below between both came come could does doing done down each even every from have having here into just like made make many more most much must never only other over same should since some such than that their them then there these they this those through very want were what when where which while will with would your yours you're we're it's that's there's isn't don't can't won't".split(" "));
4423
+ const SENTENCE = /[^.!?\n]+[.!?]*/gu;
4424
+ const WORD = /[\p{L}\p{N}$][\p{L}\p{N}$'’.-]*[\p{L}\p{N}]|[\p{L}\p{N}]/gu;
4425
+ const MIN_SHARED = 3;
4426
+ const MAX_EVIDENCE = 8;
4427
+ const tokens = (sentence) => {
4428
+ const out = /* @__PURE__ */ new Set();
4429
+ for (const [raw] of sentence.toLowerCase().matchAll(WORD)) {
4430
+ const word = raw.replaceAll("’", "'");
4431
+ if (/\d/u.test(word)) out.add(word);
4432
+ else if (word.length >= 4 && !STOP.has(word)) out.add(word.length > 4 ? word.replace(/(?:'s|s)$/u, "") : word);
4433
+ }
4434
+ return out;
4435
+ };
4436
+ const isProse = (unit) => (unit.kind === "paragraph" || unit.kind === "jsx-text") && !unit.inCode && !NOT_PROSE.has(unit.context.role);
4437
+ const attachEarlier = (units) => {
4438
+ const prose = units.filter(isProse).toSorted((a, b) => a.sourceStart - b.sourceStart);
4439
+ const split = prose.map((unit) => [...unit.text.matchAll(SENTENCE)].map(([s]) => s.trim()).filter(Boolean).map((text) => ({
4440
+ text,
4441
+ words: tokens(text)
4442
+ })));
4443
+ const spread = /* @__PURE__ */ new Map();
4444
+ for (const sentences of split) for (const word of new Set(sentences.flatMap((s) => [...s.words]))) spread.set(word, (spread.get(word) ?? 0) + 1);
4445
+ const subject = (word) => prose.length >= 6 && (spread.get(word) ?? 0) * 3 > prose.length;
4446
+ const seen = [];
4447
+ const index = /* @__PURE__ */ new Map();
4448
+ for (const [u, unit] of prose.entries()) {
4449
+ const matched = /* @__PURE__ */ new Map();
4450
+ for (const sentence of split[u] ?? []) {
4451
+ const words = [...sentence.words].filter((w) => !subject(w));
4452
+ const shared = /* @__PURE__ */ new Map();
4453
+ for (const word of words) for (const id of index.get(word) ?? []) shared.set(id, (shared.get(id) ?? 0) + 1);
4454
+ for (const [id, n] of shared) if (n >= MIN_SHARED || n >= 2 && n * 2 >= words.length) matched.set(id, Math.max(matched.get(id) ?? 0, n));
4455
+ const id = seen.push(sentence.text) - 1;
4456
+ for (const word of words) index.set(word, [...index.get(word) ?? [], id]);
4457
+ }
4458
+ if (matched.size > 0) unit.context.earlier = [...matched].toSorted((a, b) => b[1] - a[1] || a[0] - b[0]).slice(0, MAX_EVIDENCE).toSorted((a, b) => a[0] - b[0]).map(([id]) => seen[id]).join("\n");
4459
+ }
4460
+ };
4461
+ //#endregion
3978
4462
  //#region src/lib/frontmatter.ts
3979
4463
  const stripBom = (text) => text.codePointAt(0) === 65279 ? ` ${text.slice(1)}` : text;
3980
4464
  const scalar = (value) => {
@@ -4763,6 +5247,7 @@ const extractSource = (config, file, source, repository) => {
4763
5247
  config,
4764
5248
  docType
4765
5249
  }) : [];
5250
+ attachEarlier(units);
4766
5251
  const whole = makeUnit(file, new LineIndex(source), {
4767
5252
  context: {
4768
5253
  docType,
@@ -5345,32 +5830,6 @@ const buildScorecard = (units, rules, findings, unknowns, silent) => {
5345
5830
  };
5346
5831
  };
5347
5832
  //#endregion
5348
- //#region src/reduce/suppress.ts
5349
- const PATTERN = /(?:taste-lint-ignore|ui-audit-ignore)\s*:\s*([a-z0-9-]+)/giu;
5350
- const suppressionsFor = (sourceLines, line) => {
5351
- const ids = /* @__PURE__ */ new Set();
5352
- for (const n of [line - 2, line - 1]) {
5353
- const text = sourceLines[n];
5354
- if (!text) continue;
5355
- for (const m of text.matchAll(PATTERN)) ids.add(m[1].toLowerCase());
5356
- }
5357
- return ids;
5358
- };
5359
- const applySuppressions = (findings, rules, sources) => {
5360
- const byId = new Map(rules.map((r) => [r.id, r]));
5361
- return findings.map((f) => {
5362
- const lines = sources.get(f.file);
5363
- if (!lines) return f;
5364
- const ids = suppressionsFor(lines, f.line);
5365
- const portedId = byId.get(f.ruleId)?.source.ruleId?.toLowerCase();
5366
- if (ids.has(f.ruleId) || portedId && ids.has(portedId) || ids.has("all")) return {
5367
- ...f,
5368
- suppressed: true
5369
- };
5370
- return f;
5371
- });
5372
- };
5373
- //#endregion
5374
5833
  //#region src/scan/git.ts
5375
5834
  const git = (root, args) => {
5376
5835
  try {
@@ -5547,6 +6006,10 @@ const runLint = async (options = {}, ctx = {}) => {
5547
6006
  sources.set(file, source.split("\n"));
5548
6007
  }
5549
6008
  if (options.extraUnits) units.push(...options.extraUnits);
6009
+ if (options.brief) {
6010
+ const brief = fs.readFileSync(path.resolve(options.brief), "utf-8").trim();
6011
+ for (const unit of units) unit.context.brief = brief;
6012
+ }
5550
6013
  const scope = {
5551
6014
  byDocType: {},
5552
6015
  diagnostics: scan.messages,
@@ -5609,7 +6072,7 @@ const runLint = async (options = {}, ctx = {}) => {
5609
6072
  onProgress: ctx.onProgress,
5610
6073
  recorder
5611
6074
  });
5612
- ctx.onEvidence?.(units, judgement.answers, judgement.negatives, rules);
6075
+ ctx.onEvidence?.(units, judgement.answers, judgement.negatives, rules, sources);
5613
6076
  const status = options.dryRun ? "dry-run" : judgement.usage.errors > 0 || parseFailures.size > 0 || units.length === 0 ? "incomplete" : "complete";
5614
6077
  const jf = jevFindings(judgement.jobs, judgement.answers);
5615
6078
  let findings = [...judgement.mechanical, ...jf.findings];
@@ -5863,7 +6326,7 @@ const renderTty = (result, options = {}) => {
5863
6326
  //#region src/commands/lint.ts
5864
6327
  const quote = (value) => `'${value.replaceAll("'", "'\"'\"'")}'`;
5865
6328
  function registerLintCommand(program) {
5866
- program.command("lint").description("Lint files (or a rendered page) against the taste rules").argument("[paths...]", "Files or directories to lint, relative to --root").option("--root <path>", "Project root for globs and config", process.cwd()).option("--only <ids>", "Comma-separated rule ids").option("--exclude <globs>", "Comma-separated globs to skip (added to taste-lint.config.json exclude)").option("--dry-run", "Plan and estimate cost without calling Jev").option("--print-requests", "With --dry-run and --output tty, print request JSONL to stderr").addOption(new Option("--profile <name>", "Scope: product (tsx, jsx, css), writing (md, mdx), instructions (AGENTS.md, skills, plans) or all").choices(PROFILE_NAMES)).option("--since <ref>", "Report only findings on lines changed since this Git revision").option("--samples <file>", "Write blind labelling samples for the Jev rules that ran, without their predicted scores").option("--no-cache", "Ignore cached answers (new answers are still recorded)").addOption(new Option("--fail-on <severity>", "Lowest severity that fails the run").choices(SEVERITIES).default("minor")).option("--fix", "Apply deterministic fixes for act-band findings").addOption(new Option("--output <format>", "Output format").choices([
6329
+ program.command("lint").description("Lint files (or a rendered page) against the taste rules").argument("[paths...]", "Files or directories to lint, relative to --root").option("--root <path>", "Project root for globs and config", process.cwd()).option("--only <ids>", "Comma-separated rule ids").option("--exclude <globs>", "Comma-separated globs to skip (added to taste-lint.config.json exclude)").option("--dry-run", "Plan and estimate cost without calling Jev").option("--print-requests", "With --dry-run and --output tty, print request JSONL to stderr").addOption(new Option("--profile <name>", "Scope: product (tsx, jsx, css), writing (md, mdx), instructions (AGENTS.md, skills, plans) or all").choices(PROFILE_NAMES)).option("--since <ref>", "Report only findings on lines changed since this Git revision").option("--brief <file>", "What the product is and does; copy that states anything else is flagged").option("--samples <file>", "Write blind labelling samples for the Jev rules that ran, without their predicted scores").option("--no-cache", "Ignore cached answers (new answers are still recorded)").addOption(new Option("--fail-on <severity>", "Lowest severity that fails the run").choices(SEVERITIES).default("minor")).option("--fix", "Apply deterministic fixes for act-band findings").addOption(new Option("--output <format>", "Output format").choices([
5867
6330
  "tty",
5868
6331
  "json",
5869
6332
  "sarif"
@@ -5881,6 +6344,7 @@ function registerLintCommand(program) {
5881
6344
  let lastProgress = 0;
5882
6345
  let samples = [];
5883
6346
  const result = await runLint({
6347
+ brief: options.brief,
5884
6348
  dryRun: options.dryRun,
5885
6349
  exclude: options.exclude?.split(",").map((s) => s.trim()).filter(Boolean),
5886
6350
  extraUnits,
@@ -5896,8 +6360,8 @@ function registerLintCommand(program) {
5896
6360
  since: options.since,
5897
6361
  targets
5898
6362
  }, {
5899
- onEvidence: options.samples ? (units, answers, negatives, rules) => {
5900
- samples = makeSamples(units, rules, answers, negatives, 3, path.basename(config.root));
6363
+ onEvidence: options.samples ? (units, answers, negatives, rules, sources) => {
6364
+ samples = makeSamples(units, rules, answers, negatives, 3, path.basename(config.root), sources);
5901
6365
  } : void 0,
5902
6366
  onProgress: (progress) => {
5903
6367
  if (!(options.progress || process.stderr.isTTY)) return;
@@ -5929,10 +6393,11 @@ function registerLintCommand(program) {
5929
6393
  if (options.url) retryArgs.push("--url", options.url, "--selector", options.selector);
5930
6394
  if (options.profile) retryArgs.push("--profile", options.profile);
5931
6395
  if (options.since) retryArgs.push("--since", options.since);
6396
+ if (options.brief) retryArgs.push("--brief", options.brief);
5932
6397
  retryArgs.push("--", ...targets);
5933
6398
  const rerun = result.status === "incomplete" ? retryArgs.map(quote).join(" ") : void 0;
5934
6399
  if (options.samples) writeJson(options.samples, {
5935
- instructions: "Blind review: set label true for a violation, false for acceptable, or leave null when uncertain. No predicted probabilities are included.",
6400
+ instructions: "Blind review: set label true for a violation, false for acceptable, or leave null when uncertain. No predicted probabilities are included. Likely failures come first; label them yourself with taste-lint eval label.",
5936
6401
  samples,
5937
6402
  version: 2
5938
6403
  });
@@ -5994,6 +6459,7 @@ const promotionEvidence = (items, ruleId, pairs, threshold, floor, minItems) =>
5994
6459
  };
5995
6460
  //#endregion
5996
6461
  //#region src/eval/tune.ts
6462
+ const fromSources = (items, sources) => sources ? items.filter((item) => sources.includes(item.labelSource)) : items;
5997
6463
  const THRESHOLDS = Array.from({ length: 10 }, (_, i) => Math.round((.5 + i * .05) * 100) / 100);
5998
6464
  const decideThreshold = (pairs, currentAct, floor, minItems, review = 0) => {
5999
6465
  const n = pairs.length;
@@ -6047,7 +6513,7 @@ const runTune = async (options, ctx = {}) => {
6047
6513
  only: options.only
6048
6514
  }).filter((r) => r.tier !== "mechanical");
6049
6515
  const knownRuleIds = new Set(loadRules(rulesDir, { allowDraft: true }).map((r) => r.id));
6050
- const allItems = loadCorpus(resolveCorpusDir(options.corpusDir), rules, { knownRuleIds });
6516
+ const allItems = fromSources(loadCorpus(resolveCorpusDir(options.corpusDir), rules, { knownRuleIds }), options.sources);
6051
6517
  const items = allItems.filter((item) => item.split === "dev");
6052
6518
  const model = options.model ?? "jev-latest";
6053
6519
  const resultsDir = options.resultsDir ?? defaultResultsDir();
@@ -6122,7 +6588,7 @@ const runTuneAb = async (options, ctx = {}) => {
6122
6588
  };
6123
6589
  const variant = validateRule(merged, options.variantFile, rule.id);
6124
6590
  const knownRuleIds = new Set(loadRules(rulesDir, { allowDraft: true }).map((r) => r.id));
6125
- const items = loadCorpus(resolveCorpusDir(options.corpusDir), [rule], { knownRuleIds }).filter((i) => i.split === "dev" && rule.id in i.labels);
6591
+ const items = fromSources(loadCorpus(resolveCorpusDir(options.corpusDir), [rule], { knownRuleIds }), options.sources).filter((i) => i.split === "dev" && rule.id in i.labels);
6126
6592
  const model = options.model ?? "jev-latest";
6127
6593
  const resultsDir = options.resultsDir ?? defaultResultsDir();
6128
6594
  const cache = new AnswerCache(path.join(resultsDir, "cache"));
@@ -6168,7 +6634,7 @@ const ruleToRaw = (rule) => {
6168
6634
  //#endregion
6169
6635
  //#region src/commands/tune.ts
6170
6636
  function registerTuneCommand(program) {
6171
- program.command("tune").description("Pick act thresholds per rule from the dev split; --write updates data/rules/tuning.json").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--only <ids>", "Comma-separated rule ids").option("--floor <p>", "Required lower bound on act-band precision", "0.8").option("--min-items <n>", "Labelled dev items a rule needs before a decision", "10").option("--write", "Write tuning.json").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
6637
+ program.command("tune").description("Pick act thresholds per rule from the dev split; --write updates data/rules/tuning.json").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--only <ids>", "Comma-separated rule ids").option("--floor <p>", "Required lower bound on act-band precision", "0.8").option("--min-items <n>", "Labelled dev items a rule needs before a decision", "10").option("--write", "Write tuning.json").option("--source <sources>", "Only these label sources; tune on hand to fit the judge to you").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
6172
6638
  const result = await runTune({
6173
6639
  corpusDir: options.corpus,
6174
6640
  floor: Number(options.floor),
@@ -6177,17 +6643,19 @@ function registerTuneCommand(program) {
6177
6643
  only: options.only?.split(",").map((s) => s.trim()).filter(Boolean),
6178
6644
  resultsDir: options.resultsDir,
6179
6645
  rulesDir: options.rules,
6646
+ sources: parseSources(options.source),
6180
6647
  write: options.write
6181
6648
  });
6182
6649
  process.stdout.write(result.report);
6183
6650
  process.exitCode = result.exitCode;
6184
- }).command("ab").description("Compare two question wordings for one rule over the dev split (McNemar)").requiredOption("--rule <id>", "Rule id").requiredOption("--variant <file>", "YAML file with a replacement question").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
6651
+ }).command("ab").description("Compare two question wordings for one rule over the dev split (McNemar)").requiredOption("--rule <id>", "Rule id").requiredOption("--variant <file>", "YAML file with a replacement question").option("--corpus <path>", "Corpus directory").option("--rules <path>", "Rules directory").option("--source <sources>", "Only these label sources; tune on hand to fit the judge to you").option("--results-dir <path>", "Where logs and cache live").option("--model <id>", "Jev model id", DEFAULT_MODEL).action(async (options) => {
6185
6652
  const result = await runTuneAb({
6186
6653
  corpusDir: options.corpus,
6187
6654
  model: options.model,
6188
6655
  resultsDir: options.resultsDir,
6189
6656
  ruleId: options.rule,
6190
6657
  rulesDir: options.rules,
6658
+ sources: parseSources(options.source),
6191
6659
  variantFile: options.variant
6192
6660
  });
6193
6661
  process.stdout.write(result.report);