raterkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
raterkit/__init__.py ADDED
@@ -0,0 +1,21 @@
1
+ """raterkit: audit a labeled dataset before you trust it."""
2
+
3
+ from .probes import run_audit
4
+ from .result import AuditResult, ProbeResult, SkippedProbe
5
+ from .schema import Config, Gold, Rating, load_config, load_gold, load_ratings
6
+
7
+ __version__ = "0.1.0"
8
+
9
+ __all__ = [
10
+ "AuditResult",
11
+ "Config",
12
+ "Gold",
13
+ "ProbeResult",
14
+ "Rating",
15
+ "SkippedProbe",
16
+ "load_config",
17
+ "load_gold",
18
+ "load_ratings",
19
+ "run_audit",
20
+ "__version__",
21
+ ]
raterkit/cli.py ADDED
@@ -0,0 +1,74 @@
1
+ """Command-line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ from . import __version__
11
+ from .probes import run_audit
12
+ from .report import inject_readme, render_report, write_demo
13
+ from .schema import Config, load_config, load_gold, load_ratings
14
+
15
+
16
+ def main(argv: list[str] | None = None) -> int:
17
+ parser = argparse.ArgumentParser(
18
+ prog="raterkit", description="Audit a labeled dataset before you trust it."
19
+ )
20
+ parser.add_argument("--version", action="version", version=f"raterkit {__version__}")
21
+ sub = parser.add_subparsers(dest="command", required=True)
22
+
23
+ rep = sub.add_parser("report", help="audit a ratings file and write a report")
24
+ rep.add_argument("ratings", help="JSONL file, one rating per line")
25
+ rep.add_argument("--gold", help="JSONL file of gold labels")
26
+ rep.add_argument("--config", help="JSON audit configuration")
27
+ rep.add_argument("--out", default="audit", help="output directory")
28
+ rep.add_argument("--fail-on-flags", action="store_true", help="exit 2 when any probe flags")
29
+
30
+ demo = sub.add_parser("demo", help="run the eight-project demo")
31
+ demo.add_argument("--out", default="results", help="output directory")
32
+
33
+ inj = sub.add_parser("inject-readme", help="refresh the demo table inside a README")
34
+ inj.add_argument("readme")
35
+ inj.add_argument("--results", default="results", help="results directory")
36
+
37
+ args = parser.parse_args(argv)
38
+
39
+ if args.command == "report":
40
+ ratings = load_ratings(args.ratings)
41
+ gold = load_gold(args.gold) if args.gold else None
42
+ config = load_config(args.config) if args.config else Config()
43
+ audit = run_audit(ratings, gold, config)
44
+ out = Path(args.out)
45
+ out.mkdir(parents=True, exist_ok=True)
46
+ (out / "report.md").write_text(render_report(audit), encoding="utf-8")
47
+ (out / "report.json").write_text(
48
+ json.dumps(audit.to_dict(), indent=2) + "\n", encoding="utf-8"
49
+ )
50
+ flags = audit.flags
51
+ print(f"{len(flags)} flag(s)" + (": " + ", ".join(p.name for p in flags) if flags else ""))
52
+ print(f"wrote {out / 'report.md'} and {out / 'report.json'}")
53
+ if args.fail_on_flags and flags:
54
+ return 2
55
+ return 0
56
+
57
+ if args.command == "demo":
58
+ results = write_demo(args.out)
59
+ for row, seed, audit in results:
60
+ flag_text = ", ".join(sorted(f.name for f in audit.flags)) or "none"
61
+ print(f"{row.name:16s} seed {seed:2d} flags: {flag_text}")
62
+ print(f"wrote {args.out}/table.md, report.md, demo.json and figures/")
63
+ return 0
64
+
65
+ if args.command == "inject-readme":
66
+ inject_readme(args.readme, args.results)
67
+ print(f"updated demo table in {args.readme}")
68
+ return 0
69
+
70
+ return 1
71
+
72
+
73
+ if __name__ == "__main__":
74
+ sys.exit(main())
raterkit/dataset.py ADDED
@@ -0,0 +1,117 @@
1
+ """Indexed view over a list of ratings.
2
+
3
+ Builds the items x categories count table Krippendorff's alpha needs, plus the
4
+ per-rater and pairwise structures the rater-level probes need. All ordering is
5
+ deterministic: categories follow the config order when given, otherwise sorted
6
+ label order; items and raters are sorted by id.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+
13
+ import numpy as np
14
+
15
+ from .schema import Config, Label, Rating
16
+
17
+
18
+ def _category_order(ratings: list[Rating], config: Config) -> list[Label]:
19
+ if config.labels is not None:
20
+ return list(config.labels)
21
+ seen = {r.label for r in ratings}
22
+ if all(isinstance(v, int | float) and not isinstance(v, bool) for v in seen):
23
+ return sorted(seen)
24
+ return sorted(str(v) for v in seen)
25
+
26
+
27
+ @dataclass
28
+ class Dataset:
29
+ items: list[str]
30
+ raters: list[str]
31
+ cats: list[Label]
32
+ values: list[float]
33
+ counts: np.ndarray # items x categories
34
+ ratings: list[Rating]
35
+ config: Config
36
+
37
+ @classmethod
38
+ def build(cls, ratings: list[Rating], config: Config) -> Dataset:
39
+ cats = _category_order(ratings, config)
40
+ cat_index = {c: i for i, c in enumerate(cats)}
41
+ items = sorted({r.item_id for r in ratings})
42
+ raters = sorted({r.rater_id for r in ratings})
43
+ item_index = {it: i for i, it in enumerate(items)}
44
+ counts = np.zeros((len(items), len(cats)), dtype=float)
45
+ for r in ratings:
46
+ key = r.label if r.label in cat_index else str(r.label)
47
+ if key not in cat_index:
48
+ raise ValueError(f"label {r.label!r} not in configured labels {cats!r}")
49
+ counts[item_index[r.item_id], cat_index[key]] += 1
50
+ values = [
51
+ float(c) if isinstance(c, int | float) and not isinstance(c, bool) else float(i)
52
+ for i, c in enumerate(cats)
53
+ ]
54
+ return cls(items, raters, cats, values, counts, list(ratings), config)
55
+
56
+ # -- derived structures -------------------------------------------------
57
+
58
+ def cat_of(self, label: Label) -> int:
59
+ try:
60
+ return self.cats.index(label)
61
+ except ValueError:
62
+ return self.cats.index(str(label))
63
+
64
+ def by_item(self) -> dict[str, list[Rating]]:
65
+ out: dict[str, list[Rating]] = {it: [] for it in self.items}
66
+ for r in self.ratings:
67
+ out[r.item_id].append(r)
68
+ return out
69
+
70
+ def ratings_of(self, rater_id: str) -> list[Rating]:
71
+ return [r for r in self.ratings if r.rater_id == rater_id]
72
+
73
+ def pair_table(self, rater_id: str) -> tuple[np.ndarray, list[int]]:
74
+ """Pair units between one rater and every co-rater.
75
+
76
+ Each pair of (rater, other) labels on the same item becomes one
77
+ two-rating unit (a row with two counts). Returns the pair count table
78
+ and, per row, the index of the item it came from, so bootstrap
79
+ resampling can stay item-level.
80
+ """
81
+ rows: list[np.ndarray] = []
82
+ row_items: list[int] = []
83
+ item_index = {it: i for i, it in enumerate(self.items)}
84
+ for item_id, group in self.by_item().items():
85
+ mine = [r for r in group if r.rater_id == rater_id]
86
+ others = [r for r in group if r.rater_id != rater_id]
87
+ for a in mine:
88
+ for b in others:
89
+ row = np.zeros(len(self.cats))
90
+ row[self.cat_of(a.label)] += 1
91
+ row[self.cat_of(b.label)] += 1
92
+ rows.append(row)
93
+ row_items.append(item_index[item_id])
94
+ if not rows:
95
+ return np.zeros((0, len(self.cats))), []
96
+ return np.stack(rows), row_items
97
+
98
+ def marginal(self, rater_id: str | None = None, exclude: str | None = None) -> np.ndarray:
99
+ """Label distribution of one rater, or of everyone but ``exclude``."""
100
+ vec = np.zeros(len(self.cats))
101
+ for r in self.ratings:
102
+ if rater_id is not None and r.rater_id != rater_id:
103
+ continue
104
+ if exclude is not None and r.rater_id == exclude:
105
+ continue
106
+ vec[self.cat_of(r.label)] += 1
107
+ return vec
108
+
109
+ def item_times(self) -> dict[str, float] | None:
110
+ """Median rating time per item; None unless every rating has ``t``."""
111
+ if any(r.t is None for r in self.ratings) or not self.ratings:
112
+ return None
113
+ times: dict[str, list[float]] = {}
114
+ for r in self.ratings:
115
+ assert r.t is not None
116
+ times.setdefault(r.item_id, []).append(r.t)
117
+ return {it: float(np.median(np.array(ts))) for it, ts in times.items()}
raterkit/demo.py ADDED
@@ -0,0 +1,107 @@
1
+ """Eight demo annotation projects, seven implanted defects.
2
+
3
+ Each row implants exactly one defect and must produce exactly its target flag
4
+ set. Seeds are found by search (`_find_seed`) and then confirmed by running
5
+ the full audit: for threshold-riding defects — a lean or a drift sized near
6
+ the innocent band — the seed selection IS the defect being demonstrated, a
7
+ project where the failure is large enough to clear the evidence bar.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass, field
13
+
14
+ from .probes import run_audit
15
+ from .result import AuditResult
16
+ from .synthetic import simulate
17
+
18
+ MAX_SEED = 60
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class DemoRow:
23
+ name: str
24
+ defect: str
25
+ target: frozenset[str]
26
+ kwargs: dict = field(default_factory=dict)
27
+
28
+
29
+ ROWS: list[DemoRow] = [
30
+ DemoRow(
31
+ "clean",
32
+ "nothing",
33
+ frozenset(),
34
+ dict(competence=0.92),
35
+ ),
36
+ DemoRow(
37
+ "careless-crowd",
38
+ "every rater at 55% accuracy",
39
+ frozenset({"reliability"}),
40
+ dict(competence=0.55),
41
+ ),
42
+ DemoRow(
43
+ "one-spammer",
44
+ "one rater answers uniformly at random",
45
+ frozenset({"rogue raters"}),
46
+ dict(competence=0.93, n_raters=12, spammers=1, spammer_load=0.7),
47
+ ),
48
+ DemoRow(
49
+ "leaning-rater",
50
+ "one rater always takes the same side of ambiguous items",
51
+ frozenset({"rater bias"}),
52
+ dict(
53
+ competence=0.95,
54
+ n_items=400,
55
+ n_raters=8,
56
+ hard_share=0.4,
57
+ leaners=1,
58
+ with_gold=False,
59
+ ),
60
+ ),
61
+ DemoRow(
62
+ "tiring-crowd",
63
+ "competence decays 0.97 to 0.73 over the queue",
64
+ frozenset({"drift"}),
65
+ dict(competence=0.97, drift=0.24),
66
+ ),
67
+ DemoRow(
68
+ "leaky-split",
69
+ "12 test items duplicate a train item's text",
70
+ frozenset({"split leakage"}),
71
+ dict(competence=0.92, dup_across_splits=12),
72
+ ),
73
+ DemoRow(
74
+ "stale-gold",
75
+ "8% of gold labels are wrong",
76
+ frozenset({"stale gold"}),
77
+ dict(competence=0.92, gold_error=0.08),
78
+ ),
79
+ DemoRow(
80
+ "easy-but-skewed",
81
+ "94% of items are one class; raters 90% accurate",
82
+ frozenset({"reliability"}),
83
+ dict(prevalence=[0.94, 0.02, 0.02, 0.02], competence=0.9),
84
+ ),
85
+ ]
86
+
87
+
88
+ def _audit_row(row: DemoRow, seed: int) -> AuditResult:
89
+ ratings, gold, config = simulate(seed=seed, **row.kwargs)
90
+ return run_audit(ratings, gold, config)
91
+
92
+
93
+ def _find_seed(row: DemoRow) -> tuple[int, AuditResult]:
94
+ for seed in range(MAX_SEED):
95
+ audit = _audit_row(row, seed)
96
+ if frozenset(f.name for f in audit.flags) == row.target:
97
+ return seed, audit
98
+ raise RuntimeError(f"no seed below {MAX_SEED} lands {row.name} on its target flags")
99
+
100
+
101
+ def build() -> list[tuple[DemoRow, int, AuditResult]]:
102
+ """Audit all demo rows, confirming each lands exactly its target flag set."""
103
+ out = []
104
+ for row in ROWS:
105
+ seed, audit = _find_seed(row)
106
+ out.append((row, seed, audit))
107
+ return out