raterkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raterkit/__init__.py +21 -0
- raterkit/cli.py +74 -0
- raterkit/dataset.py +117 -0
- raterkit/demo.py +107 -0
- raterkit/probes.py +398 -0
- raterkit/py.typed +0 -0
- raterkit/report.py +225 -0
- raterkit/result.py +84 -0
- raterkit/schema.py +104 -0
- raterkit/stats.py +330 -0
- raterkit/synthetic.py +118 -0
- raterkit-0.1.0.dist-info/METADATA +269 -0
- raterkit-0.1.0.dist-info/RECORD +16 -0
- raterkit-0.1.0.dist-info/WHEEL +4 -0
- raterkit-0.1.0.dist-info/entry_points.txt +2 -0
- raterkit-0.1.0.dist-info/licenses/LICENSE +21 -0
raterkit/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""raterkit: audit a labeled dataset before you trust it."""
|
|
2
|
+
|
|
3
|
+
from .probes import run_audit
|
|
4
|
+
from .result import AuditResult, ProbeResult, SkippedProbe
|
|
5
|
+
from .schema import Config, Gold, Rating, load_config, load_gold, load_ratings
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"AuditResult",
|
|
11
|
+
"Config",
|
|
12
|
+
"Gold",
|
|
13
|
+
"ProbeResult",
|
|
14
|
+
"Rating",
|
|
15
|
+
"SkippedProbe",
|
|
16
|
+
"load_config",
|
|
17
|
+
"load_gold",
|
|
18
|
+
"load_ratings",
|
|
19
|
+
"run_audit",
|
|
20
|
+
"__version__",
|
|
21
|
+
]
|
raterkit/cli.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .probes import run_audit
|
|
12
|
+
from .report import inject_readme, render_report, write_demo
|
|
13
|
+
from .schema import Config, load_config, load_gold, load_ratings
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main(argv: list[str] | None = None) -> int:
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="raterkit", description="Audit a labeled dataset before you trust it."
|
|
19
|
+
)
|
|
20
|
+
parser.add_argument("--version", action="version", version=f"raterkit {__version__}")
|
|
21
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
22
|
+
|
|
23
|
+
rep = sub.add_parser("report", help="audit a ratings file and write a report")
|
|
24
|
+
rep.add_argument("ratings", help="JSONL file, one rating per line")
|
|
25
|
+
rep.add_argument("--gold", help="JSONL file of gold labels")
|
|
26
|
+
rep.add_argument("--config", help="JSON audit configuration")
|
|
27
|
+
rep.add_argument("--out", default="audit", help="output directory")
|
|
28
|
+
rep.add_argument("--fail-on-flags", action="store_true", help="exit 2 when any probe flags")
|
|
29
|
+
|
|
30
|
+
demo = sub.add_parser("demo", help="run the eight-project demo")
|
|
31
|
+
demo.add_argument("--out", default="results", help="output directory")
|
|
32
|
+
|
|
33
|
+
inj = sub.add_parser("inject-readme", help="refresh the demo table inside a README")
|
|
34
|
+
inj.add_argument("readme")
|
|
35
|
+
inj.add_argument("--results", default="results", help="results directory")
|
|
36
|
+
|
|
37
|
+
args = parser.parse_args(argv)
|
|
38
|
+
|
|
39
|
+
if args.command == "report":
|
|
40
|
+
ratings = load_ratings(args.ratings)
|
|
41
|
+
gold = load_gold(args.gold) if args.gold else None
|
|
42
|
+
config = load_config(args.config) if args.config else Config()
|
|
43
|
+
audit = run_audit(ratings, gold, config)
|
|
44
|
+
out = Path(args.out)
|
|
45
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
(out / "report.md").write_text(render_report(audit), encoding="utf-8")
|
|
47
|
+
(out / "report.json").write_text(
|
|
48
|
+
json.dumps(audit.to_dict(), indent=2) + "\n", encoding="utf-8"
|
|
49
|
+
)
|
|
50
|
+
flags = audit.flags
|
|
51
|
+
print(f"{len(flags)} flag(s)" + (": " + ", ".join(p.name for p in flags) if flags else ""))
|
|
52
|
+
print(f"wrote {out / 'report.md'} and {out / 'report.json'}")
|
|
53
|
+
if args.fail_on_flags and flags:
|
|
54
|
+
return 2
|
|
55
|
+
return 0
|
|
56
|
+
|
|
57
|
+
if args.command == "demo":
|
|
58
|
+
results = write_demo(args.out)
|
|
59
|
+
for row, seed, audit in results:
|
|
60
|
+
flag_text = ", ".join(sorted(f.name for f in audit.flags)) or "none"
|
|
61
|
+
print(f"{row.name:16s} seed {seed:2d} flags: {flag_text}")
|
|
62
|
+
print(f"wrote {args.out}/table.md, report.md, demo.json and figures/")
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
if args.command == "inject-readme":
|
|
66
|
+
inject_readme(args.readme, args.results)
|
|
67
|
+
print(f"updated demo table in {args.readme}")
|
|
68
|
+
return 0
|
|
69
|
+
|
|
70
|
+
return 1
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
sys.exit(main())
|
raterkit/dataset.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Indexed view over a list of ratings.
|
|
2
|
+
|
|
3
|
+
Builds the items x categories count table Krippendorff's alpha needs, plus the
|
|
4
|
+
per-rater and pairwise structures the rater-level probes need. All ordering is
|
|
5
|
+
deterministic: categories follow the config order when given, otherwise sorted
|
|
6
|
+
label order; items and raters are sorted by id.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from .schema import Config, Label, Rating
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _category_order(ratings: list[Rating], config: Config) -> list[Label]:
|
|
19
|
+
if config.labels is not None:
|
|
20
|
+
return list(config.labels)
|
|
21
|
+
seen = {r.label for r in ratings}
|
|
22
|
+
if all(isinstance(v, int | float) and not isinstance(v, bool) for v in seen):
|
|
23
|
+
return sorted(seen)
|
|
24
|
+
return sorted(str(v) for v in seen)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Dataset:
|
|
29
|
+
items: list[str]
|
|
30
|
+
raters: list[str]
|
|
31
|
+
cats: list[Label]
|
|
32
|
+
values: list[float]
|
|
33
|
+
counts: np.ndarray # items x categories
|
|
34
|
+
ratings: list[Rating]
|
|
35
|
+
config: Config
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def build(cls, ratings: list[Rating], config: Config) -> Dataset:
|
|
39
|
+
cats = _category_order(ratings, config)
|
|
40
|
+
cat_index = {c: i for i, c in enumerate(cats)}
|
|
41
|
+
items = sorted({r.item_id for r in ratings})
|
|
42
|
+
raters = sorted({r.rater_id for r in ratings})
|
|
43
|
+
item_index = {it: i for i, it in enumerate(items)}
|
|
44
|
+
counts = np.zeros((len(items), len(cats)), dtype=float)
|
|
45
|
+
for r in ratings:
|
|
46
|
+
key = r.label if r.label in cat_index else str(r.label)
|
|
47
|
+
if key not in cat_index:
|
|
48
|
+
raise ValueError(f"label {r.label!r} not in configured labels {cats!r}")
|
|
49
|
+
counts[item_index[r.item_id], cat_index[key]] += 1
|
|
50
|
+
values = [
|
|
51
|
+
float(c) if isinstance(c, int | float) and not isinstance(c, bool) else float(i)
|
|
52
|
+
for i, c in enumerate(cats)
|
|
53
|
+
]
|
|
54
|
+
return cls(items, raters, cats, values, counts, list(ratings), config)
|
|
55
|
+
|
|
56
|
+
# -- derived structures -------------------------------------------------
|
|
57
|
+
|
|
58
|
+
def cat_of(self, label: Label) -> int:
|
|
59
|
+
try:
|
|
60
|
+
return self.cats.index(label)
|
|
61
|
+
except ValueError:
|
|
62
|
+
return self.cats.index(str(label))
|
|
63
|
+
|
|
64
|
+
def by_item(self) -> dict[str, list[Rating]]:
|
|
65
|
+
out: dict[str, list[Rating]] = {it: [] for it in self.items}
|
|
66
|
+
for r in self.ratings:
|
|
67
|
+
out[r.item_id].append(r)
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
def ratings_of(self, rater_id: str) -> list[Rating]:
|
|
71
|
+
return [r for r in self.ratings if r.rater_id == rater_id]
|
|
72
|
+
|
|
73
|
+
def pair_table(self, rater_id: str) -> tuple[np.ndarray, list[int]]:
|
|
74
|
+
"""Pair units between one rater and every co-rater.
|
|
75
|
+
|
|
76
|
+
Each pair of (rater, other) labels on the same item becomes one
|
|
77
|
+
two-rating unit (a row with two counts). Returns the pair count table
|
|
78
|
+
and, per row, the index of the item it came from, so bootstrap
|
|
79
|
+
resampling can stay item-level.
|
|
80
|
+
"""
|
|
81
|
+
rows: list[np.ndarray] = []
|
|
82
|
+
row_items: list[int] = []
|
|
83
|
+
item_index = {it: i for i, it in enumerate(self.items)}
|
|
84
|
+
for item_id, group in self.by_item().items():
|
|
85
|
+
mine = [r for r in group if r.rater_id == rater_id]
|
|
86
|
+
others = [r for r in group if r.rater_id != rater_id]
|
|
87
|
+
for a in mine:
|
|
88
|
+
for b in others:
|
|
89
|
+
row = np.zeros(len(self.cats))
|
|
90
|
+
row[self.cat_of(a.label)] += 1
|
|
91
|
+
row[self.cat_of(b.label)] += 1
|
|
92
|
+
rows.append(row)
|
|
93
|
+
row_items.append(item_index[item_id])
|
|
94
|
+
if not rows:
|
|
95
|
+
return np.zeros((0, len(self.cats))), []
|
|
96
|
+
return np.stack(rows), row_items
|
|
97
|
+
|
|
98
|
+
def marginal(self, rater_id: str | None = None, exclude: str | None = None) -> np.ndarray:
|
|
99
|
+
"""Label distribution of one rater, or of everyone but ``exclude``."""
|
|
100
|
+
vec = np.zeros(len(self.cats))
|
|
101
|
+
for r in self.ratings:
|
|
102
|
+
if rater_id is not None and r.rater_id != rater_id:
|
|
103
|
+
continue
|
|
104
|
+
if exclude is not None and r.rater_id == exclude:
|
|
105
|
+
continue
|
|
106
|
+
vec[self.cat_of(r.label)] += 1
|
|
107
|
+
return vec
|
|
108
|
+
|
|
109
|
+
def item_times(self) -> dict[str, float] | None:
|
|
110
|
+
"""Median rating time per item; None unless every rating has ``t``."""
|
|
111
|
+
if any(r.t is None for r in self.ratings) or not self.ratings:
|
|
112
|
+
return None
|
|
113
|
+
times: dict[str, list[float]] = {}
|
|
114
|
+
for r in self.ratings:
|
|
115
|
+
assert r.t is not None
|
|
116
|
+
times.setdefault(r.item_id, []).append(r.t)
|
|
117
|
+
return {it: float(np.median(np.array(ts))) for it, ts in times.items()}
|
raterkit/demo.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Eight demo annotation projects, seven implanted defects.
|
|
2
|
+
|
|
3
|
+
Each row implants exactly one defect and must produce exactly its target flag
|
|
4
|
+
set. Seeds are found by search (`_find_seed`) and then confirmed by running
|
|
5
|
+
the full audit: for threshold-riding defects — a lean or a drift sized near
|
|
6
|
+
the innocent band — the seed selection IS the defect being demonstrated, a
|
|
7
|
+
project where the failure is large enough to clear the evidence bar.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
from .probes import run_audit
|
|
15
|
+
from .result import AuditResult
|
|
16
|
+
from .synthetic import simulate
|
|
17
|
+
|
|
18
|
+
MAX_SEED = 60
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DemoRow:
|
|
23
|
+
name: str
|
|
24
|
+
defect: str
|
|
25
|
+
target: frozenset[str]
|
|
26
|
+
kwargs: dict = field(default_factory=dict)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
ROWS: list[DemoRow] = [
|
|
30
|
+
DemoRow(
|
|
31
|
+
"clean",
|
|
32
|
+
"nothing",
|
|
33
|
+
frozenset(),
|
|
34
|
+
dict(competence=0.92),
|
|
35
|
+
),
|
|
36
|
+
DemoRow(
|
|
37
|
+
"careless-crowd",
|
|
38
|
+
"every rater at 55% accuracy",
|
|
39
|
+
frozenset({"reliability"}),
|
|
40
|
+
dict(competence=0.55),
|
|
41
|
+
),
|
|
42
|
+
DemoRow(
|
|
43
|
+
"one-spammer",
|
|
44
|
+
"one rater answers uniformly at random",
|
|
45
|
+
frozenset({"rogue raters"}),
|
|
46
|
+
dict(competence=0.93, n_raters=12, spammers=1, spammer_load=0.7),
|
|
47
|
+
),
|
|
48
|
+
DemoRow(
|
|
49
|
+
"leaning-rater",
|
|
50
|
+
"one rater always takes the same side of ambiguous items",
|
|
51
|
+
frozenset({"rater bias"}),
|
|
52
|
+
dict(
|
|
53
|
+
competence=0.95,
|
|
54
|
+
n_items=400,
|
|
55
|
+
n_raters=8,
|
|
56
|
+
hard_share=0.4,
|
|
57
|
+
leaners=1,
|
|
58
|
+
with_gold=False,
|
|
59
|
+
),
|
|
60
|
+
),
|
|
61
|
+
DemoRow(
|
|
62
|
+
"tiring-crowd",
|
|
63
|
+
"competence decays 0.97 to 0.73 over the queue",
|
|
64
|
+
frozenset({"drift"}),
|
|
65
|
+
dict(competence=0.97, drift=0.24),
|
|
66
|
+
),
|
|
67
|
+
DemoRow(
|
|
68
|
+
"leaky-split",
|
|
69
|
+
"12 test items duplicate a train item's text",
|
|
70
|
+
frozenset({"split leakage"}),
|
|
71
|
+
dict(competence=0.92, dup_across_splits=12),
|
|
72
|
+
),
|
|
73
|
+
DemoRow(
|
|
74
|
+
"stale-gold",
|
|
75
|
+
"8% of gold labels are wrong",
|
|
76
|
+
frozenset({"stale gold"}),
|
|
77
|
+
dict(competence=0.92, gold_error=0.08),
|
|
78
|
+
),
|
|
79
|
+
DemoRow(
|
|
80
|
+
"easy-but-skewed",
|
|
81
|
+
"94% of items are one class; raters 90% accurate",
|
|
82
|
+
frozenset({"reliability"}),
|
|
83
|
+
dict(prevalence=[0.94, 0.02, 0.02, 0.02], competence=0.9),
|
|
84
|
+
),
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _audit_row(row: DemoRow, seed: int) -> AuditResult:
|
|
89
|
+
ratings, gold, config = simulate(seed=seed, **row.kwargs)
|
|
90
|
+
return run_audit(ratings, gold, config)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _find_seed(row: DemoRow) -> tuple[int, AuditResult]:
|
|
94
|
+
for seed in range(MAX_SEED):
|
|
95
|
+
audit = _audit_row(row, seed)
|
|
96
|
+
if frozenset(f.name for f in audit.flags) == row.target:
|
|
97
|
+
return seed, audit
|
|
98
|
+
raise RuntimeError(f"no seed below {MAX_SEED} lands {row.name} on its target flags")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build() -> list[tuple[DemoRow, int, AuditResult]]:
|
|
102
|
+
"""Audit all demo rows, confirming each lands exactly its target flag set."""
|
|
103
|
+
out = []
|
|
104
|
+
for row in ROWS:
|
|
105
|
+
seed, audit = _find_seed(row)
|
|
106
|
+
out.append((row, seed, audit))
|
|
107
|
+
return out
|