evalnoise 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
evalnoise/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """evalnoise: where does the noise in an eval score come from, and where should the next calls go?"""
2
+
3
+ from .files import ScoreTable, read_scores
4
+ from .study import Component, Plan, Projection, Study, compare, study, to_score
5
+
6
+ __version__ = "0.1.0"
7
+
8
+ __all__ = ["Component", "Plan", "Projection", "Study", "ScoreTable", "compare", "read_scores", "study", "to_score", "__version__"]
evalnoise/_stats.py ADDED
@@ -0,0 +1,73 @@
1
+ """Small numeric helpers: the t distribution, without scipy."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+
7
+
8
+ def _beta_fraction(a: float, b: float, x: float) -> float:
9
+ """Continued fraction for the regularized incomplete beta function (modified Lentz)."""
10
+ tiny = 1e-300
11
+ c, d = 1.0, 1.0 - (a + b) * x / (a + 1.0)
12
+ d = 1.0 / (d if abs(d) > tiny else tiny)
13
+ result = d
14
+ for m in range(1, 500):
15
+ m2 = 2 * m
16
+ for numerator in (
17
+ m * (b - m) * x / ((a + m2 - 1.0) * (a + m2)),
18
+ -(a + m) * (a + b + m) * x / ((a + m2) * (a + m2 + 1.0)),
19
+ ):
20
+ d = 1.0 + numerator * d
21
+ d = 1.0 / (d if abs(d) > tiny else tiny)
22
+ c = 1.0 + numerator / c
23
+ c = c if abs(c) > tiny else tiny
24
+ result *= d * c
25
+ if abs(d * c - 1.0) < 1e-14:
26
+ break
27
+ return result
28
+
29
+
30
+ def incomplete_beta(a: float, b: float, x: float) -> float:
31
+ """Regularized incomplete beta function I_x(a, b)."""
32
+ if x <= 0:
33
+ return 0.0
34
+ if x >= 1:
35
+ return 1.0
36
+ log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)
37
+ if x < (a + 1.0) / (a + b + 2.0):
38
+ return math.exp(log_front) * _beta_fraction(a, b, x) / a
39
+ return 1.0 - math.exp(log_front) * _beta_fraction(b, a, 1.0 - x) / b
40
+
41
+
42
+ def t_cdf(t: float, df: float) -> float:
43
+ """Chance that a t variable with ``df`` degrees of freedom is at most ``t``.
44
+
45
+ >>> round(t_cdf(2.228, 10), 3)
46
+ 0.975
47
+ """
48
+ tail = 0.5 * incomplete_beta(df / 2.0, 0.5, df / (df + t * t))
49
+ return 1.0 - tail if t > 0 else tail
50
+
51
+
52
+ def t_quantile(p: float, df: float) -> float:
53
+ """The value a t variable with ``df`` degrees of freedom stays below with chance ``p``.
54
+
55
+ >>> round(t_quantile(0.975, 1), 2), round(t_quantile(0.975, 10), 3), round(t_quantile(0.975, 1e9), 3)
56
+ (12.71, 2.228, 1.96)
57
+ """
58
+ if not 0 < p < 1:
59
+ raise ValueError("p must be between 0 and 1")
60
+ if p < 0.5:
61
+ return -t_quantile(1 - p, df)
62
+ low, high = 0.0, 1.0
63
+ while t_cdf(high, df) < p:
64
+ high *= 2
65
+ for _ in range(200):
66
+ mid = (low + high) / 2
67
+ if t_cdf(mid, df) < p:
68
+ low = mid
69
+ else:
70
+ high = mid
71
+ if high - low < 1e-10:
72
+ break
73
+ return (low + high) / 2
evalnoise/cli.py ADDED
@@ -0,0 +1,118 @@
1
+ """Command line: ``evalnoise check``, ``evalnoise plan`` and ``evalnoise project``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import sys
7
+ from typing import List, Optional
8
+
9
+ from .files import read_scores
10
+
11
+
12
+ def _study(args: argparse.Namespace):
13
+ table = read_scores(args.file, item=args.item, score=args.score, facets=args.facets, system=args.system)
14
+ if args.verbose:
15
+ print(table.report(), file=sys.stderr)
16
+ nested = [] if args.runs_crossed else None
17
+ if table.system is not None and len(table.systems) > 1 and args.only is None:
18
+ return table.compare(args.new, args.old, fixed=args.fixed, judging=args.judging, nested=nested)
19
+ return table.study(fixed=args.fixed, judging=args.judging, system=args.only, nested=nested)
20
+
21
+
22
+ def _cmd_check(args: argparse.Namespace) -> str:
23
+ return _study(args).report(level=args.level)
24
+
25
+
26
+ def _limits(pairs: List[str]) -> dict:
27
+ limits = {}
28
+ for pair in pairs:
29
+ name, _, value = pair.partition("=")
30
+ try:
31
+ limits[name.strip()] = int(value)
32
+ except ValueError:
33
+ raise ValueError(f"--limit takes name=number, like judge=3 (got {pair!r})") from None
34
+ return limits
35
+
36
+
37
+ def _cmd_plan(args: argparse.Namespace) -> str:
38
+ if (args.budget is None) == (args.target is None):
39
+ raise ValueError("give either --budget or --target")
40
+ found = _study(args)
41
+ plan = found.plan(
42
+ budget=args.budget,
43
+ target=args.target,
44
+ generation_cost=args.model_cost,
45
+ judge_cost=args.judge_cost,
46
+ max_items=args.max_items,
47
+ max_per_facet=args.max_per_facet,
48
+ limits=_limits(args.limit),
49
+ level=args.level,
50
+ )
51
+ return found.report(level=args.level) + "\n\n" + plan.summary()
52
+
53
+
54
+ def _cmd_project(args: argparse.Namespace) -> str:
55
+ found = _study(args)
56
+ sizes = _limits(args.design) if args.design else {}
57
+ return found.project(level=args.level, **sizes).summary()
58
+
59
+
60
+ def build_parser() -> argparse.ArgumentParser:
61
+ parser = argparse.ArgumentParser(
62
+ prog="evalnoise",
63
+ description="Where does the noise in an eval score come from, and where should the next calls go?",
64
+ )
65
+ sub = parser.add_subparsers(dest="command", required=True)
66
+
67
+ def common(p: argparse.ArgumentParser) -> None:
68
+ p.add_argument("file", help="CSV, JSON or JSON Lines with one row per score")
69
+ p.add_argument("--item", help="column naming the question (found automatically if left out)")
70
+ p.add_argument("--score", help="column with the score (found automatically if left out)")
71
+ p.add_argument("--facets", nargs="+", help="columns to count as sources of noise, like judge prompt run")
72
+ p.add_argument("--system", help="column naming the model or version, to compare two of them")
73
+ p.add_argument("--new", help="the new system, when comparing")
74
+ p.add_argument("--old", help="the old system, when comparing")
75
+ p.add_argument("--only", help="analyse just this one system, when the file has several")
76
+ p.add_argument("--fixed", nargs="+", default=(), help="facets that are not a sample, like a judge you will always use")
77
+ p.add_argument("--judging", nargs="+", help="facets that cost a judge call (found by name if left out)")
78
+ p.add_argument(
79
+ "--runs-crossed", action="store_true",
80
+ help="run 1 means the same thing for every question (for example one batch per run)",
81
+ )
82
+ p.add_argument("--level", type=float, default=0.95, help="confidence level (default 0.95)")
83
+ p.add_argument("-v", "--verbose", action="store_true", help="say which columns were used")
84
+
85
+ p = sub.add_parser("check", help="split the noise in a score, or in the difference between two systems")
86
+ common(p)
87
+ p.set_defaults(run=_cmd_check)
88
+
89
+ p = sub.add_parser("plan", help="the best design for a budget, or the cheapest for a target")
90
+ common(p)
91
+ p.add_argument("--budget", type=float, help="total to spend")
92
+ p.add_argument("--target", type=float, help="wanted half width of the interval, like 0.02")
93
+ p.add_argument("--model-cost", type=float, default=1.0, help="cost of one call to the model being tested")
94
+ p.add_argument("--judge-cost", type=float, default=0.0, help="cost of one judge call (default 0)")
95
+ p.add_argument("--max-items", type=int, help="how many questions exist")
96
+ p.add_argument("--max-per-facet", type=int, default=5, help="most judges, wordings or runs to consider (default 5)")
97
+ p.add_argument("--limit", nargs="+", default=[], metavar="NAME=N", help="per-facet maximums, like judge=3")
98
+ p.set_defaults(run=_cmd_plan)
99
+
100
+ p = sub.add_parser("project", help="expected noise for another design, like items=500 judges=1")
101
+ common(p)
102
+ p.add_argument("design", nargs="*", metavar="NAME=N", help="sizes to change, like items=500 judges=1")
103
+ p.set_defaults(run=_cmd_project)
104
+ return parser
105
+
106
+
107
+ def main(argv: Optional[List[str]] = None) -> int:
108
+ args = build_parser().parse_args(argv)
109
+ try:
110
+ print(args.run(args))
111
+ except (ValueError, OSError) as exc:
112
+ print(f"evalnoise: {exc}", file=sys.stderr)
113
+ return 2
114
+ return 0
115
+
116
+
117
+ if __name__ == "__main__":
118
+ sys.exit(main())
evalnoise/files.py ADDED
@@ -0,0 +1,219 @@
1
+ """Read eval results from CSV, JSON or JSON Lines, finding the columns automatically."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import csv
6
+ import json
7
+ import os
8
+ import re
9
+ from dataclasses import dataclass, field
10
+ from typing import Dict, Iterable, List, Optional, Sequence
11
+
12
+ from .study import Study, _build, _build_difference, _kind, pick_pair, to_score
13
+
14
+ ITEM_NAMES = (
15
+ "item", "itemid", "question", "questionid", "example", "exampleid", "case", "caseid", "testcase",
16
+ "testcaseid", "task", "taskid", "problem", "problemid", "qid", "id", "inputid", "datasetitemid",
17
+ )
18
+ SCORE_NAMES = (
19
+ "score", "judgescore", "grade", "rating", "verdict", "judgeverdict", "passed", "pass", "correct",
20
+ "iscorrect", "result", "outcome", "label", "success", "value",
21
+ )
22
+ SYSTEM_NAMES = ("system", "model", "modelname", "version", "arm", "group", "candidate", "setup")
23
+
24
+
25
+ def _norm(name: str) -> str:
26
+ return re.sub(r"[^a-z0-9]", "", name.lower())
27
+
28
+
29
+ def _flatten(record: dict, prefix: str = "") -> dict:
30
+ flat = {}
31
+ for key, value in record.items():
32
+ name = f"{prefix}{key}"
33
+ if isinstance(value, dict):
34
+ flat.update(_flatten(value, name + "."))
35
+ else:
36
+ flat[name] = value
37
+ return flat
38
+
39
+
40
+ def _read_records(path: str) -> List[dict]:
41
+ path = os.fspath(path)
42
+ lower = path.lower()
43
+ with open(path, encoding="utf-8-sig") as handle:
44
+ if lower.endswith(".csv"):
45
+ return list(csv.DictReader(handle))
46
+ if lower.endswith(".tsv"):
47
+ return list(csv.DictReader(handle, delimiter="\t"))
48
+ text = handle.read()
49
+ stripped = text.lstrip()
50
+ if lower.endswith((".jsonl", ".ndjson")) or (stripped.startswith("{") and "\n{" in stripped):
51
+ records = [json.loads(line) for line in text.splitlines() if line.strip()]
52
+ else:
53
+ data = json.loads(text)
54
+ if isinstance(data, dict):
55
+ data = next((v for v in data.values() if isinstance(v, list)), [data])
56
+ records = data
57
+ if not all(isinstance(r, dict) for r in records):
58
+ raise ValueError(f"{path}: expected one object per row")
59
+ return [_flatten(r) for r in records]
60
+
61
+
62
+ def _find(columns: Sequence[str], names: Sequence[str], taken: Iterable[str] = ()) -> Optional[str]:
63
+ taken = set(taken)
64
+ by_norm: Dict[str, str] = {}
65
+ for c in columns:
66
+ if c not in taken:
67
+ by_norm.setdefault(_norm(c.split(".")[-1]), c)
68
+ by_norm.setdefault(_norm(c), c)
69
+ for name in names:
70
+ if name in by_norm:
71
+ return by_norm[name]
72
+ return None
73
+
74
+
75
+ @dataclass
76
+ class ScoreTable:
77
+ """Scores read from a file, with the columns that were used.
78
+
79
+ Use :meth:`study` for one system, or :meth:`compare` for two.
80
+ """
81
+
82
+ rows: List[dict]
83
+ item: str
84
+ score: str
85
+ facets: List[str]
86
+ system: Optional[str] = None
87
+ ignored: List[str] = field(default_factory=list)
88
+ path: str = ""
89
+ notes: List[str] = field(default_factory=list)
90
+
91
+ @property
92
+ def systems(self) -> List[str]:
93
+ if self.system is None:
94
+ return []
95
+ return list(dict.fromkeys(str(r[self.system]) for r in self.rows))
96
+
97
+ def report(self) -> str:
98
+ parts = [f"question from {self.item!r}", f"score from {self.score!r}"]
99
+ parts += [f"{_kind(f) if _kind(f) != 'other' else 'facet'} from {f!r}" for f in self.facets]
100
+ if self.system:
101
+ parts.append(f"system from {self.system!r} ({', '.join(self.systems)})")
102
+ lines = [f"Read {len(self.rows):,} rows: " + ", ".join(parts) + "."]
103
+ if self.ignored:
104
+ lines.append(f"Not used: {', '.join(self.ignored)}. Name them with facets=[...] to count them as sources of noise.")
105
+ lines.extend(f"Note: {n}." for n in self.notes)
106
+ return "\n".join(lines)
107
+
108
+ def _records(self, rows):
109
+ for line, r in enumerate(rows, start=2):
110
+ try:
111
+ value = to_score(r[self.score])
112
+ except ValueError as exc:
113
+ raise ValueError(f"{self.path}, row {line}: {exc}") from None
114
+ yield r, value
115
+
116
+ def study(
117
+ self,
118
+ fixed: Iterable[str] = (),
119
+ judging: Optional[Iterable[str]] = None,
120
+ system: Optional[str] = None,
121
+ nested: Optional[Iterable[str]] = None,
122
+ ) -> Study:
123
+ """Analyse one system. If the file has several, name one with ``system``."""
124
+ rows = self.rows
125
+ if self.system is not None and len(self.systems) > 1:
126
+ if system is None:
127
+ raise ValueError(
128
+ f"the file has several systems ({', '.join(self.systems)}): pick one with system=..., "
129
+ "or compare two with compare()"
130
+ )
131
+ rows = [r for r in rows if str(r[self.system]) == str(system)]
132
+ if not rows:
133
+ raise ValueError(f"no rows for system {system!r}; the systems are {self.systems}")
134
+ return _build(
135
+ ((r[self.item], tuple(r[f] for f in self.facets), v) for r, v in self._records(rows)),
136
+ self.item, self.facets, fixed, judging, nested=nested,
137
+ )
138
+
139
+ def compare(
140
+ self,
141
+ new: Optional[str] = None,
142
+ old: Optional[str] = None,
143
+ fixed: Iterable[str] = (),
144
+ judging: Optional[Iterable[str]] = None,
145
+ nested: Optional[Iterable[str]] = None,
146
+ ) -> Study:
147
+ """Analyse the difference between two systems scored on the same questions, new minus old."""
148
+ if self.system is None or len(self.systems) < 2:
149
+ raise ValueError("comparing needs a column naming the system (model or version) with at least two values")
150
+ new, old = pick_pair(self.systems, new, old)
151
+ return _build_difference(
152
+ ((str(r[self.system]), r[self.item], tuple(r[f] for f in self.facets), v) for r, v in self._records(self.rows)),
153
+ new, old, self.item, self.facets, fixed, judging, nested,
154
+ )
155
+
156
+
157
+ def read_scores(
158
+ path,
159
+ item: Optional[str] = None,
160
+ score: Optional[str] = None,
161
+ facets: Optional[Sequence[str]] = None,
162
+ system: Optional[str] = None,
163
+ ) -> ScoreTable:
164
+ """Read eval scores from a CSV, JSON or JSON Lines file.
165
+
166
+ One row per score, with columns for the question, the score, and the
167
+ facets that varied: judge, prompt wording, run, and so on. Columns are
168
+ found by their names when not given; nested JSON fields are named with
169
+ dots, like ``metadata.judge``.
170
+
171
+ Args:
172
+ path: the file.
173
+ item: column naming the question.
174
+ score: column with the score (numbers, true/false, pass/fail...).
175
+ facets: columns to count as sources of noise. By default, columns
176
+ whose names say judge, grader, prompt, template, run, seed, sample
177
+ and similar.
178
+ system: column naming the model or version, for comparisons.
179
+ """
180
+ records = _read_records(path)
181
+ if not records:
182
+ raise ValueError(f"{path} has no rows")
183
+ columns = list(dict.fromkeys(c for r in records for c in r))
184
+ for given in (item, score, system, *(facets or [])):
185
+ if given is not None and given not in columns:
186
+ raise ValueError(f"{path} has no column {given!r}; the columns are {columns}")
187
+ item = item or _find(columns, ITEM_NAMES)
188
+ if item is None:
189
+ raise ValueError(f"{path}: no column naming the question (like item, question or id); name it with item=...")
190
+ score = score or _find(columns, SCORE_NAMES, taken=[item])
191
+ if score is None:
192
+ score = next((c for c in columns if _norm(c).endswith("score") and c != item), None)
193
+ if score is None:
194
+ raise ValueError(f"{path}: no score column (like score, verdict or passed); name it with score=...")
195
+ if system is None:
196
+ system = _find(columns, SYSTEM_NAMES, taken=[item, score])
197
+ if system is not None and len({str(r.get(system)) for r in records}) < 2:
198
+ system = None
199
+ taken = {item, score, system}
200
+ notes = []
201
+ if facets is None:
202
+ questions = len({str(r.get(item)) for r in records})
203
+ facets = []
204
+ for c in columns:
205
+ if c in taken or _kind(c.split(".")[-1]) == "other":
206
+ continue
207
+ distinct = len({str(r.get(c)) for r in records})
208
+ if distinct > 20 or (questions >= 10 and distinct >= questions):
209
+ notes.append(f"{c!r} has {distinct:,} different values, so it looks like text or an id, not a facet")
210
+ else:
211
+ facets.append(c)
212
+ facets = list(facets)
213
+ ignored = [c for c in columns if c not in taken and c not in facets]
214
+ needed = [c for c in [item, score, system, *facets] if c]
215
+ for line, r in enumerate(records, start=2):
216
+ missing = [c for c in needed if r.get(c) in (None, "")]
217
+ if missing:
218
+ raise ValueError(f"{path}, row {line}: no value for {', '.join(missing)}")
219
+ return ScoreTable(records, item, score, facets, system, ignored, os.fspath(path), notes)