evalnoise 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalnoise/__init__.py +8 -0
- evalnoise/_stats.py +73 -0
- evalnoise/cli.py +118 -0
- evalnoise/files.py +219 -0
- evalnoise/study.py +811 -0
- evalnoise-0.1.0.dist-info/METADATA +292 -0
- evalnoise-0.1.0.dist-info/RECORD +11 -0
- evalnoise-0.1.0.dist-info/WHEEL +5 -0
- evalnoise-0.1.0.dist-info/entry_points.txt +2 -0
- evalnoise-0.1.0.dist-info/licenses/LICENSE +21 -0
- evalnoise-0.1.0.dist-info/top_level.txt +1 -0
evalnoise/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""evalnoise: where does the noise in an eval score come from, and where should the next calls go?"""
|
|
2
|
+
|
|
3
|
+
from .files import ScoreTable, read_scores
|
|
4
|
+
from .study import Component, Plan, Projection, Study, compare, study, to_score
|
|
5
|
+
|
|
6
|
+
__version__ = "0.1.0"
|
|
7
|
+
|
|
8
|
+
__all__ = ["Component", "Plan", "Projection", "Study", "ScoreTable", "compare", "read_scores", "study", "to_score", "__version__"]
|
evalnoise/_stats.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Small numeric helpers: the t distribution, without scipy."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _beta_fraction(a: float, b: float, x: float) -> float:
|
|
9
|
+
"""Continued fraction for the regularized incomplete beta function (modified Lentz)."""
|
|
10
|
+
tiny = 1e-300
|
|
11
|
+
c, d = 1.0, 1.0 - (a + b) * x / (a + 1.0)
|
|
12
|
+
d = 1.0 / (d if abs(d) > tiny else tiny)
|
|
13
|
+
result = d
|
|
14
|
+
for m in range(1, 500):
|
|
15
|
+
m2 = 2 * m
|
|
16
|
+
for numerator in (
|
|
17
|
+
m * (b - m) * x / ((a + m2 - 1.0) * (a + m2)),
|
|
18
|
+
-(a + m) * (a + b + m) * x / ((a + m2) * (a + m2 + 1.0)),
|
|
19
|
+
):
|
|
20
|
+
d = 1.0 + numerator * d
|
|
21
|
+
d = 1.0 / (d if abs(d) > tiny else tiny)
|
|
22
|
+
c = 1.0 + numerator / c
|
|
23
|
+
c = c if abs(c) > tiny else tiny
|
|
24
|
+
result *= d * c
|
|
25
|
+
if abs(d * c - 1.0) < 1e-14:
|
|
26
|
+
break
|
|
27
|
+
return result
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def incomplete_beta(a: float, b: float, x: float) -> float:
|
|
31
|
+
"""Regularized incomplete beta function I_x(a, b)."""
|
|
32
|
+
if x <= 0:
|
|
33
|
+
return 0.0
|
|
34
|
+
if x >= 1:
|
|
35
|
+
return 1.0
|
|
36
|
+
log_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)
|
|
37
|
+
if x < (a + 1.0) / (a + b + 2.0):
|
|
38
|
+
return math.exp(log_front) * _beta_fraction(a, b, x) / a
|
|
39
|
+
return 1.0 - math.exp(log_front) * _beta_fraction(b, a, 1.0 - x) / b
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def t_cdf(t: float, df: float) -> float:
|
|
43
|
+
"""Chance that a t variable with ``df`` degrees of freedom is at most ``t``.
|
|
44
|
+
|
|
45
|
+
>>> round(t_cdf(2.228, 10), 3)
|
|
46
|
+
0.975
|
|
47
|
+
"""
|
|
48
|
+
tail = 0.5 * incomplete_beta(df / 2.0, 0.5, df / (df + t * t))
|
|
49
|
+
return 1.0 - tail if t > 0 else tail
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def t_quantile(p: float, df: float) -> float:
|
|
53
|
+
"""The value a t variable with ``df`` degrees of freedom stays below with chance ``p``.
|
|
54
|
+
|
|
55
|
+
>>> round(t_quantile(0.975, 1), 2), round(t_quantile(0.975, 10), 3), round(t_quantile(0.975, 1e9), 3)
|
|
56
|
+
(12.71, 2.228, 1.96)
|
|
57
|
+
"""
|
|
58
|
+
if not 0 < p < 1:
|
|
59
|
+
raise ValueError("p must be between 0 and 1")
|
|
60
|
+
if p < 0.5:
|
|
61
|
+
return -t_quantile(1 - p, df)
|
|
62
|
+
low, high = 0.0, 1.0
|
|
63
|
+
while t_cdf(high, df) < p:
|
|
64
|
+
high *= 2
|
|
65
|
+
for _ in range(200):
|
|
66
|
+
mid = (low + high) / 2
|
|
67
|
+
if t_cdf(mid, df) < p:
|
|
68
|
+
low = mid
|
|
69
|
+
else:
|
|
70
|
+
high = mid
|
|
71
|
+
if high - low < 1e-10:
|
|
72
|
+
break
|
|
73
|
+
return (low + high) / 2
|
evalnoise/cli.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Command line: ``evalnoise check``, ``evalnoise plan`` and ``evalnoise project``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from .files import read_scores
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _study(args: argparse.Namespace):
|
|
13
|
+
table = read_scores(args.file, item=args.item, score=args.score, facets=args.facets, system=args.system)
|
|
14
|
+
if args.verbose:
|
|
15
|
+
print(table.report(), file=sys.stderr)
|
|
16
|
+
nested = [] if args.runs_crossed else None
|
|
17
|
+
if table.system is not None and len(table.systems) > 1 and args.only is None:
|
|
18
|
+
return table.compare(args.new, args.old, fixed=args.fixed, judging=args.judging, nested=nested)
|
|
19
|
+
return table.study(fixed=args.fixed, judging=args.judging, system=args.only, nested=nested)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _cmd_check(args: argparse.Namespace) -> str:
|
|
23
|
+
return _study(args).report(level=args.level)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _limits(pairs: List[str]) -> dict:
|
|
27
|
+
limits = {}
|
|
28
|
+
for pair in pairs:
|
|
29
|
+
name, _, value = pair.partition("=")
|
|
30
|
+
try:
|
|
31
|
+
limits[name.strip()] = int(value)
|
|
32
|
+
except ValueError:
|
|
33
|
+
raise ValueError(f"--limit takes name=number, like judge=3 (got {pair!r})") from None
|
|
34
|
+
return limits
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _cmd_plan(args: argparse.Namespace) -> str:
|
|
38
|
+
if (args.budget is None) == (args.target is None):
|
|
39
|
+
raise ValueError("give either --budget or --target")
|
|
40
|
+
found = _study(args)
|
|
41
|
+
plan = found.plan(
|
|
42
|
+
budget=args.budget,
|
|
43
|
+
target=args.target,
|
|
44
|
+
generation_cost=args.model_cost,
|
|
45
|
+
judge_cost=args.judge_cost,
|
|
46
|
+
max_items=args.max_items,
|
|
47
|
+
max_per_facet=args.max_per_facet,
|
|
48
|
+
limits=_limits(args.limit),
|
|
49
|
+
level=args.level,
|
|
50
|
+
)
|
|
51
|
+
return found.report(level=args.level) + "\n\n" + plan.summary()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _cmd_project(args: argparse.Namespace) -> str:
|
|
55
|
+
found = _study(args)
|
|
56
|
+
sizes = _limits(args.design) if args.design else {}
|
|
57
|
+
return found.project(level=args.level, **sizes).summary()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
61
|
+
parser = argparse.ArgumentParser(
|
|
62
|
+
prog="evalnoise",
|
|
63
|
+
description="Where does the noise in an eval score come from, and where should the next calls go?",
|
|
64
|
+
)
|
|
65
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
66
|
+
|
|
67
|
+
def common(p: argparse.ArgumentParser) -> None:
|
|
68
|
+
p.add_argument("file", help="CSV, JSON or JSON Lines with one row per score")
|
|
69
|
+
p.add_argument("--item", help="column naming the question (found automatically if left out)")
|
|
70
|
+
p.add_argument("--score", help="column with the score (found automatically if left out)")
|
|
71
|
+
p.add_argument("--facets", nargs="+", help="columns to count as sources of noise, like judge prompt run")
|
|
72
|
+
p.add_argument("--system", help="column naming the model or version, to compare two of them")
|
|
73
|
+
p.add_argument("--new", help="the new system, when comparing")
|
|
74
|
+
p.add_argument("--old", help="the old system, when comparing")
|
|
75
|
+
p.add_argument("--only", help="analyse just this one system, when the file has several")
|
|
76
|
+
p.add_argument("--fixed", nargs="+", default=(), help="facets that are not a sample, like a judge you will always use")
|
|
77
|
+
p.add_argument("--judging", nargs="+", help="facets that cost a judge call (found by name if left out)")
|
|
78
|
+
p.add_argument(
|
|
79
|
+
"--runs-crossed", action="store_true",
|
|
80
|
+
help="run 1 means the same thing for every question (for example one batch per run)",
|
|
81
|
+
)
|
|
82
|
+
p.add_argument("--level", type=float, default=0.95, help="confidence level (default 0.95)")
|
|
83
|
+
p.add_argument("-v", "--verbose", action="store_true", help="say which columns were used")
|
|
84
|
+
|
|
85
|
+
p = sub.add_parser("check", help="split the noise in a score, or in the difference between two systems")
|
|
86
|
+
common(p)
|
|
87
|
+
p.set_defaults(run=_cmd_check)
|
|
88
|
+
|
|
89
|
+
p = sub.add_parser("plan", help="the best design for a budget, or the cheapest for a target")
|
|
90
|
+
common(p)
|
|
91
|
+
p.add_argument("--budget", type=float, help="total to spend")
|
|
92
|
+
p.add_argument("--target", type=float, help="wanted half width of the interval, like 0.02")
|
|
93
|
+
p.add_argument("--model-cost", type=float, default=1.0, help="cost of one call to the model being tested")
|
|
94
|
+
p.add_argument("--judge-cost", type=float, default=0.0, help="cost of one judge call (default 0)")
|
|
95
|
+
p.add_argument("--max-items", type=int, help="how many questions exist")
|
|
96
|
+
p.add_argument("--max-per-facet", type=int, default=5, help="most judges, wordings or runs to consider (default 5)")
|
|
97
|
+
p.add_argument("--limit", nargs="+", default=[], metavar="NAME=N", help="per-facet maximums, like judge=3")
|
|
98
|
+
p.set_defaults(run=_cmd_plan)
|
|
99
|
+
|
|
100
|
+
p = sub.add_parser("project", help="expected noise for another design, like items=500 judges=1")
|
|
101
|
+
common(p)
|
|
102
|
+
p.add_argument("design", nargs="*", metavar="NAME=N", help="sizes to change, like items=500 judges=1")
|
|
103
|
+
p.set_defaults(run=_cmd_project)
|
|
104
|
+
return parser
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
108
|
+
args = build_parser().parse_args(argv)
|
|
109
|
+
try:
|
|
110
|
+
print(args.run(args))
|
|
111
|
+
except (ValueError, OSError) as exc:
|
|
112
|
+
print(f"evalnoise: {exc}", file=sys.stderr)
|
|
113
|
+
return 2
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == "__main__":
|
|
118
|
+
sys.exit(main())
|
evalnoise/files.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Read eval results from CSV, JSON or JSON Lines, finding the columns automatically."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Dict, Iterable, List, Optional, Sequence
|
|
11
|
+
|
|
12
|
+
from .study import Study, _build, _build_difference, _kind, pick_pair, to_score
|
|
13
|
+
|
|
14
|
+
ITEM_NAMES = (
|
|
15
|
+
"item", "itemid", "question", "questionid", "example", "exampleid", "case", "caseid", "testcase",
|
|
16
|
+
"testcaseid", "task", "taskid", "problem", "problemid", "qid", "id", "inputid", "datasetitemid",
|
|
17
|
+
)
|
|
18
|
+
SCORE_NAMES = (
|
|
19
|
+
"score", "judgescore", "grade", "rating", "verdict", "judgeverdict", "passed", "pass", "correct",
|
|
20
|
+
"iscorrect", "result", "outcome", "label", "success", "value",
|
|
21
|
+
)
|
|
22
|
+
SYSTEM_NAMES = ("system", "model", "modelname", "version", "arm", "group", "candidate", "setup")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _norm(name: str) -> str:
|
|
26
|
+
return re.sub(r"[^a-z0-9]", "", name.lower())
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _flatten(record: dict, prefix: str = "") -> dict:
|
|
30
|
+
flat = {}
|
|
31
|
+
for key, value in record.items():
|
|
32
|
+
name = f"{prefix}{key}"
|
|
33
|
+
if isinstance(value, dict):
|
|
34
|
+
flat.update(_flatten(value, name + "."))
|
|
35
|
+
else:
|
|
36
|
+
flat[name] = value
|
|
37
|
+
return flat
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _read_records(path: str) -> List[dict]:
|
|
41
|
+
path = os.fspath(path)
|
|
42
|
+
lower = path.lower()
|
|
43
|
+
with open(path, encoding="utf-8-sig") as handle:
|
|
44
|
+
if lower.endswith(".csv"):
|
|
45
|
+
return list(csv.DictReader(handle))
|
|
46
|
+
if lower.endswith(".tsv"):
|
|
47
|
+
return list(csv.DictReader(handle, delimiter="\t"))
|
|
48
|
+
text = handle.read()
|
|
49
|
+
stripped = text.lstrip()
|
|
50
|
+
if lower.endswith((".jsonl", ".ndjson")) or (stripped.startswith("{") and "\n{" in stripped):
|
|
51
|
+
records = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
52
|
+
else:
|
|
53
|
+
data = json.loads(text)
|
|
54
|
+
if isinstance(data, dict):
|
|
55
|
+
data = next((v for v in data.values() if isinstance(v, list)), [data])
|
|
56
|
+
records = data
|
|
57
|
+
if not all(isinstance(r, dict) for r in records):
|
|
58
|
+
raise ValueError(f"{path}: expected one object per row")
|
|
59
|
+
return [_flatten(r) for r in records]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _find(columns: Sequence[str], names: Sequence[str], taken: Iterable[str] = ()) -> Optional[str]:
|
|
63
|
+
taken = set(taken)
|
|
64
|
+
by_norm: Dict[str, str] = {}
|
|
65
|
+
for c in columns:
|
|
66
|
+
if c not in taken:
|
|
67
|
+
by_norm.setdefault(_norm(c.split(".")[-1]), c)
|
|
68
|
+
by_norm.setdefault(_norm(c), c)
|
|
69
|
+
for name in names:
|
|
70
|
+
if name in by_norm:
|
|
71
|
+
return by_norm[name]
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class ScoreTable:
|
|
77
|
+
"""Scores read from a file, with the columns that were used.
|
|
78
|
+
|
|
79
|
+
Use :meth:`study` for one system, or :meth:`compare` for two.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
rows: List[dict]
|
|
83
|
+
item: str
|
|
84
|
+
score: str
|
|
85
|
+
facets: List[str]
|
|
86
|
+
system: Optional[str] = None
|
|
87
|
+
ignored: List[str] = field(default_factory=list)
|
|
88
|
+
path: str = ""
|
|
89
|
+
notes: List[str] = field(default_factory=list)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def systems(self) -> List[str]:
|
|
93
|
+
if self.system is None:
|
|
94
|
+
return []
|
|
95
|
+
return list(dict.fromkeys(str(r[self.system]) for r in self.rows))
|
|
96
|
+
|
|
97
|
+
def report(self) -> str:
|
|
98
|
+
parts = [f"question from {self.item!r}", f"score from {self.score!r}"]
|
|
99
|
+
parts += [f"{_kind(f) if _kind(f) != 'other' else 'facet'} from {f!r}" for f in self.facets]
|
|
100
|
+
if self.system:
|
|
101
|
+
parts.append(f"system from {self.system!r} ({', '.join(self.systems)})")
|
|
102
|
+
lines = [f"Read {len(self.rows):,} rows: " + ", ".join(parts) + "."]
|
|
103
|
+
if self.ignored:
|
|
104
|
+
lines.append(f"Not used: {', '.join(self.ignored)}. Name them with facets=[...] to count them as sources of noise.")
|
|
105
|
+
lines.extend(f"Note: {n}." for n in self.notes)
|
|
106
|
+
return "\n".join(lines)
|
|
107
|
+
|
|
108
|
+
def _records(self, rows):
|
|
109
|
+
for line, r in enumerate(rows, start=2):
|
|
110
|
+
try:
|
|
111
|
+
value = to_score(r[self.score])
|
|
112
|
+
except ValueError as exc:
|
|
113
|
+
raise ValueError(f"{self.path}, row {line}: {exc}") from None
|
|
114
|
+
yield r, value
|
|
115
|
+
|
|
116
|
+
def study(
|
|
117
|
+
self,
|
|
118
|
+
fixed: Iterable[str] = (),
|
|
119
|
+
judging: Optional[Iterable[str]] = None,
|
|
120
|
+
system: Optional[str] = None,
|
|
121
|
+
nested: Optional[Iterable[str]] = None,
|
|
122
|
+
) -> Study:
|
|
123
|
+
"""Analyse one system. If the file has several, name one with ``system``."""
|
|
124
|
+
rows = self.rows
|
|
125
|
+
if self.system is not None and len(self.systems) > 1:
|
|
126
|
+
if system is None:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
f"the file has several systems ({', '.join(self.systems)}): pick one with system=..., "
|
|
129
|
+
"or compare two with compare()"
|
|
130
|
+
)
|
|
131
|
+
rows = [r for r in rows if str(r[self.system]) == str(system)]
|
|
132
|
+
if not rows:
|
|
133
|
+
raise ValueError(f"no rows for system {system!r}; the systems are {self.systems}")
|
|
134
|
+
return _build(
|
|
135
|
+
((r[self.item], tuple(r[f] for f in self.facets), v) for r, v in self._records(rows)),
|
|
136
|
+
self.item, self.facets, fixed, judging, nested=nested,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def compare(
|
|
140
|
+
self,
|
|
141
|
+
new: Optional[str] = None,
|
|
142
|
+
old: Optional[str] = None,
|
|
143
|
+
fixed: Iterable[str] = (),
|
|
144
|
+
judging: Optional[Iterable[str]] = None,
|
|
145
|
+
nested: Optional[Iterable[str]] = None,
|
|
146
|
+
) -> Study:
|
|
147
|
+
"""Analyse the difference between two systems scored on the same questions, new minus old."""
|
|
148
|
+
if self.system is None or len(self.systems) < 2:
|
|
149
|
+
raise ValueError("comparing needs a column naming the system (model or version) with at least two values")
|
|
150
|
+
new, old = pick_pair(self.systems, new, old)
|
|
151
|
+
return _build_difference(
|
|
152
|
+
((str(r[self.system]), r[self.item], tuple(r[f] for f in self.facets), v) for r, v in self._records(self.rows)),
|
|
153
|
+
new, old, self.item, self.facets, fixed, judging, nested,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def read_scores(
|
|
158
|
+
path,
|
|
159
|
+
item: Optional[str] = None,
|
|
160
|
+
score: Optional[str] = None,
|
|
161
|
+
facets: Optional[Sequence[str]] = None,
|
|
162
|
+
system: Optional[str] = None,
|
|
163
|
+
) -> ScoreTable:
|
|
164
|
+
"""Read eval scores from a CSV, JSON or JSON Lines file.
|
|
165
|
+
|
|
166
|
+
One row per score, with columns for the question, the score, and the
|
|
167
|
+
facets that varied: judge, prompt wording, run, and so on. Columns are
|
|
168
|
+
found by their names when not given; nested JSON fields are named with
|
|
169
|
+
dots, like ``metadata.judge``.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
path: the file.
|
|
173
|
+
item: column naming the question.
|
|
174
|
+
score: column with the score (numbers, true/false, pass/fail...).
|
|
175
|
+
facets: columns to count as sources of noise. By default, columns
|
|
176
|
+
whose names say judge, grader, prompt, template, run, seed, sample
|
|
177
|
+
and similar.
|
|
178
|
+
system: column naming the model or version, for comparisons.
|
|
179
|
+
"""
|
|
180
|
+
records = _read_records(path)
|
|
181
|
+
if not records:
|
|
182
|
+
raise ValueError(f"{path} has no rows")
|
|
183
|
+
columns = list(dict.fromkeys(c for r in records for c in r))
|
|
184
|
+
for given in (item, score, system, *(facets or [])):
|
|
185
|
+
if given is not None and given not in columns:
|
|
186
|
+
raise ValueError(f"{path} has no column {given!r}; the columns are {columns}")
|
|
187
|
+
item = item or _find(columns, ITEM_NAMES)
|
|
188
|
+
if item is None:
|
|
189
|
+
raise ValueError(f"{path}: no column naming the question (like item, question or id); name it with item=...")
|
|
190
|
+
score = score or _find(columns, SCORE_NAMES, taken=[item])
|
|
191
|
+
if score is None:
|
|
192
|
+
score = next((c for c in columns if _norm(c).endswith("score") and c != item), None)
|
|
193
|
+
if score is None:
|
|
194
|
+
raise ValueError(f"{path}: no score column (like score, verdict or passed); name it with score=...")
|
|
195
|
+
if system is None:
|
|
196
|
+
system = _find(columns, SYSTEM_NAMES, taken=[item, score])
|
|
197
|
+
if system is not None and len({str(r.get(system)) for r in records}) < 2:
|
|
198
|
+
system = None
|
|
199
|
+
taken = {item, score, system}
|
|
200
|
+
notes = []
|
|
201
|
+
if facets is None:
|
|
202
|
+
questions = len({str(r.get(item)) for r in records})
|
|
203
|
+
facets = []
|
|
204
|
+
for c in columns:
|
|
205
|
+
if c in taken or _kind(c.split(".")[-1]) == "other":
|
|
206
|
+
continue
|
|
207
|
+
distinct = len({str(r.get(c)) for r in records})
|
|
208
|
+
if distinct > 20 or (questions >= 10 and distinct >= questions):
|
|
209
|
+
notes.append(f"{c!r} has {distinct:,} different values, so it looks like text or an id, not a facet")
|
|
210
|
+
else:
|
|
211
|
+
facets.append(c)
|
|
212
|
+
facets = list(facets)
|
|
213
|
+
ignored = [c for c in columns if c not in taken and c not in facets]
|
|
214
|
+
needed = [c for c in [item, score, system, *facets] if c]
|
|
215
|
+
for line, r in enumerate(records, start=2):
|
|
216
|
+
missing = [c for c in needed if r.get(c) in (None, "")]
|
|
217
|
+
if missing:
|
|
218
|
+
raise ValueError(f"{path}, row {line}: no value for {', '.join(missing)}")
|
|
219
|
+
return ScoreTable(records, item, score, facets, system, ignored, os.fspath(path), notes)
|