text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,205 @@
1
+ """Compare two runs case by case (ADR 0035, 0036, 0037).
2
+
3
+ What a researcher used to do in a spreadsheet: which cases both runs got
4
+ right, which only one got right, and how the kinds of mistakes changed.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import csv
10
+ from collections import Counter
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+ from typing import Literal
14
+
15
+ from text2sql_eval.csv_safety import excel_safe
16
+ from text2sql_eval.judge.ex import status_of
17
+ from text2sql_eval.runfile import CaseRecord, Run
18
+
19
+ Change = Literal["both_correct", "fixed", "regressed", "both_wrong"]
20
+
21
+ CHANGE_LABELS_KO: dict[Change, str] = {
22
+ "both_correct": "둘 다 맞음",
23
+ "fixed": "head만 맞음 (고쳐짐)",
24
+ "regressed": "base만 맞음 (나빠짐)",
25
+ "both_wrong": "둘 다 틀림",
26
+ }
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class CaseChange:
31
+ """One case seen in both runs."""
32
+
33
+ base: CaseRecord
34
+ head: CaseRecord
35
+
36
+ @property
37
+ def change(self) -> Change:
38
+ base_ok = status_of(self.base.verdict) == "correct"
39
+ head_ok = status_of(self.head.verdict) == "correct"
40
+ if base_ok and head_ok:
41
+ return "both_correct"
42
+ if head_ok:
43
+ return "fixed"
44
+ if base_ok:
45
+ return "regressed"
46
+ return "both_wrong"
47
+
48
+
49
+ @dataclass(frozen=True)
50
+ class GroupEx:
51
+ """EX of both runs over the common cases of one group (e.g. a difficulty)."""
52
+
53
+ name: str
54
+ cases: int
55
+ base_ex: float
56
+ head_ex: float
57
+
58
+
59
+ @dataclass(frozen=True)
60
+ class Comparison:
61
+ """Two runs compared over the cases they share, in the base run's order."""
62
+
63
+ cases: tuple[CaseChange, ...]
64
+ dataset_mismatch: bool
65
+ only_in_base: tuple[str, ...]
66
+ only_in_head: tuple[str, ...]
67
+
68
+ def counts(self) -> Counter[Change]:
69
+ return Counter(case.change for case in self.cases)
70
+
71
+ @property
72
+ def base_ex(self) -> float:
73
+ return _ex([c.base for c in self.cases])
74
+
75
+ @property
76
+ def head_ex(self) -> float:
77
+ return _ex([c.head for c in self.cases])
78
+
79
+ @property
80
+ def delta_pp(self) -> float:
81
+ """Change in EX, in percentage points (head minus base)."""
82
+ return (self.head_ex - self.base_ex) * 100
83
+
84
+ def by(self, key: Literal["difficulty", "db_id"]) -> list[GroupEx]:
85
+ """EX per difficulty or per database, groups in name order."""
86
+ groups: dict[str, list[CaseChange]] = {}
87
+ for case in self.cases:
88
+ value = case.base.difficulty if key == "difficulty" else case.base.db_id
89
+ groups.setdefault(value or "(none)", []).append(case)
90
+ return [
91
+ GroupEx(
92
+ name=name,
93
+ cases=len(members),
94
+ base_ex=_ex([m.base for m in members]),
95
+ head_ex=_ex([m.head for m in members]),
96
+ )
97
+ for name, members in sorted(groups.items())
98
+ ]
99
+
100
+ def output_differences(self) -> tuple[Counter[str], Counter[str]]:
101
+ """Counts of each output difference kind in base and in head."""
102
+ return (
103
+ Counter(d for c in self.cases for d in _outputs(c.base)),
104
+ Counter(d for c in self.cases for d in _outputs(c.head)),
105
+ )
106
+
107
+ def structure_differences(self) -> tuple[Counter[str], Counter[str]]:
108
+ """Counts of each SQL structure difference kind in base and in head."""
109
+ return (
110
+ Counter(f for c in self.cases for f in _structures(c.base)),
111
+ Counter(f for c in self.cases for f in _structures(c.head)),
112
+ )
113
+
114
+
115
+ def compare_runs(base: Run, head: Run) -> Comparison:
116
+ """Pair up the cases two runs share; note what only one of them has.
117
+
118
+ A different dataset hash is reported, not refused (ADR 0037); callers show
119
+ it as a warning.
120
+
121
+ Example:
122
+ >>> comparison = compare_runs(read_run(no_evidence), read_run(evidence))
123
+ >>> comparison.counts()["fixed"], round(comparison.delta_pp, 1)
124
+ (123, 24.3)
125
+ """
126
+ head_by_id = {case.id: case for case in head.cases}
127
+ base_ids = {case.id for case in base.cases}
128
+ pairs = tuple(
129
+ CaseChange(base=case, head=head_by_id[case.id])
130
+ for case in base.cases
131
+ if case.id in head_by_id
132
+ )
133
+ return Comparison(
134
+ cases=pairs,
135
+ dataset_mismatch=base.dataset.sha256 != head.dataset.sha256,
136
+ only_in_base=tuple(c.id for c in base.cases if c.id not in head_by_id),
137
+ only_in_head=tuple(c.id for c in head.cases if c.id not in base_ids),
138
+ )
139
+
140
+
141
+ def _ex(records: list[CaseRecord]) -> float:
142
+ if not records:
143
+ return 0.0
144
+ return sum(status_of(r.verdict) == "correct" for r in records) / len(records)
145
+
146
+
147
+ def _outputs(record: CaseRecord) -> tuple[str, ...]:
148
+ return () if record.diff is None else record.diff.output.differences
149
+
150
+
151
+ def _structures(record: CaseRecord) -> tuple[str, ...]:
152
+ if record.diff is None:
153
+ return ()
154
+ return tuple(fact.kind for fact in record.diff.structure.facts)
155
+
156
+
157
+ CSV_COLUMNS = (
158
+ "id",
159
+ "db_id",
160
+ "difficulty",
161
+ "change",
162
+ "base_status",
163
+ "head_status",
164
+ "base_output_differences",
165
+ "head_output_differences",
166
+ "question",
167
+ "gold_sql",
168
+ "base_pred_sql",
169
+ "head_pred_sql",
170
+ )
171
+
172
+
173
+ def write_comparison_csv(comparison: Comparison, path: Path) -> None:
174
+ """One row per common case, for spreadsheet analysis.
175
+
176
+ Written as UTF-8 with a byte order mark so Excel shows Korean text and
177
+ other non-ASCII characters correctly. Multiple differences are joined
178
+ with ``;``. A cell that starts like a formula (``=``, ``-`` ...) gets a
179
+ leading ``'`` so Excel shows it as text (ADR 0040).
180
+
181
+ Example:
182
+ >>> comparison = compare_runs(read_run(base_path), read_run(head_path))
183
+ >>> write_comparison_csv(comparison, Path("runs/flips.csv"))
184
+ """
185
+ path.parent.mkdir(parents=True, exist_ok=True)
186
+ with path.open("w", encoding="utf-8-sig", newline="") as file:
187
+ writer = csv.writer(file)
188
+ writer.writerow(CSV_COLUMNS)
189
+ for case in comparison.cases:
190
+ base, head = case.base, case.head
191
+ cells = (
192
+ base.id,
193
+ base.db_id,
194
+ base.difficulty or "",
195
+ case.change,
196
+ status_of(base.verdict),
197
+ status_of(head.verdict),
198
+ ";".join(_outputs(base)),
199
+ ";".join(_outputs(head)),
200
+ base.question,
201
+ base.gold_sql,
202
+ base.pred_sql,
203
+ head.pred_sql,
204
+ )
205
+ writer.writerow(excel_safe(cell) for cell in cells)
@@ -0,0 +1,32 @@
1
+ """Keep spreadsheet programs from running CSV cells as formulas (ADR 0040).
2
+
3
+ Excel reads a cell that starts with ``=``, ``+``, ``-`` or ``@`` (and a few
4
+ others) as a formula, so a predicted SQL such as ``=HYPERLINK(...)`` or one
5
+ starting with a ``--`` comment would run or show ``#NAME?``. Following OWASP's
6
+ advice on CSV injection, such cells get a leading apostrophe, which Excel
7
+ hides and which makes it show the cell as text.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ FORMULA_STARTS = frozenset("=+-@\t\r\n=+-@")
13
+ """First characters that make a spreadsheet treat a cell as a formula.
14
+
15
+ The list, full-width forms included, is OWASP's (checked 2026-11-11).
16
+ """
17
+
18
+
19
+ def excel_safe(cell: str) -> str:
20
+ """``cell`` with a leading ``'`` if its first character starts a formula.
21
+
22
+ Only the first character matters; ``=`` inside a cell is left alone.
23
+
24
+ Example:
25
+ >>> excel_safe("=HYPERLINK(\\"http://example.com\\")")
26
+ '\\'=HYPERLINK("http://example.com")'
27
+ >>> excel_safe("SELECT name FROM hero WHERE id = 1")
28
+ 'SELECT name FROM hero WHERE id = 1'
29
+ """
30
+ if cell and cell[0] in FORMULA_STARTS:
31
+ return "'" + cell
32
+ return cell
@@ -0,0 +1,5 @@
1
+ """Readers for datasets and prediction files.
2
+
3
+ Not part of the public API yet; the public surface is decided once ``eval``
4
+ works end to end.
5
+ """
@@ -0,0 +1,73 @@
1
+ """Small helpers that turn ``json.loads`` output into precisely typed values."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import cast
8
+
9
+ from text2sql_eval.errors import InputError
10
+
11
+
12
+ def read_utf8(path: Path) -> str:
13
+ """Read a whole text file as UTF-8, turning OS and decoding errors into InputError.
14
+
15
+ A leading byte order mark is dropped: Windows editors and Excel add one, and
16
+ left in place it would break the first JSON line or prefix the first SQL.
17
+ """
18
+ try:
19
+ return path.read_text(encoding="utf-8-sig")
20
+ except FileNotFoundError:
21
+ raise InputError(path, "file not found", hint="check the path") from None
22
+ except UnicodeDecodeError as exc:
23
+ raise InputError(
24
+ path,
25
+ f"not valid UTF-8 (byte {exc.start})",
26
+ hint="save the file with UTF-8 encoding",
27
+ ) from None
28
+
29
+
30
+ def parse_json(text: str, path: Path, location: str | None = None) -> object:
31
+ """Parse JSON text, reporting syntax errors with their position."""
32
+ try:
33
+ value: object = json.loads(text)
34
+ except json.JSONDecodeError as exc:
35
+ where = location or f"line {exc.lineno}"
36
+ raise InputError(
37
+ path,
38
+ f"invalid JSON at column {exc.colno}: {exc.msg}",
39
+ location=where,
40
+ hint="check quotes, commas and brackets on this line",
41
+ ) from None
42
+ return value
43
+
44
+
45
+ def as_object(value: object) -> dict[str, object] | None:
46
+ """Return ``value`` as a JSON object, or ``None`` if it is something else."""
47
+ if not isinstance(value, dict):
48
+ return None
49
+ # JSON object keys are always strings, so this narrowing is exact.
50
+ return cast("dict[str, object]", value)
51
+
52
+
53
+ def as_array(value: object) -> list[object] | None:
54
+ """Return ``value`` as a JSON array, or ``None`` if it is something else."""
55
+ if not isinstance(value, list):
56
+ return None
57
+ # Elements are arbitrary JSON values, which ``object`` describes exactly.
58
+ return cast("list[object]", value)
59
+
60
+
61
+ def json_type_name(value: object) -> str:
62
+ """Name a JSON value's type the way users see it in the file."""
63
+ if value is None:
64
+ return "null"
65
+ if isinstance(value, bool):
66
+ return "boolean"
67
+ if isinstance(value, int | float):
68
+ return "number"
69
+ if isinstance(value, str):
70
+ return "string"
71
+ if isinstance(value, list):
72
+ return "array"
73
+ return "object"
@@ -0,0 +1,187 @@
1
+ """Load evaluation cases from a BIRD-format question file (for example ``dev.json``)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import TypeVar
8
+
9
+ from text2sql_eval.datasets._json import (
10
+ as_array,
11
+ as_object,
12
+ json_type_name,
13
+ parse_json,
14
+ read_utf8,
15
+ )
16
+ from text2sql_eval.errors import InputError
17
+
18
+ # A constrained TypeVar instead of 3.12's ``def f[T: (int, str)]`` (ADR 0009).
19
+ _Scalar = TypeVar("_Scalar", int, str)
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Case:
24
+ """One evaluation question with its gold SQL.
25
+
26
+ ``id`` is the dataset's question id as a string (BIRD's integer
27
+ ``question_id`` 718 becomes ``"718"``), so ids from any source compare the
28
+ same way.
29
+ """
30
+
31
+ id: str
32
+ db_id: str
33
+ question: str
34
+ evidence: str
35
+ gold_sql: str
36
+ difficulty: str | None
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class Dataset:
41
+ """Cases in file order. Order matters: text prediction files follow it."""
42
+
43
+ path: Path
44
+ cases: tuple[Case, ...]
45
+
46
+ @property
47
+ def ids(self) -> tuple[str, ...]:
48
+ """Case ids in file order."""
49
+ return tuple(case.id for case in self.cases)
50
+
51
+
52
+ def load_bird(path: Path) -> Dataset:
53
+ """Read a BIRD question file: a JSON array of question objects.
54
+
55
+ Each object needs ``question_id`` (integer), ``db_id``, ``question``,
56
+ ``evidence`` and ``SQL`` (strings); ``difficulty`` (string) is optional.
57
+ Other fields are ignored.
58
+
59
+ Raises:
60
+ InputError: if the file is unreadable, is not a JSON array of objects,
61
+ misses a required field, has a field of the wrong type, or repeats
62
+ a ``question_id``.
63
+
64
+ Example:
65
+ >>> dataset = load_bird(Path("examples/bird-mini/dev.json"))
66
+ >>> len(dataset.cases)
67
+ 432
68
+ """
69
+ items = as_array(parse_json(read_utf8(path), path))
70
+ if items is None:
71
+ raise InputError(
72
+ path,
73
+ "expected a JSON array of question objects",
74
+ hint='BIRD files look like [{"question_id": 0, ...}, ...]',
75
+ )
76
+
77
+ cases: list[Case] = []
78
+ first_seen: dict[str, int] = {}
79
+ for index, item in enumerate(items):
80
+ location = f"item {index}"
81
+ case = _parse_case(item, path, location)
82
+ if case.id in first_seen:
83
+ raise InputError(
84
+ path,
85
+ f"question_id {case.id} is repeated (first at item {first_seen[case.id]})",
86
+ location=location,
87
+ hint="every question_id must be unique",
88
+ )
89
+ first_seen[case.id] = index
90
+ cases.append(case)
91
+ return Dataset(path=path, cases=tuple(cases))
92
+
93
+
94
+ def load_bird_tied(path: Path, dataset: Dataset) -> dict[str, str]:
95
+ """Read BIRD's tied-rows alternative gold file (``dev_tied_append.json``).
96
+
97
+ The file has the same shape as the question file. Each entry gives, for an
98
+ existing question, a gold SQL that returns every tied row instead of one.
99
+ It is used only for flags, never for the verdict (ADR 0024).
100
+
101
+ Returns:
102
+ Alternative gold SQL by case id.
103
+
104
+ Raises:
105
+ InputError: for any problem ``load_bird`` reports, or an entry whose
106
+ question is not in ``dataset`` or belongs to a different database.
107
+
108
+ Example:
109
+ >>> tied = load_bird_tied(Path("examples/bird-mini/dev_tied_append.json"), dataset)
110
+ >>> len(tied)
111
+ 13
112
+ """
113
+ known = {case.id: case for case in dataset.cases}
114
+ alternatives: dict[str, str] = {}
115
+ for index, entry in enumerate(load_bird(path).cases):
116
+ location = f"item {index}"
117
+ original = known.get(entry.id)
118
+ if original is None:
119
+ raise InputError(
120
+ path,
121
+ f"question_id {entry.id} is not in the dataset {dataset.path}",
122
+ location=location,
123
+ hint="use the tied file that belongs to the same dataset split",
124
+ )
125
+ if original.db_id != entry.db_id:
126
+ raise InputError(
127
+ path,
128
+ f"question_id {entry.id} is for database '{entry.db_id}', "
129
+ f"but the dataset has it for '{original.db_id}'",
130
+ location=location,
131
+ )
132
+ alternatives[entry.id] = entry.gold_sql
133
+ return alternatives
134
+
135
+
136
+ def _parse_case(item: object, path: Path, location: str) -> Case:
137
+ obj = as_object(item)
138
+ if obj is None:
139
+ raise InputError(
140
+ path,
141
+ f"expected a question object, found {json_type_name(item)}",
142
+ location=location,
143
+ )
144
+ question_id = _require(obj, "question_id", int, path, location)
145
+ difficulty = obj.get("difficulty")
146
+ if difficulty is not None and not isinstance(difficulty, str):
147
+ raise _wrong_type(path, location, "difficulty", "a string", difficulty)
148
+ return Case(
149
+ id=str(question_id),
150
+ db_id=_require(obj, "db_id", str, path, location),
151
+ question=_require(obj, "question", str, path, location),
152
+ evidence=_require(obj, "evidence", str, path, location),
153
+ gold_sql=_require(obj, "SQL", str, path, location),
154
+ difficulty=difficulty,
155
+ )
156
+
157
+
158
+ def _require(
159
+ obj: dict[str, object],
160
+ key: str,
161
+ kind: type[_Scalar],
162
+ path: Path,
163
+ location: str,
164
+ ) -> _Scalar:
165
+ if key not in obj:
166
+ raise InputError(
167
+ path,
168
+ f"missing field '{key}'",
169
+ location=location,
170
+ hint=f"add '{key}' to every question object",
171
+ )
172
+ value = obj[key]
173
+ # bool is a subclass of int in Python, but true/false is never a valid id.
174
+ if isinstance(value, kind) and not isinstance(value, bool):
175
+ return value
176
+ expected = "an integer" if kind is int else "a string"
177
+ raise _wrong_type(path, location, key, expected, value)
178
+
179
+
180
+ def _wrong_type(
181
+ path: Path, location: str, key: str, expected: str, value: object
182
+ ) -> InputError:
183
+ return InputError(
184
+ path,
185
+ f"field '{key}' must be {expected}, found {json_type_name(value)}",
186
+ location=location,
187
+ )
@@ -0,0 +1,187 @@
1
+ """Load model predictions in the two formats from ADR 0018.
2
+
3
+ * Text: one single-line SQL per line; line N predicts the dataset's N-th case.
4
+ * JSONL: one JSON object per line with ``id``, ``sql`` and optional ``meta``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass, field
10
+ from pathlib import Path
11
+
12
+ from text2sql_eval.datasets._json import (
13
+ as_object,
14
+ json_type_name,
15
+ parse_json,
16
+ read_utf8,
17
+ )
18
+ from text2sql_eval.datasets.bird import Dataset
19
+ from text2sql_eval.errors import InputError
20
+
21
+ # How many missing ids to list in an error before summarising the rest.
22
+ _MAX_IDS_SHOWN = 5
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Prediction:
27
+ """A model's SQL for one case. An empty ``sql`` means the model gave none."""
28
+
29
+ case_id: str
30
+ sql: str
31
+ meta: dict[str, object] = field(default_factory=dict[str, object])
32
+
33
+
34
+ def load_text_predictions(path: Path, dataset: Dataset) -> dict[str, Prediction]:
35
+ """Read a text file with one SQL per line, matched to cases by position.
36
+
37
+ Lines are split on ``\\n`` only (a trailing ``\\r`` is dropped), so Unicode
38
+ line separators inside a SQL string do not shift later lines. Surrounding
39
+ whitespace is removed. An empty line is an empty prediction, which keeps
40
+ every later line aligned with its case. A single newline at the end of the
41
+ file is allowed.
42
+
43
+ Raises:
44
+ InputError: if the file is unreadable or its line count differs from
45
+ the number of cases.
46
+
47
+ Example:
48
+ >>> preds = load_text_predictions(Path("preds.txt"), dataset)
49
+ >>> preds[dataset.cases[0].id].sql
50
+ 'SELECT ...'
51
+ """
52
+ lines = read_utf8(path).split("\n")
53
+ if lines and lines[-1] == "":
54
+ lines.pop()
55
+ expected = len(dataset.cases)
56
+ if len(lines) != expected:
57
+ raise InputError(
58
+ path,
59
+ f"has {_count(len(lines), 'line')} but the dataset {dataset.path} "
60
+ f"has {_count(expected, 'case')}",
61
+ hint=(
62
+ "write exactly one line per case, in dataset order; "
63
+ "keep an empty line where the model produced no SQL"
64
+ ),
65
+ )
66
+ return {
67
+ case.id: Prediction(case_id=case.id, sql=line.strip())
68
+ for case, line in zip(dataset.cases, lines, strict=True)
69
+ }
70
+
71
+
72
+ def load_jsonl_predictions(
73
+ path: Path, dataset: Dataset, *, allow_missing: bool = False
74
+ ) -> dict[str, Prediction]:
75
+ """Read a JSONL file where each line is ``{"id": ..., "sql": ..., "meta": {...}}``.
76
+
77
+ ``id`` may be an integer or a string and is matched against case ids as a
78
+ string (``718`` and ``"718"`` are the same case). Blank lines are skipped.
79
+
80
+ Args:
81
+ path: The JSONL file.
82
+ dataset: The cases the predictions belong to.
83
+ allow_missing: If true, cases without a line are left out of the
84
+ result instead of raising an error.
85
+
86
+ Raises:
87
+ InputError: on invalid JSON, a non-object line, a missing or mistyped
88
+ field, an id not in the dataset, a repeated id, or (unless
89
+ ``allow_missing``) cases with no prediction.
90
+
91
+ Example:
92
+ >>> preds = load_jsonl_predictions(Path("preds.jsonl"), dataset)
93
+ >>> preds["718"].meta
94
+ {'latency_ms': 1830}
95
+ """
96
+ known = set(dataset.ids)
97
+ predictions: dict[str, Prediction] = {}
98
+ first_line: dict[str, int] = {}
99
+ for line_number, line in enumerate(read_utf8(path).split("\n"), start=1):
100
+ if not line.strip():
101
+ continue
102
+ location = f"line {line_number}"
103
+ prediction = _parse_prediction_line(line, path, location)
104
+ case_id = prediction.case_id
105
+ if case_id not in known:
106
+ raise InputError(
107
+ path,
108
+ f"id {case_id} is not in the dataset {dataset.path}",
109
+ location=location,
110
+ hint="ids must match the dataset's question ids",
111
+ )
112
+ if case_id in first_line:
113
+ raise InputError(
114
+ path,
115
+ f"id {case_id} appears twice (first on line {first_line[case_id]})",
116
+ location=location,
117
+ hint="keep one prediction per case",
118
+ )
119
+ first_line[case_id] = line_number
120
+ predictions[case_id] = prediction
121
+
122
+ missing = [case_id for case_id in dataset.ids if case_id not in predictions]
123
+ if missing and not allow_missing:
124
+ shown = ", ".join(missing[:_MAX_IDS_SHOWN])
125
+ more = len(missing) - _MAX_IDS_SHOWN
126
+ listing = shown if more <= 0 else f"{shown} and {more} more"
127
+ raise InputError(
128
+ path,
129
+ f"{len(missing)} of {len(dataset.cases)} cases have no prediction: {listing}",
130
+ hint="add a line for each case, or allow missing predictions "
131
+ "(--allow-missing, or allow_missing=True in Python)",
132
+ )
133
+ return predictions
134
+
135
+
136
+ def _parse_prediction_line(line: str, path: Path, location: str) -> Prediction:
137
+ value = parse_json(line, path, location)
138
+ obj = as_object(value)
139
+ if obj is None:
140
+ raise InputError(
141
+ path,
142
+ f"expected a JSON object, found {json_type_name(value)}",
143
+ location=location,
144
+ hint='each line should look like {"id": 718, "sql": "SELECT ..."}',
145
+ )
146
+
147
+ for key in ("id", "sql"):
148
+ if key not in obj:
149
+ raise InputError(
150
+ path,
151
+ f"missing field '{key}'",
152
+ location=location,
153
+ hint='each line should look like {"id": 718, "sql": "SELECT ..."}',
154
+ )
155
+
156
+ raw_id = obj["id"]
157
+ if isinstance(raw_id, bool) or not isinstance(raw_id, int | str) or raw_id == "":
158
+ raise InputError(
159
+ path,
160
+ "'id' must be an integer or a non-empty string, "
161
+ f"found {json_type_name(raw_id)}",
162
+ location=location,
163
+ )
164
+
165
+ sql = obj["sql"]
166
+ if not isinstance(sql, str):
167
+ raise InputError(
168
+ path,
169
+ f"'sql' must be a string, found {json_type_name(sql)}",
170
+ location=location,
171
+ hint='use "" when the model produced no SQL',
172
+ )
173
+
174
+ meta_value = obj.get("meta", {})
175
+ meta = as_object(meta_value)
176
+ if meta is None:
177
+ raise InputError(
178
+ path,
179
+ f"'meta' must be an object, found {json_type_name(meta_value)}",
180
+ location=location,
181
+ )
182
+ return Prediction(case_id=str(raw_id), sql=sql.strip(), meta=meta)
183
+
184
+
185
+ def _count(number: int, noun: str) -> str:
186
+ """``1 line``, ``2 lines``: messages read naturally for any count."""
187
+ return f"{number} {noun}" if number == 1 else f"{number} {noun}s"