text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Compare two runs case by case (ADR 0035, 0036, 0037).
|
|
2
|
+
|
|
3
|
+
What a researcher used to do in a spreadsheet: which cases both runs got
|
|
4
|
+
right, which only one got right, and how the kinds of mistakes changed.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import csv
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
from text2sql_eval.csv_safety import excel_safe
|
|
16
|
+
from text2sql_eval.judge.ex import status_of
|
|
17
|
+
from text2sql_eval.runfile import CaseRecord, Run
|
|
18
|
+
|
|
19
|
+
Change = Literal["both_correct", "fixed", "regressed", "both_wrong"]
|
|
20
|
+
|
|
21
|
+
CHANGE_LABELS_KO: dict[Change, str] = {
|
|
22
|
+
"both_correct": "둘 다 맞음",
|
|
23
|
+
"fixed": "head만 맞음 (고쳐짐)",
|
|
24
|
+
"regressed": "base만 맞음 (나빠짐)",
|
|
25
|
+
"both_wrong": "둘 다 틀림",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class CaseChange:
|
|
31
|
+
"""One case seen in both runs."""
|
|
32
|
+
|
|
33
|
+
base: CaseRecord
|
|
34
|
+
head: CaseRecord
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def change(self) -> Change:
|
|
38
|
+
base_ok = status_of(self.base.verdict) == "correct"
|
|
39
|
+
head_ok = status_of(self.head.verdict) == "correct"
|
|
40
|
+
if base_ok and head_ok:
|
|
41
|
+
return "both_correct"
|
|
42
|
+
if head_ok:
|
|
43
|
+
return "fixed"
|
|
44
|
+
if base_ok:
|
|
45
|
+
return "regressed"
|
|
46
|
+
return "both_wrong"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class GroupEx:
|
|
51
|
+
"""EX of both runs over the common cases of one group (e.g. a difficulty)."""
|
|
52
|
+
|
|
53
|
+
name: str
|
|
54
|
+
cases: int
|
|
55
|
+
base_ex: float
|
|
56
|
+
head_ex: float
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class Comparison:
|
|
61
|
+
"""Two runs compared over the cases they share, in the base run's order."""
|
|
62
|
+
|
|
63
|
+
cases: tuple[CaseChange, ...]
|
|
64
|
+
dataset_mismatch: bool
|
|
65
|
+
only_in_base: tuple[str, ...]
|
|
66
|
+
only_in_head: tuple[str, ...]
|
|
67
|
+
|
|
68
|
+
def counts(self) -> Counter[Change]:
|
|
69
|
+
return Counter(case.change for case in self.cases)
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def base_ex(self) -> float:
|
|
73
|
+
return _ex([c.base for c in self.cases])
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def head_ex(self) -> float:
|
|
77
|
+
return _ex([c.head for c in self.cases])
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def delta_pp(self) -> float:
|
|
81
|
+
"""Change in EX, in percentage points (head minus base)."""
|
|
82
|
+
return (self.head_ex - self.base_ex) * 100
|
|
83
|
+
|
|
84
|
+
def by(self, key: Literal["difficulty", "db_id"]) -> list[GroupEx]:
|
|
85
|
+
"""EX per difficulty or per database, groups in name order."""
|
|
86
|
+
groups: dict[str, list[CaseChange]] = {}
|
|
87
|
+
for case in self.cases:
|
|
88
|
+
value = case.base.difficulty if key == "difficulty" else case.base.db_id
|
|
89
|
+
groups.setdefault(value or "(none)", []).append(case)
|
|
90
|
+
return [
|
|
91
|
+
GroupEx(
|
|
92
|
+
name=name,
|
|
93
|
+
cases=len(members),
|
|
94
|
+
base_ex=_ex([m.base for m in members]),
|
|
95
|
+
head_ex=_ex([m.head for m in members]),
|
|
96
|
+
)
|
|
97
|
+
for name, members in sorted(groups.items())
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
def output_differences(self) -> tuple[Counter[str], Counter[str]]:
|
|
101
|
+
"""Counts of each output difference kind in base and in head."""
|
|
102
|
+
return (
|
|
103
|
+
Counter(d for c in self.cases for d in _outputs(c.base)),
|
|
104
|
+
Counter(d for c in self.cases for d in _outputs(c.head)),
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def structure_differences(self) -> tuple[Counter[str], Counter[str]]:
|
|
108
|
+
"""Counts of each SQL structure difference kind in base and in head."""
|
|
109
|
+
return (
|
|
110
|
+
Counter(f for c in self.cases for f in _structures(c.base)),
|
|
111
|
+
Counter(f for c in self.cases for f in _structures(c.head)),
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def compare_runs(base: Run, head: Run) -> Comparison:
|
|
116
|
+
"""Pair up the cases two runs share; note what only one of them has.
|
|
117
|
+
|
|
118
|
+
A different dataset hash is reported, not refused (ADR 0037); callers show
|
|
119
|
+
it as a warning.
|
|
120
|
+
|
|
121
|
+
Example:
|
|
122
|
+
>>> comparison = compare_runs(read_run(no_evidence), read_run(evidence))
|
|
123
|
+
>>> comparison.counts()["fixed"], round(comparison.delta_pp, 1)
|
|
124
|
+
(123, 24.3)
|
|
125
|
+
"""
|
|
126
|
+
head_by_id = {case.id: case for case in head.cases}
|
|
127
|
+
base_ids = {case.id for case in base.cases}
|
|
128
|
+
pairs = tuple(
|
|
129
|
+
CaseChange(base=case, head=head_by_id[case.id])
|
|
130
|
+
for case in base.cases
|
|
131
|
+
if case.id in head_by_id
|
|
132
|
+
)
|
|
133
|
+
return Comparison(
|
|
134
|
+
cases=pairs,
|
|
135
|
+
dataset_mismatch=base.dataset.sha256 != head.dataset.sha256,
|
|
136
|
+
only_in_base=tuple(c.id for c in base.cases if c.id not in head_by_id),
|
|
137
|
+
only_in_head=tuple(c.id for c in head.cases if c.id not in base_ids),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _ex(records: list[CaseRecord]) -> float:
|
|
142
|
+
if not records:
|
|
143
|
+
return 0.0
|
|
144
|
+
return sum(status_of(r.verdict) == "correct" for r in records) / len(records)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _outputs(record: CaseRecord) -> tuple[str, ...]:
|
|
148
|
+
return () if record.diff is None else record.diff.output.differences
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _structures(record: CaseRecord) -> tuple[str, ...]:
|
|
152
|
+
if record.diff is None:
|
|
153
|
+
return ()
|
|
154
|
+
return tuple(fact.kind for fact in record.diff.structure.facts)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
CSV_COLUMNS = (
|
|
158
|
+
"id",
|
|
159
|
+
"db_id",
|
|
160
|
+
"difficulty",
|
|
161
|
+
"change",
|
|
162
|
+
"base_status",
|
|
163
|
+
"head_status",
|
|
164
|
+
"base_output_differences",
|
|
165
|
+
"head_output_differences",
|
|
166
|
+
"question",
|
|
167
|
+
"gold_sql",
|
|
168
|
+
"base_pred_sql",
|
|
169
|
+
"head_pred_sql",
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def write_comparison_csv(comparison: Comparison, path: Path) -> None:
|
|
174
|
+
"""One row per common case, for spreadsheet analysis.
|
|
175
|
+
|
|
176
|
+
Written as UTF-8 with a byte order mark so Excel shows Korean text and
|
|
177
|
+
other non-ASCII characters correctly. Multiple differences are joined
|
|
178
|
+
with ``;``. A cell that starts like a formula (``=``, ``-`` ...) gets a
|
|
179
|
+
leading ``'`` so Excel shows it as text (ADR 0040).
|
|
180
|
+
|
|
181
|
+
Example:
|
|
182
|
+
>>> comparison = compare_runs(read_run(base_path), read_run(head_path))
|
|
183
|
+
>>> write_comparison_csv(comparison, Path("runs/flips.csv"))
|
|
184
|
+
"""
|
|
185
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
186
|
+
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
|
187
|
+
writer = csv.writer(file)
|
|
188
|
+
writer.writerow(CSV_COLUMNS)
|
|
189
|
+
for case in comparison.cases:
|
|
190
|
+
base, head = case.base, case.head
|
|
191
|
+
cells = (
|
|
192
|
+
base.id,
|
|
193
|
+
base.db_id,
|
|
194
|
+
base.difficulty or "",
|
|
195
|
+
case.change,
|
|
196
|
+
status_of(base.verdict),
|
|
197
|
+
status_of(head.verdict),
|
|
198
|
+
";".join(_outputs(base)),
|
|
199
|
+
";".join(_outputs(head)),
|
|
200
|
+
base.question,
|
|
201
|
+
base.gold_sql,
|
|
202
|
+
base.pred_sql,
|
|
203
|
+
head.pred_sql,
|
|
204
|
+
)
|
|
205
|
+
writer.writerow(excel_safe(cell) for cell in cells)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Keep spreadsheet programs from running CSV cells as formulas (ADR 0040).
|
|
2
|
+
|
|
3
|
+
Excel reads a cell that starts with ``=``, ``+``, ``-`` or ``@`` (and a few
|
|
4
|
+
others) as a formula, so a predicted SQL such as ``=HYPERLINK(...)`` or one
|
|
5
|
+
starting with a ``--`` comment would run or show ``#NAME?``. Following OWASP's
|
|
6
|
+
advice on CSV injection, such cells get a leading apostrophe, which Excel
|
|
7
|
+
hides and which makes it show the cell as text.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
FORMULA_STARTS = frozenset("=+-@\t\r\n=+-@")
|
|
13
|
+
"""First characters that make a spreadsheet treat a cell as a formula.
|
|
14
|
+
|
|
15
|
+
The list, full-width forms included, is OWASP's (checked 2026-11-11).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def excel_safe(cell: str) -> str:
|
|
20
|
+
"""``cell`` with a leading ``'`` if its first character starts a formula.
|
|
21
|
+
|
|
22
|
+
Only the first character matters; ``=`` inside a cell is left alone.
|
|
23
|
+
|
|
24
|
+
Example:
|
|
25
|
+
>>> excel_safe("=HYPERLINK(\\"http://example.com\\")")
|
|
26
|
+
'\\'=HYPERLINK("http://example.com")'
|
|
27
|
+
>>> excel_safe("SELECT name FROM hero WHERE id = 1")
|
|
28
|
+
'SELECT name FROM hero WHERE id = 1'
|
|
29
|
+
"""
|
|
30
|
+
if cell and cell[0] in FORMULA_STARTS:
|
|
31
|
+
return "'" + cell
|
|
32
|
+
return cell
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Small helpers that turn ``json.loads`` output into precisely typed values."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import cast
|
|
8
|
+
|
|
9
|
+
from text2sql_eval.errors import InputError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def read_utf8(path: Path) -> str:
|
|
13
|
+
"""Read a whole text file as UTF-8, turning OS and decoding errors into InputError.
|
|
14
|
+
|
|
15
|
+
A leading byte order mark is dropped: Windows editors and Excel add one, and
|
|
16
|
+
left in place it would break the first JSON line or prefix the first SQL.
|
|
17
|
+
"""
|
|
18
|
+
try:
|
|
19
|
+
return path.read_text(encoding="utf-8-sig")
|
|
20
|
+
except FileNotFoundError:
|
|
21
|
+
raise InputError(path, "file not found", hint="check the path") from None
|
|
22
|
+
except UnicodeDecodeError as exc:
|
|
23
|
+
raise InputError(
|
|
24
|
+
path,
|
|
25
|
+
f"not valid UTF-8 (byte {exc.start})",
|
|
26
|
+
hint="save the file with UTF-8 encoding",
|
|
27
|
+
) from None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_json(text: str, path: Path, location: str | None = None) -> object:
|
|
31
|
+
"""Parse JSON text, reporting syntax errors with their position."""
|
|
32
|
+
try:
|
|
33
|
+
value: object = json.loads(text)
|
|
34
|
+
except json.JSONDecodeError as exc:
|
|
35
|
+
where = location or f"line {exc.lineno}"
|
|
36
|
+
raise InputError(
|
|
37
|
+
path,
|
|
38
|
+
f"invalid JSON at column {exc.colno}: {exc.msg}",
|
|
39
|
+
location=where,
|
|
40
|
+
hint="check quotes, commas and brackets on this line",
|
|
41
|
+
) from None
|
|
42
|
+
return value
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def as_object(value: object) -> dict[str, object] | None:
|
|
46
|
+
"""Return ``value`` as a JSON object, or ``None`` if it is something else."""
|
|
47
|
+
if not isinstance(value, dict):
|
|
48
|
+
return None
|
|
49
|
+
# JSON object keys are always strings, so this narrowing is exact.
|
|
50
|
+
return cast("dict[str, object]", value)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def as_array(value: object) -> list[object] | None:
|
|
54
|
+
"""Return ``value`` as a JSON array, or ``None`` if it is something else."""
|
|
55
|
+
if not isinstance(value, list):
|
|
56
|
+
return None
|
|
57
|
+
# Elements are arbitrary JSON values, which ``object`` describes exactly.
|
|
58
|
+
return cast("list[object]", value)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def json_type_name(value: object) -> str:
|
|
62
|
+
"""Name a JSON value's type the way users see it in the file."""
|
|
63
|
+
if value is None:
|
|
64
|
+
return "null"
|
|
65
|
+
if isinstance(value, bool):
|
|
66
|
+
return "boolean"
|
|
67
|
+
if isinstance(value, int | float):
|
|
68
|
+
return "number"
|
|
69
|
+
if isinstance(value, str):
|
|
70
|
+
return "string"
|
|
71
|
+
if isinstance(value, list):
|
|
72
|
+
return "array"
|
|
73
|
+
return "object"
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Load evaluation cases from a BIRD-format question file (for example ``dev.json``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TypeVar
|
|
8
|
+
|
|
9
|
+
from text2sql_eval.datasets._json import (
|
|
10
|
+
as_array,
|
|
11
|
+
as_object,
|
|
12
|
+
json_type_name,
|
|
13
|
+
parse_json,
|
|
14
|
+
read_utf8,
|
|
15
|
+
)
|
|
16
|
+
from text2sql_eval.errors import InputError
|
|
17
|
+
|
|
18
|
+
# A constrained TypeVar instead of 3.12's ``def f[T: (int, str)]`` (ADR 0009).
|
|
19
|
+
_Scalar = TypeVar("_Scalar", int, str)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Case:
|
|
24
|
+
"""One evaluation question with its gold SQL.
|
|
25
|
+
|
|
26
|
+
``id`` is the dataset's question id as a string (BIRD's integer
|
|
27
|
+
``question_id`` 718 becomes ``"718"``), so ids from any source compare the
|
|
28
|
+
same way.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
id: str
|
|
32
|
+
db_id: str
|
|
33
|
+
question: str
|
|
34
|
+
evidence: str
|
|
35
|
+
gold_sql: str
|
|
36
|
+
difficulty: str | None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Dataset:
|
|
41
|
+
"""Cases in file order. Order matters: text prediction files follow it."""
|
|
42
|
+
|
|
43
|
+
path: Path
|
|
44
|
+
cases: tuple[Case, ...]
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def ids(self) -> tuple[str, ...]:
|
|
48
|
+
"""Case ids in file order."""
|
|
49
|
+
return tuple(case.id for case in self.cases)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load_bird(path: Path) -> Dataset:
|
|
53
|
+
"""Read a BIRD question file: a JSON array of question objects.
|
|
54
|
+
|
|
55
|
+
Each object needs ``question_id`` (integer), ``db_id``, ``question``,
|
|
56
|
+
``evidence`` and ``SQL`` (strings); ``difficulty`` (string) is optional.
|
|
57
|
+
Other fields are ignored.
|
|
58
|
+
|
|
59
|
+
Raises:
|
|
60
|
+
InputError: if the file is unreadable, is not a JSON array of objects,
|
|
61
|
+
misses a required field, has a field of the wrong type, or repeats
|
|
62
|
+
a ``question_id``.
|
|
63
|
+
|
|
64
|
+
Example:
|
|
65
|
+
>>> dataset = load_bird(Path("examples/bird-mini/dev.json"))
|
|
66
|
+
>>> len(dataset.cases)
|
|
67
|
+
432
|
|
68
|
+
"""
|
|
69
|
+
items = as_array(parse_json(read_utf8(path), path))
|
|
70
|
+
if items is None:
|
|
71
|
+
raise InputError(
|
|
72
|
+
path,
|
|
73
|
+
"expected a JSON array of question objects",
|
|
74
|
+
hint='BIRD files look like [{"question_id": 0, ...}, ...]',
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
cases: list[Case] = []
|
|
78
|
+
first_seen: dict[str, int] = {}
|
|
79
|
+
for index, item in enumerate(items):
|
|
80
|
+
location = f"item {index}"
|
|
81
|
+
case = _parse_case(item, path, location)
|
|
82
|
+
if case.id in first_seen:
|
|
83
|
+
raise InputError(
|
|
84
|
+
path,
|
|
85
|
+
f"question_id {case.id} is repeated (first at item {first_seen[case.id]})",
|
|
86
|
+
location=location,
|
|
87
|
+
hint="every question_id must be unique",
|
|
88
|
+
)
|
|
89
|
+
first_seen[case.id] = index
|
|
90
|
+
cases.append(case)
|
|
91
|
+
return Dataset(path=path, cases=tuple(cases))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def load_bird_tied(path: Path, dataset: Dataset) -> dict[str, str]:
|
|
95
|
+
"""Read BIRD's tied-rows alternative gold file (``dev_tied_append.json``).
|
|
96
|
+
|
|
97
|
+
The file has the same shape as the question file. Each entry gives, for an
|
|
98
|
+
existing question, a gold SQL that returns every tied row instead of one.
|
|
99
|
+
It is used only for flags, never for the verdict (ADR 0024).
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
Alternative gold SQL by case id.
|
|
103
|
+
|
|
104
|
+
Raises:
|
|
105
|
+
InputError: for any problem ``load_bird`` reports, or an entry whose
|
|
106
|
+
question is not in ``dataset`` or belongs to a different database.
|
|
107
|
+
|
|
108
|
+
Example:
|
|
109
|
+
>>> tied = load_bird_tied(Path("examples/bird-mini/dev_tied_append.json"), dataset)
|
|
110
|
+
>>> len(tied)
|
|
111
|
+
13
|
|
112
|
+
"""
|
|
113
|
+
known = {case.id: case for case in dataset.cases}
|
|
114
|
+
alternatives: dict[str, str] = {}
|
|
115
|
+
for index, entry in enumerate(load_bird(path).cases):
|
|
116
|
+
location = f"item {index}"
|
|
117
|
+
original = known.get(entry.id)
|
|
118
|
+
if original is None:
|
|
119
|
+
raise InputError(
|
|
120
|
+
path,
|
|
121
|
+
f"question_id {entry.id} is not in the dataset {dataset.path}",
|
|
122
|
+
location=location,
|
|
123
|
+
hint="use the tied file that belongs to the same dataset split",
|
|
124
|
+
)
|
|
125
|
+
if original.db_id != entry.db_id:
|
|
126
|
+
raise InputError(
|
|
127
|
+
path,
|
|
128
|
+
f"question_id {entry.id} is for database '{entry.db_id}', "
|
|
129
|
+
f"but the dataset has it for '{original.db_id}'",
|
|
130
|
+
location=location,
|
|
131
|
+
)
|
|
132
|
+
alternatives[entry.id] = entry.gold_sql
|
|
133
|
+
return alternatives
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _parse_case(item: object, path: Path, location: str) -> Case:
|
|
137
|
+
obj = as_object(item)
|
|
138
|
+
if obj is None:
|
|
139
|
+
raise InputError(
|
|
140
|
+
path,
|
|
141
|
+
f"expected a question object, found {json_type_name(item)}",
|
|
142
|
+
location=location,
|
|
143
|
+
)
|
|
144
|
+
question_id = _require(obj, "question_id", int, path, location)
|
|
145
|
+
difficulty = obj.get("difficulty")
|
|
146
|
+
if difficulty is not None and not isinstance(difficulty, str):
|
|
147
|
+
raise _wrong_type(path, location, "difficulty", "a string", difficulty)
|
|
148
|
+
return Case(
|
|
149
|
+
id=str(question_id),
|
|
150
|
+
db_id=_require(obj, "db_id", str, path, location),
|
|
151
|
+
question=_require(obj, "question", str, path, location),
|
|
152
|
+
evidence=_require(obj, "evidence", str, path, location),
|
|
153
|
+
gold_sql=_require(obj, "SQL", str, path, location),
|
|
154
|
+
difficulty=difficulty,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _require(
|
|
159
|
+
obj: dict[str, object],
|
|
160
|
+
key: str,
|
|
161
|
+
kind: type[_Scalar],
|
|
162
|
+
path: Path,
|
|
163
|
+
location: str,
|
|
164
|
+
) -> _Scalar:
|
|
165
|
+
if key not in obj:
|
|
166
|
+
raise InputError(
|
|
167
|
+
path,
|
|
168
|
+
f"missing field '{key}'",
|
|
169
|
+
location=location,
|
|
170
|
+
hint=f"add '{key}' to every question object",
|
|
171
|
+
)
|
|
172
|
+
value = obj[key]
|
|
173
|
+
# bool is a subclass of int in Python, but true/false is never a valid id.
|
|
174
|
+
if isinstance(value, kind) and not isinstance(value, bool):
|
|
175
|
+
return value
|
|
176
|
+
expected = "an integer" if kind is int else "a string"
|
|
177
|
+
raise _wrong_type(path, location, key, expected, value)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _wrong_type(
|
|
181
|
+
path: Path, location: str, key: str, expected: str, value: object
|
|
182
|
+
) -> InputError:
|
|
183
|
+
return InputError(
|
|
184
|
+
path,
|
|
185
|
+
f"field '{key}' must be {expected}, found {json_type_name(value)}",
|
|
186
|
+
location=location,
|
|
187
|
+
)
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Load model predictions in the two formats from ADR 0018.
|
|
2
|
+
|
|
3
|
+
* Text: one single-line SQL per line; line N predicts the dataset's N-th case.
|
|
4
|
+
* JSONL: one JSON object per line with ``id``, ``sql`` and optional ``meta``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from text2sql_eval.datasets._json import (
|
|
13
|
+
as_object,
|
|
14
|
+
json_type_name,
|
|
15
|
+
parse_json,
|
|
16
|
+
read_utf8,
|
|
17
|
+
)
|
|
18
|
+
from text2sql_eval.datasets.bird import Dataset
|
|
19
|
+
from text2sql_eval.errors import InputError
|
|
20
|
+
|
|
21
|
+
# How many missing ids to list in an error before summarising the rest.
|
|
22
|
+
_MAX_IDS_SHOWN = 5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Prediction:
|
|
27
|
+
"""A model's SQL for one case. An empty ``sql`` means the model gave none."""
|
|
28
|
+
|
|
29
|
+
case_id: str
|
|
30
|
+
sql: str
|
|
31
|
+
meta: dict[str, object] = field(default_factory=dict[str, object])
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_text_predictions(path: Path, dataset: Dataset) -> dict[str, Prediction]:
|
|
35
|
+
"""Read a text file with one SQL per line, matched to cases by position.
|
|
36
|
+
|
|
37
|
+
Lines are split on ``\\n`` only (a trailing ``\\r`` is dropped), so Unicode
|
|
38
|
+
line separators inside a SQL string do not shift later lines. Surrounding
|
|
39
|
+
whitespace is removed. An empty line is an empty prediction, which keeps
|
|
40
|
+
every later line aligned with its case. A single newline at the end of the
|
|
41
|
+
file is allowed.
|
|
42
|
+
|
|
43
|
+
Raises:
|
|
44
|
+
InputError: if the file is unreadable or its line count differs from
|
|
45
|
+
the number of cases.
|
|
46
|
+
|
|
47
|
+
Example:
|
|
48
|
+
>>> preds = load_text_predictions(Path("preds.txt"), dataset)
|
|
49
|
+
>>> preds[dataset.cases[0].id].sql
|
|
50
|
+
'SELECT ...'
|
|
51
|
+
"""
|
|
52
|
+
lines = read_utf8(path).split("\n")
|
|
53
|
+
if lines and lines[-1] == "":
|
|
54
|
+
lines.pop()
|
|
55
|
+
expected = len(dataset.cases)
|
|
56
|
+
if len(lines) != expected:
|
|
57
|
+
raise InputError(
|
|
58
|
+
path,
|
|
59
|
+
f"has {_count(len(lines), 'line')} but the dataset {dataset.path} "
|
|
60
|
+
f"has {_count(expected, 'case')}",
|
|
61
|
+
hint=(
|
|
62
|
+
"write exactly one line per case, in dataset order; "
|
|
63
|
+
"keep an empty line where the model produced no SQL"
|
|
64
|
+
),
|
|
65
|
+
)
|
|
66
|
+
return {
|
|
67
|
+
case.id: Prediction(case_id=case.id, sql=line.strip())
|
|
68
|
+
for case, line in zip(dataset.cases, lines, strict=True)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def load_jsonl_predictions(
|
|
73
|
+
path: Path, dataset: Dataset, *, allow_missing: bool = False
|
|
74
|
+
) -> dict[str, Prediction]:
|
|
75
|
+
"""Read a JSONL file where each line is ``{"id": ..., "sql": ..., "meta": {...}}``.
|
|
76
|
+
|
|
77
|
+
``id`` may be an integer or a string and is matched against case ids as a
|
|
78
|
+
string (``718`` and ``"718"`` are the same case). Blank lines are skipped.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
path: The JSONL file.
|
|
82
|
+
dataset: The cases the predictions belong to.
|
|
83
|
+
allow_missing: If true, cases without a line are left out of the
|
|
84
|
+
result instead of raising an error.
|
|
85
|
+
|
|
86
|
+
Raises:
|
|
87
|
+
InputError: on invalid JSON, a non-object line, a missing or mistyped
|
|
88
|
+
field, an id not in the dataset, a repeated id, or (unless
|
|
89
|
+
``allow_missing``) cases with no prediction.
|
|
90
|
+
|
|
91
|
+
Example:
|
|
92
|
+
>>> preds = load_jsonl_predictions(Path("preds.jsonl"), dataset)
|
|
93
|
+
>>> preds["718"].meta
|
|
94
|
+
{'latency_ms': 1830}
|
|
95
|
+
"""
|
|
96
|
+
known = set(dataset.ids)
|
|
97
|
+
predictions: dict[str, Prediction] = {}
|
|
98
|
+
first_line: dict[str, int] = {}
|
|
99
|
+
for line_number, line in enumerate(read_utf8(path).split("\n"), start=1):
|
|
100
|
+
if not line.strip():
|
|
101
|
+
continue
|
|
102
|
+
location = f"line {line_number}"
|
|
103
|
+
prediction = _parse_prediction_line(line, path, location)
|
|
104
|
+
case_id = prediction.case_id
|
|
105
|
+
if case_id not in known:
|
|
106
|
+
raise InputError(
|
|
107
|
+
path,
|
|
108
|
+
f"id {case_id} is not in the dataset {dataset.path}",
|
|
109
|
+
location=location,
|
|
110
|
+
hint="ids must match the dataset's question ids",
|
|
111
|
+
)
|
|
112
|
+
if case_id in first_line:
|
|
113
|
+
raise InputError(
|
|
114
|
+
path,
|
|
115
|
+
f"id {case_id} appears twice (first on line {first_line[case_id]})",
|
|
116
|
+
location=location,
|
|
117
|
+
hint="keep one prediction per case",
|
|
118
|
+
)
|
|
119
|
+
first_line[case_id] = line_number
|
|
120
|
+
predictions[case_id] = prediction
|
|
121
|
+
|
|
122
|
+
missing = [case_id for case_id in dataset.ids if case_id not in predictions]
|
|
123
|
+
if missing and not allow_missing:
|
|
124
|
+
shown = ", ".join(missing[:_MAX_IDS_SHOWN])
|
|
125
|
+
more = len(missing) - _MAX_IDS_SHOWN
|
|
126
|
+
listing = shown if more <= 0 else f"{shown} and {more} more"
|
|
127
|
+
raise InputError(
|
|
128
|
+
path,
|
|
129
|
+
f"{len(missing)} of {len(dataset.cases)} cases have no prediction: {listing}",
|
|
130
|
+
hint="add a line for each case, or allow missing predictions "
|
|
131
|
+
"(--allow-missing, or allow_missing=True in Python)",
|
|
132
|
+
)
|
|
133
|
+
return predictions
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _parse_prediction_line(line: str, path: Path, location: str) -> Prediction:
|
|
137
|
+
value = parse_json(line, path, location)
|
|
138
|
+
obj = as_object(value)
|
|
139
|
+
if obj is None:
|
|
140
|
+
raise InputError(
|
|
141
|
+
path,
|
|
142
|
+
f"expected a JSON object, found {json_type_name(value)}",
|
|
143
|
+
location=location,
|
|
144
|
+
hint='each line should look like {"id": 718, "sql": "SELECT ..."}',
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
for key in ("id", "sql"):
|
|
148
|
+
if key not in obj:
|
|
149
|
+
raise InputError(
|
|
150
|
+
path,
|
|
151
|
+
f"missing field '{key}'",
|
|
152
|
+
location=location,
|
|
153
|
+
hint='each line should look like {"id": 718, "sql": "SELECT ..."}',
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
raw_id = obj["id"]
|
|
157
|
+
if isinstance(raw_id, bool) or not isinstance(raw_id, int | str) or raw_id == "":
|
|
158
|
+
raise InputError(
|
|
159
|
+
path,
|
|
160
|
+
"'id' must be an integer or a non-empty string, "
|
|
161
|
+
f"found {json_type_name(raw_id)}",
|
|
162
|
+
location=location,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
sql = obj["sql"]
|
|
166
|
+
if not isinstance(sql, str):
|
|
167
|
+
raise InputError(
|
|
168
|
+
path,
|
|
169
|
+
f"'sql' must be a string, found {json_type_name(sql)}",
|
|
170
|
+
location=location,
|
|
171
|
+
hint='use "" when the model produced no SQL',
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
meta_value = obj.get("meta", {})
|
|
175
|
+
meta = as_object(meta_value)
|
|
176
|
+
if meta is None:
|
|
177
|
+
raise InputError(
|
|
178
|
+
path,
|
|
179
|
+
f"'meta' must be an object, found {json_type_name(meta_value)}",
|
|
180
|
+
location=location,
|
|
181
|
+
)
|
|
182
|
+
return Prediction(case_id=str(raw_id), sql=sql.strip(), meta=meta)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _count(number: int, noun: str) -> str:
|
|
186
|
+
"""``1 line``, ``2 lines``: messages read naturally for any count."""
|
|
187
|
+
return f"{number} {noun}" if number == 1 else f"{number} {noun}s"
|