text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""One CSV row per case of a run, for spreadsheet analysis (ADR 0007).
|
|
2
|
+
|
|
3
|
+
Researchers used to classify wrong cases by hand in a spreadsheet. This file
|
|
4
|
+
starts that sheet with the verdict, the differences and the flags already
|
|
5
|
+
filled in, and leaves ``my_category`` and ``notes`` empty for the user.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from text2sql_eval.csv_safety import excel_safe
|
|
14
|
+
from text2sql_eval.judge.ex import ExecError, Verdict, status_of
|
|
15
|
+
from text2sql_eval.runfile import CaseRecord, Run
|
|
16
|
+
|
|
17
|
+
CSV_COLUMNS = (
|
|
18
|
+
"id",
|
|
19
|
+
"db_id",
|
|
20
|
+
"difficulty",
|
|
21
|
+
"status",
|
|
22
|
+
"prediction",
|
|
23
|
+
"output_differences",
|
|
24
|
+
"sql_structure_differences",
|
|
25
|
+
"flags",
|
|
26
|
+
"question",
|
|
27
|
+
"evidence",
|
|
28
|
+
"gold_sql",
|
|
29
|
+
"pred_sql",
|
|
30
|
+
"error",
|
|
31
|
+
"my_category",
|
|
32
|
+
"notes",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def write_report_csv(run: Run, path: Path) -> None:
|
|
37
|
+
"""Write every case of ``run`` as one CSV row, in the run's order.
|
|
38
|
+
|
|
39
|
+
Written as UTF-8 with a byte order mark so Excel shows Korean text and
|
|
40
|
+
other non-ASCII characters correctly. Multiple differences or flags are
|
|
41
|
+
joined with ``;``. ``prediction`` is ``given``, ``empty`` or ``missing``.
|
|
42
|
+
A cell that starts like a formula (``=``, ``-`` ...) gets a leading ``'``
|
|
43
|
+
so Excel shows it as text (ADR 0040).
|
|
44
|
+
|
|
45
|
+
Example:
|
|
46
|
+
>>> write_report_csv(read_run(Path("runs/v3.json")), Path("runs/v3.csv"))
|
|
47
|
+
"""
|
|
48
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
|
50
|
+
writer = csv.writer(file)
|
|
51
|
+
writer.writerow(CSV_COLUMNS)
|
|
52
|
+
for case in run.cases:
|
|
53
|
+
writer.writerow(excel_safe(cell) for cell in _row(case))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _row(case: CaseRecord) -> tuple[str, ...]:
|
|
57
|
+
outputs = () if case.diff is None else case.diff.output.differences
|
|
58
|
+
facts = () if case.diff is None else case.diff.structure.facts
|
|
59
|
+
return (
|
|
60
|
+
case.id,
|
|
61
|
+
case.db_id,
|
|
62
|
+
case.difficulty or "",
|
|
63
|
+
status_of(case.verdict),
|
|
64
|
+
case.prediction_state,
|
|
65
|
+
";".join(outputs),
|
|
66
|
+
";".join(fact.kind for fact in facts),
|
|
67
|
+
";".join(flag.kind for flag in case.verdict.flags),
|
|
68
|
+
case.question,
|
|
69
|
+
case.evidence,
|
|
70
|
+
case.gold_sql,
|
|
71
|
+
case.pred_sql,
|
|
72
|
+
_error(case.verdict),
|
|
73
|
+
"",
|
|
74
|
+
"",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _error(verdict: Verdict) -> str:
|
|
79
|
+
if isinstance(verdict, ExecError):
|
|
80
|
+
return f"{verdict.side} {verdict.kind}: {verdict.message}"
|
|
81
|
+
return ""
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""What to highlight in the report: differing SQL clauses and result rows (ADR 0039).
|
|
2
|
+
|
|
3
|
+
Highlighting is display only. It never changes a verdict, an output
|
|
4
|
+
difference, a SQL structure difference or a flag.
|
|
5
|
+
|
|
6
|
+
Clauses are compared after the same normalization as the SQL structure
|
|
7
|
+
differences (table aliases resolved, identifiers lower-cased, column
|
|
8
|
+
qualifiers dropped), so ``T1.name`` and ``s.name`` are not a difference.
|
|
9
|
+
Writing order that SQLite ignores is not a difference either: the order of
|
|
10
|
+
conditions joined by AND, the two sides of ``=``, the order of inner joins,
|
|
11
|
+
and ``INNER JOIN`` versus ``JOIN``. Only the outer query's clauses are
|
|
12
|
+
compared; a subquery belongs to the clause it sits in.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
from typing import Literal
|
|
19
|
+
|
|
20
|
+
from sqlglot import exp
|
|
21
|
+
|
|
22
|
+
# sqlglot 30's node base class; see the note in facts/structure.py.
|
|
23
|
+
from sqlglot.expressions.core import Expr
|
|
24
|
+
|
|
25
|
+
from text2sql_eval.facts.structure import DIALECT, normalize, unqualified
|
|
26
|
+
from text2sql_eval.runfile import QuerySummary, ResultSample
|
|
27
|
+
|
|
28
|
+
Clause = Literal["with", "select", "from", "where", "group", "having", "order", "limit"]
|
|
29
|
+
|
|
30
|
+
# First words of the lines that start each clause in sqlglot's pretty output.
|
|
31
|
+
# FROM and every kind of JOIN form one clause, as do LIMIT and OFFSET.
|
|
32
|
+
_CLAUSE_STARTS: tuple[tuple[str, Clause], ...] = (
|
|
33
|
+
("WITH", "with"),
|
|
34
|
+
("SELECT", "select"),
|
|
35
|
+
("FROM", "from"),
|
|
36
|
+
("WHERE", "where"),
|
|
37
|
+
("GROUP BY", "group"),
|
|
38
|
+
("HAVING", "having"),
|
|
39
|
+
("ORDER BY", "order"),
|
|
40
|
+
("LIMIT", "limit"),
|
|
41
|
+
("OFFSET", "limit"),
|
|
42
|
+
)
|
|
43
|
+
# Clauses held in one node under the same name in sqlglot's Select.args;
|
|
44
|
+
# WHERE and HAVING are compared as conditions instead.
|
|
45
|
+
_SINGLE_NODE_CLAUSES: tuple[Clause, ...] = ("group", "order")
|
|
46
|
+
_JOIN_WORDS = frozenset({"JOIN", "INNER", "LEFT", "RIGHT", "FULL", "CROSS", "NATURAL"})
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def differing_clauses(gold: Expr, pred: Expr) -> frozenset[Clause]:
|
|
50
|
+
"""Clauses whose normalized SQL differs, or that only one query has.
|
|
51
|
+
|
|
52
|
+
Empty when either query is not a plain SELECT (for example a UNION),
|
|
53
|
+
because its clauses cannot be lined up with the other query's.
|
|
54
|
+
|
|
55
|
+
Example:
|
|
56
|
+
>>> differing_clauses(
|
|
57
|
+
... parse("SELECT T1.name FROM hero AS T1 WHERE T1.power = 'Fly'"),
|
|
58
|
+
... parse("SELECT h.name FROM hero AS h WHERE h.power = 'fly'"),
|
|
59
|
+
... )
|
|
60
|
+
frozenset({'where'})
|
|
61
|
+
"""
|
|
62
|
+
if not isinstance(gold, exp.Select) or not isinstance(pred, exp.Select):
|
|
63
|
+
return frozenset()
|
|
64
|
+
gold_parts = _clause_texts(gold)
|
|
65
|
+
pred_parts = _clause_texts(pred)
|
|
66
|
+
return frozenset(
|
|
67
|
+
clause
|
|
68
|
+
for clause in gold_parts.keys() | pred_parts.keys()
|
|
69
|
+
if gold_parts.get(clause) != pred_parts.get(clause)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def clause_blocks(pretty_sql: str) -> list[tuple[Clause | None, str]]:
|
|
74
|
+
"""Split sqlglot's pretty output into one block per outer clause.
|
|
75
|
+
|
|
76
|
+
sqlglot starts each outer clause at the beginning of a line and indents
|
|
77
|
+
everything inside it. Lines before the first known clause get ``None``.
|
|
78
|
+
|
|
79
|
+
Example:
|
|
80
|
+
>>> clause_blocks("SELECT\n name\nFROM hero\nJOIN power\n ON hero.id = power.id")
|
|
81
|
+
[('select', 'SELECT\n name'), ('from', 'FROM hero\nJOIN power\n ON hero.id = power.id')]
|
|
82
|
+
"""
|
|
83
|
+
blocks: list[tuple[Clause | None, list[str]]] = []
|
|
84
|
+
for line in pretty_sql.split("\n"):
|
|
85
|
+
clause = _clause_starting(line)
|
|
86
|
+
if clause is not None and (not blocks or blocks[-1][0] != clause):
|
|
87
|
+
blocks.append((clause, [line]))
|
|
88
|
+
elif blocks:
|
|
89
|
+
blocks[-1][1].append(line)
|
|
90
|
+
else:
|
|
91
|
+
blocks.append((None, [line]))
|
|
92
|
+
return [(clause, "\n".join(lines)) for clause, lines in blocks]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def differing_rows(
|
|
96
|
+
gold: QuerySummary, pred: QuerySummary
|
|
97
|
+
) -> tuple[tuple[bool, ...], tuple[bool, ...]]:
|
|
98
|
+
"""For each stored row of each side, whether the other result lacks it.
|
|
99
|
+
|
|
100
|
+
The run file keeps only the first rows of each result. A row missing from
|
|
101
|
+
the other side's sample may still be in its full result, so a side's rows
|
|
102
|
+
are marked only when the other side's sample is the complete result. Rows
|
|
103
|
+
are compared like EX does (as tuples, ``1 == 1.0``), and only when both
|
|
104
|
+
results have the same number of columns.
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
>>> gold = ResultSample(("name",), (("Surtur",), ("Ymir",)), row_count=2)
|
|
108
|
+
>>> pred = ResultSample(("name",), (("Surtur",), ("Thor",)), row_count=2)
|
|
109
|
+
>>> differing_rows(gold, pred)
|
|
110
|
+
((False, True), (False, True))
|
|
111
|
+
"""
|
|
112
|
+
gold_marks = tuple(False for _ in _rows(gold))
|
|
113
|
+
pred_marks = tuple(False for _ in _rows(pred))
|
|
114
|
+
if not isinstance(gold, ResultSample) or not isinstance(pred, ResultSample):
|
|
115
|
+
return gold_marks, pred_marks
|
|
116
|
+
if len(gold.columns) != len(pred.columns) or not gold.columns:
|
|
117
|
+
return gold_marks, pred_marks
|
|
118
|
+
if _complete(pred):
|
|
119
|
+
pred_rows = set(pred.rows)
|
|
120
|
+
gold_marks = tuple(row not in pred_rows for row in gold.rows)
|
|
121
|
+
if _complete(gold):
|
|
122
|
+
gold_rows = set(gold.rows)
|
|
123
|
+
pred_marks = tuple(row not in gold_rows for row in pred.rows)
|
|
124
|
+
return gold_marks, pred_marks
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _clause_texts(select: exp.Select) -> dict[Clause, str]:
|
|
128
|
+
"""Normalized SQL of each clause the query has."""
|
|
129
|
+
query = normalize(select)
|
|
130
|
+
parts: dict[Clause, str] = {}
|
|
131
|
+
with_ = query.args.get("with_")
|
|
132
|
+
if isinstance(with_, Expr):
|
|
133
|
+
parts["with"] = unqualified(with_)
|
|
134
|
+
# Output column names (AS ...) do not matter to EX, so aliases are dropped.
|
|
135
|
+
columns = ", ".join(unqualified(e.unalias()) for e in query.expressions)
|
|
136
|
+
distinct = "DISTINCT " if query.args.get("distinct") else ""
|
|
137
|
+
parts["select"] = distinct + columns
|
|
138
|
+
sources = _sources_text(query)
|
|
139
|
+
if sources is not None:
|
|
140
|
+
parts["from"] = sources
|
|
141
|
+
for clause in ("where", "having"):
|
|
142
|
+
node = query.args.get(clause)
|
|
143
|
+
if isinstance(node, Expr):
|
|
144
|
+
parts[clause] = _condition_text(node.this, unqualified)
|
|
145
|
+
for clause in _SINGLE_NODE_CLAUSES:
|
|
146
|
+
node = query.args.get(clause)
|
|
147
|
+
if isinstance(node, Expr):
|
|
148
|
+
parts[clause] = unqualified(node)
|
|
149
|
+
limits = [query.args.get("limit"), query.args.get("offset")]
|
|
150
|
+
if any(isinstance(node, Expr) for node in limits):
|
|
151
|
+
parts["limit"] = " ".join(
|
|
152
|
+
unqualified(node) for node in limits if isinstance(node, Expr)
|
|
153
|
+
)
|
|
154
|
+
return parts
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _sources_text(query: Expr) -> str | None:
|
|
158
|
+
"""FROM and JOINs, where the order of inner joins does not matter.
|
|
159
|
+
|
|
160
|
+
Join conditions keep their table qualifiers: in ``ON`` they say which
|
|
161
|
+
tables are joined. LEFT, RIGHT and FULL joins keep their written order.
|
|
162
|
+
"""
|
|
163
|
+
from_ = query.args.get("from_")
|
|
164
|
+
joins = [j for j in query.args.get("joins") or () if isinstance(j, exp.Join)]
|
|
165
|
+
if not isinstance(from_, exp.From) and not joins:
|
|
166
|
+
return None
|
|
167
|
+
tables = [_qualified(from_.this)] if isinstance(from_, exp.From) else []
|
|
168
|
+
conditions: list[str] = []
|
|
169
|
+
outer: list[str] = []
|
|
170
|
+
for join in joins:
|
|
171
|
+
on = join.args.get("on")
|
|
172
|
+
using = [u for u in join.args.get("using") or () if isinstance(u, Expr)]
|
|
173
|
+
join_conditions = (
|
|
174
|
+
_conjunct_texts(on, _qualified) if isinstance(on, Expr) else []
|
|
175
|
+
)
|
|
176
|
+
join_conditions.extend(f"USING {_qualified(u)}" for u in using)
|
|
177
|
+
side = str(join.args.get("side") or "").upper()
|
|
178
|
+
if side:
|
|
179
|
+
outer.append(
|
|
180
|
+
f"{side} JOIN {_qualified(join.this)} ON "
|
|
181
|
+
+ " AND ".join(sorted(join_conditions))
|
|
182
|
+
)
|
|
183
|
+
else: # inner, CROSS or plain JOIN: all the same to SQLite
|
|
184
|
+
tables.append(_qualified(join.this))
|
|
185
|
+
conditions.extend(join_conditions)
|
|
186
|
+
inner = ", ".join(sorted(tables)) + " ON " + " AND ".join(sorted(conditions))
|
|
187
|
+
return "; ".join([inner, *outer])
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _condition_text(node: Expr, text: Callable[[Expr], str]) -> str:
|
|
191
|
+
return " AND ".join(sorted(_conjunct_texts(node, text)))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _conjunct_texts(node: Expr, text: Callable[[Expr], str]) -> list[str]:
|
|
195
|
+
"""Each condition joined by AND, as text, with ``=`` sides in a fixed order."""
|
|
196
|
+
return [text(_symmetric(part, text)) for part in _conjuncts(node)]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _conjuncts(node: Expr) -> list[Expr]:
|
|
200
|
+
node = node.unnest()
|
|
201
|
+
if isinstance(node, exp.And):
|
|
202
|
+
return [*_conjuncts(node.left), *_conjuncts(node.right)]
|
|
203
|
+
return [node]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _symmetric(node: Expr, text: Callable[[Expr], str]) -> Expr:
|
|
207
|
+
"""A copy in which the two sides of every ``=`` are ordered by their text."""
|
|
208
|
+
|
|
209
|
+
def order(child: Expr) -> Expr:
|
|
210
|
+
if isinstance(child, exp.EQ) and text(child.left) > text(child.right):
|
|
211
|
+
return exp.EQ(this=child.right.copy(), expression=child.left.copy())
|
|
212
|
+
return child
|
|
213
|
+
|
|
214
|
+
return node.copy().transform(order)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _qualified(node: Expr) -> str:
|
|
218
|
+
return node.sql(dialect=DIALECT)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _clause_starting(line: str) -> Clause | None:
|
|
222
|
+
"""The clause a line starts, if it is an outer clause line."""
|
|
223
|
+
if not line or line[0].isspace():
|
|
224
|
+
return None
|
|
225
|
+
for word, clause in _CLAUSE_STARTS:
|
|
226
|
+
if line == word or line.startswith(word + " "):
|
|
227
|
+
return clause
|
|
228
|
+
if line.split(" ", 1)[0] in _JOIN_WORDS and " JOIN" in f" {line}":
|
|
229
|
+
return "from"
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _rows(result: QuerySummary) -> tuple[tuple[object, ...], ...]:
|
|
234
|
+
return result.rows if isinstance(result, ResultSample) else ()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _complete(result: ResultSample) -> bool:
|
|
238
|
+
return result.row_count <= len(result.rows)
|