text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Output differences of a wrong case: how its result differs from gold (ADR 0033).
|
|
2
|
+
|
|
3
|
+
Each check looks at one aspect of the executed results, so a case can have
|
|
4
|
+
several differences at once (for example a different row count and different
|
|
5
|
+
rows). Decided from executed rows alone, so no SQL parser is needed.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from itertools import permutations
|
|
12
|
+
from typing import Literal
|
|
13
|
+
|
|
14
|
+
from text2sql_eval.execution.executor import QueryFailure, QueryResult, Rows, SqlValue
|
|
15
|
+
|
|
16
|
+
OutputDifference = Literal[
|
|
17
|
+
"execution_failed",
|
|
18
|
+
"empty_result",
|
|
19
|
+
"column_mismatch",
|
|
20
|
+
"row_count_differs",
|
|
21
|
+
"single_value_differs",
|
|
22
|
+
"row_set_differs",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
Relation = Literal["subset", "superset", "disjoint", "overlap"]
|
|
26
|
+
|
|
27
|
+
OUTPUT_LABELS_KO: dict[OutputDifference, str] = {
|
|
28
|
+
"execution_failed": "실행 실패",
|
|
29
|
+
"empty_result": "빈 결과",
|
|
30
|
+
"column_mismatch": "컬럼 수가 다름",
|
|
31
|
+
"row_count_differs": "행 수가 다름",
|
|
32
|
+
"single_value_differs": "값 하나가 다름",
|
|
33
|
+
"row_set_differs": "행 내용이 다름",
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
# Beyond this many predicted columns, trying every column subset to find the
|
|
37
|
+
# gold rows gets expensive (n! / (n-k)!); the hint is then left unknown.
|
|
38
|
+
_MAX_PROJECTION_COLUMNS = 8
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class OutputDiff:
|
|
43
|
+
"""All output differences of one wrong case, plus hints for later flags.
|
|
44
|
+
|
|
45
|
+
Row counts are distinct rows, because EX compares rows as sets.
|
|
46
|
+
``gold_contained`` says whether the gold rows appear unchanged inside the
|
|
47
|
+
predicted rows (after projecting onto some subset and order of the
|
|
48
|
+
predicted columns when the widths differ); ``None`` means not applicable
|
|
49
|
+
or not computed. ``relation`` compares the row sets when widths match.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
differences: tuple[OutputDifference, ...]
|
|
53
|
+
gold_columns: int
|
|
54
|
+
pred_columns: int
|
|
55
|
+
gold_distinct_rows: int
|
|
56
|
+
pred_distinct_rows: int
|
|
57
|
+
gold_contained: bool | None = None
|
|
58
|
+
relation: Relation | None = None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def compare_outputs(gold: QueryResult, pred: QueryResult) -> OutputDiff:
|
|
62
|
+
"""Every output difference of a case already judged not correct.
|
|
63
|
+
|
|
64
|
+
- ``execution_failed``: a query failed; no other check is possible.
|
|
65
|
+
- ``empty_result``: the prediction returned no rows but gold did; checks of
|
|
66
|
+
row content are then moot, only ``column_mismatch`` can join it.
|
|
67
|
+
- ``column_mismatch``: the number of columns differs.
|
|
68
|
+
- ``row_count_differs``: the number of distinct rows differs.
|
|
69
|
+
- ``single_value_differs``: both results are one value (one row, one
|
|
70
|
+
column) and the values differ.
|
|
71
|
+
- ``row_set_differs``: the rows differ. With equal widths, the row sets
|
|
72
|
+
differ; with different widths, the gold rows are not found inside the
|
|
73
|
+
predicted rows under any column projection.
|
|
74
|
+
|
|
75
|
+
Example:
|
|
76
|
+
>>> gold = Rows(("name",), (("Surtur",),))
|
|
77
|
+
>>> pred = Rows(("name",), (("Surtur",), ("Ymir",)))
|
|
78
|
+
>>> compare_outputs(gold, pred).differences
|
|
79
|
+
('row_count_differs', 'row_set_differs')
|
|
80
|
+
"""
|
|
81
|
+
if isinstance(pred, QueryFailure) or isinstance(gold, QueryFailure):
|
|
82
|
+
return OutputDiff(
|
|
83
|
+
("execution_failed",),
|
|
84
|
+
gold_columns=_width(gold),
|
|
85
|
+
pred_columns=_width(pred),
|
|
86
|
+
gold_distinct_rows=_distinct(gold),
|
|
87
|
+
pred_distinct_rows=_distinct(pred),
|
|
88
|
+
)
|
|
89
|
+
gold_set, pred_set = set(gold.rows), set(pred.rows)
|
|
90
|
+
gold_width, pred_width = len(gold.columns), len(pred.columns)
|
|
91
|
+
found: list[OutputDifference] = []
|
|
92
|
+
contained: bool | None = None
|
|
93
|
+
relation: Relation | None = None
|
|
94
|
+
|
|
95
|
+
if gold_set and not pred_set:
|
|
96
|
+
found.append("empty_result")
|
|
97
|
+
# Zero columns only comes from empty SQL (no prediction); comparing
|
|
98
|
+
# column counts would call a missing answer a "column" problem.
|
|
99
|
+
if pred_width and gold_width != pred_width:
|
|
100
|
+
found.append("column_mismatch")
|
|
101
|
+
else:
|
|
102
|
+
if gold_width != pred_width:
|
|
103
|
+
found.append("column_mismatch")
|
|
104
|
+
contained = gold_rows_in_projection(gold_set, pred)
|
|
105
|
+
else:
|
|
106
|
+
relation = _relation(gold_set, pred_set)
|
|
107
|
+
contained = relation == "superset"
|
|
108
|
+
if not gold_set:
|
|
109
|
+
contained = (
|
|
110
|
+
None # "gold rows are contained" says nothing when there are none
|
|
111
|
+
)
|
|
112
|
+
if len(gold_set) != len(pred_set):
|
|
113
|
+
found.append("row_count_differs")
|
|
114
|
+
if _single_value(gold, gold_set) and _single_value(pred, pred_set):
|
|
115
|
+
found.append("single_value_differs")
|
|
116
|
+
elif gold_width == pred_width and gold_set != pred_set:
|
|
117
|
+
found.append("row_set_differs")
|
|
118
|
+
elif gold_width != pred_width and contained is False:
|
|
119
|
+
# None (too many columns to check) claims nothing about the rows.
|
|
120
|
+
found.append("row_set_differs")
|
|
121
|
+
|
|
122
|
+
return OutputDiff(
|
|
123
|
+
tuple(found),
|
|
124
|
+
gold_columns=gold_width,
|
|
125
|
+
pred_columns=pred_width,
|
|
126
|
+
gold_distinct_rows=len(gold_set),
|
|
127
|
+
pred_distinct_rows=len(pred_set),
|
|
128
|
+
gold_contained=contained,
|
|
129
|
+
relation=relation,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _single_value(result: Rows, distinct: set[tuple[SqlValue, ...]]) -> bool:
|
|
134
|
+
return len(result.columns) == 1 and len(distinct) == 1
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _relation(
|
|
138
|
+
gold_set: set[tuple[SqlValue, ...]], pred_set: set[tuple[SqlValue, ...]]
|
|
139
|
+
) -> Relation | None:
|
|
140
|
+
if pred_set == gold_set:
|
|
141
|
+
return None
|
|
142
|
+
if pred_set < gold_set:
|
|
143
|
+
return "subset"
|
|
144
|
+
if pred_set > gold_set:
|
|
145
|
+
return "superset"
|
|
146
|
+
if pred_set & gold_set:
|
|
147
|
+
return "overlap"
|
|
148
|
+
return "disjoint"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def gold_rows_in_projection(
|
|
152
|
+
gold_set: set[tuple[SqlValue, ...]], pred: Rows
|
|
153
|
+
) -> bool | None:
|
|
154
|
+
"""Whether some subset and order of ``pred``'s columns gives exactly the gold rows.
|
|
155
|
+
|
|
156
|
+
``None`` when ``pred`` is too wide to try every projection.
|
|
157
|
+
"""
|
|
158
|
+
width = len(pred.columns)
|
|
159
|
+
gold_width = len(next(iter(gold_set), ()))
|
|
160
|
+
if gold_width == 0 or gold_width > width:
|
|
161
|
+
return False
|
|
162
|
+
if width > _MAX_PROJECTION_COLUMNS:
|
|
163
|
+
return None
|
|
164
|
+
for columns in permutations(range(width), gold_width):
|
|
165
|
+
projected = {tuple(row[index] for index in columns) for row in pred.rows}
|
|
166
|
+
if projected == gold_set:
|
|
167
|
+
return True
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# Separators tried when checking whether one predicted column joins several
|
|
172
|
+
# gold columns, as in first_name || ' ' || last_name.
|
|
173
|
+
_CONCAT_SEPARATORS = (" ", ", ", "")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def concatenates_gold_columns(gold: Rows, pred: Rows) -> bool:
|
|
177
|
+
"""Whether ``pred``'s single column is each gold row's values joined into one string.
|
|
178
|
+
|
|
179
|
+
The same separator must work for every row; gold rows with NULLs are not
|
|
180
|
+
considered, since SQL concatenation with NULL yields NULL.
|
|
181
|
+
|
|
182
|
+
Example:
|
|
183
|
+
>>> gold = Rows(("first_name", "last_name"), (("Grant", "Gilmour"),))
|
|
184
|
+
>>> concatenates_gold_columns(gold, Rows(("full_name",), (("Grant Gilmour",),)))
|
|
185
|
+
True
|
|
186
|
+
"""
|
|
187
|
+
if len(gold.columns) < 2 or len(pred.columns) != 1 or not gold.rows:
|
|
188
|
+
return False
|
|
189
|
+
if any(value is None for row in gold.rows for value in row):
|
|
190
|
+
return False
|
|
191
|
+
pred_values = {str(row[0]) for row in pred.rows}
|
|
192
|
+
return any(
|
|
193
|
+
pred_values
|
|
194
|
+
== {separator.join(str(value) for value in row) for row in gold.rows}
|
|
195
|
+
for separator in _CONCAT_SEPARATORS
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _width(result: QueryResult) -> int:
|
|
200
|
+
return len(result.columns) if isinstance(result, Rows) else 0
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _distinct(result: QueryResult) -> int:
|
|
204
|
+
return len(set(result.rows)) if isinstance(result, Rows) else 0
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""SQL structure differences between gold and predicted SQL (ADR 0031, 0033).
|
|
2
|
+
|
|
3
|
+
Both queries are parsed with sqlglot (ADR 0032) and reduced to a small
|
|
4
|
+
``Shape``; the differences come from comparing shapes. Normalization is
|
|
5
|
+
deliberately minimal: identifier case, whitespace, a trailing semicolon and
|
|
6
|
+
table aliases. String literals keep their case, because a value that differs
|
|
7
|
+
only by case is exactly the kind of fact worth reporting.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
import sqlglot
|
|
16
|
+
from sqlglot import exp
|
|
17
|
+
from sqlglot.errors import SqlglotError
|
|
18
|
+
|
|
19
|
+
# sqlglot 30's node base class, defined in expressions.core and star-imported
|
|
20
|
+
# without __all__, which pyright does not treat as a re-export.
|
|
21
|
+
from sqlglot.expressions.core import Expr
|
|
22
|
+
|
|
23
|
+
DIALECT = "sqlite"
|
|
24
|
+
|
|
25
|
+
FactKind = Literal[
|
|
26
|
+
"where_value", "tables_joins", "columns", "aggregation", "order_limit"
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
STRUCTURE_LABELS_KO: dict[FactKind, str] = {
|
|
30
|
+
"where_value": "WHERE 조건의 값이 다름",
|
|
31
|
+
"tables_joins": "테이블·JOIN 구성이 다름",
|
|
32
|
+
"columns": "사용한 컬럼이 다름",
|
|
33
|
+
"aggregation": "집계 함수가 다름",
|
|
34
|
+
"order_limit": "ORDER BY/LIMIT이 다름",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Fact:
|
|
40
|
+
"""One difference, with the evidence from each side as readable SQL fragments."""
|
|
41
|
+
|
|
42
|
+
kind: FactKind
|
|
43
|
+
gold: str
|
|
44
|
+
pred: str
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class StructureReport:
|
|
49
|
+
"""Facts for one case, or the reason none could be produced."""
|
|
50
|
+
|
|
51
|
+
facts: tuple[Fact, ...]
|
|
52
|
+
parse_failed: Literal["gold", "pred"] | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class Condition:
|
|
57
|
+
"""A WHERE comparison between a column and literal value(s)."""
|
|
58
|
+
|
|
59
|
+
column: str # unqualified, lower-cased
|
|
60
|
+
operator: str
|
|
61
|
+
value: str # the literal side as SQL, case preserved
|
|
62
|
+
text: str # the whole condition, normalized, for evidence
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Shape:
|
|
67
|
+
tables: frozenset[str]
|
|
68
|
+
joins: int
|
|
69
|
+
columns: frozenset[str] # unqualified, lower-cased, select aliases removed
|
|
70
|
+
conditions: tuple[Condition, ...]
|
|
71
|
+
aggregates: frozenset[str]
|
|
72
|
+
order: str | None
|
|
73
|
+
limit: str | None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def structural_facts(gold_sql: str, pred_sql: str) -> StructureReport:
|
|
77
|
+
"""Compare two queries and report the v1 structural facts.
|
|
78
|
+
|
|
79
|
+
Example:
|
|
80
|
+
>>> report = structural_facts(
|
|
81
|
+
... "SELECT id FROM superpower WHERE power_name = 'Cryokinesis'",
|
|
82
|
+
... "SELECT id FROM superpower WHERE power_name = 'cryokinesis';",
|
|
83
|
+
... )
|
|
84
|
+
>>> report.facts
|
|
85
|
+
(Fact(kind='where_value', gold="power_name = 'Cryokinesis'", pred="power_name = 'cryokinesis'"),)
|
|
86
|
+
"""
|
|
87
|
+
gold = parse(gold_sql)
|
|
88
|
+
if gold is None:
|
|
89
|
+
return StructureReport((), parse_failed="gold")
|
|
90
|
+
pred = parse(pred_sql)
|
|
91
|
+
if pred is None:
|
|
92
|
+
return StructureReport((), parse_failed="pred")
|
|
93
|
+
return StructureReport(tuple(compare(shape_of(gold), shape_of(pred))))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def parse(sql: str) -> Expr | None:
|
|
97
|
+
"""Parse one statement, or ``None`` when sqlglot cannot."""
|
|
98
|
+
try:
|
|
99
|
+
return sqlglot.parse_one(sql, read=DIALECT)
|
|
100
|
+
except SqlglotError:
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def shape_of(query: Expr) -> Shape:
|
|
105
|
+
"""Reduce a parsed query to the parts the facts compare."""
|
|
106
|
+
query = normalize(query)
|
|
107
|
+
cte_names = {cte.alias_or_name.lower() for cte in query.find_all(exp.CTE)}
|
|
108
|
+
tables = frozenset(
|
|
109
|
+
table.name.lower()
|
|
110
|
+
for table in query.find_all(exp.Table)
|
|
111
|
+
if table.name and table.name.lower() not in cte_names
|
|
112
|
+
)
|
|
113
|
+
joins = sum(1 for _ in query.find_all(exp.Join))
|
|
114
|
+
# A select alias used later (ORDER BY percentage) parses as a column; it
|
|
115
|
+
# names an output, not a table column, so it is not counted.
|
|
116
|
+
aliases = {alias.alias.lower() for alias in query.find_all(exp.Alias)}
|
|
117
|
+
columns = frozenset(
|
|
118
|
+
column.name.lower()
|
|
119
|
+
for column in query.find_all(exp.Column)
|
|
120
|
+
if column.name and column.name.lower() not in aliases
|
|
121
|
+
)
|
|
122
|
+
conditions = tuple(
|
|
123
|
+
condition
|
|
124
|
+
for where in query.find_all(exp.Where)
|
|
125
|
+
for condition in _conditions(where)
|
|
126
|
+
)
|
|
127
|
+
aggregates = frozenset(unqualified(agg) for agg in query.find_all(exp.AggFunc))
|
|
128
|
+
order = query.args.get("order")
|
|
129
|
+
limit = query.args.get("limit")
|
|
130
|
+
return Shape(
|
|
131
|
+
tables=tables,
|
|
132
|
+
joins=joins,
|
|
133
|
+
columns=columns,
|
|
134
|
+
conditions=conditions,
|
|
135
|
+
aggregates=aggregates,
|
|
136
|
+
order=unqualified(order) if isinstance(order, Expr) else None,
|
|
137
|
+
limit=unqualified(limit) if isinstance(limit, Expr) else None,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def compare(gold: Shape, pred: Shape) -> list[Fact]:
|
|
142
|
+
"""The facts that distinguish two shapes, in a fixed order."""
|
|
143
|
+
facts: list[Fact] = []
|
|
144
|
+
facts.extend(_where_value_facts(gold, pred))
|
|
145
|
+
if gold.tables != pred.tables or gold.joins != pred.joins:
|
|
146
|
+
facts.append(
|
|
147
|
+
Fact(
|
|
148
|
+
"tables_joins",
|
|
149
|
+
gold=_describe_tables(gold),
|
|
150
|
+
pred=_describe_tables(pred),
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
if gold.columns != pred.columns:
|
|
154
|
+
# Only the columns one side uses and the other does not.
|
|
155
|
+
facts.append(
|
|
156
|
+
Fact(
|
|
157
|
+
"columns",
|
|
158
|
+
gold=", ".join(sorted(gold.columns - pred.columns)) or "(none)",
|
|
159
|
+
pred=", ".join(sorted(pred.columns - gold.columns)) or "(none)",
|
|
160
|
+
)
|
|
161
|
+
)
|
|
162
|
+
if gold.aggregates != pred.aggregates:
|
|
163
|
+
facts.append(
|
|
164
|
+
Fact(
|
|
165
|
+
"aggregation",
|
|
166
|
+
gold=", ".join(sorted(gold.aggregates)) or "(none)",
|
|
167
|
+
pred=", ".join(sorted(pred.aggregates)) or "(none)",
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
if (gold.order, gold.limit) != (pred.order, pred.limit):
|
|
171
|
+
facts.append(
|
|
172
|
+
Fact(
|
|
173
|
+
"order_limit",
|
|
174
|
+
gold=_describe_order(gold),
|
|
175
|
+
pred=_describe_order(pred),
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
return facts
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _where_value_facts(gold: Shape, pred: Shape) -> list[Fact]:
|
|
182
|
+
"""Same column and operator on both sides, different value."""
|
|
183
|
+
facts: list[Fact] = []
|
|
184
|
+
pred_by_key = {(c.column, c.operator): c for c in pred.conditions}
|
|
185
|
+
for condition in gold.conditions:
|
|
186
|
+
other = pred_by_key.get((condition.column, condition.operator))
|
|
187
|
+
if other is not None and other.value != condition.value:
|
|
188
|
+
facts.append(Fact("where_value", gold=condition.text, pred=other.text))
|
|
189
|
+
return facts
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _conditions(where: exp.Where) -> list[Condition]:
|
|
193
|
+
found: list[Condition] = []
|
|
194
|
+
for predicate in where.this.find_all(exp.Predicate):
|
|
195
|
+
if isinstance(predicate, exp.In):
|
|
196
|
+
column = predicate.this
|
|
197
|
+
values = predicate.expressions
|
|
198
|
+
if not isinstance(column, exp.Column) or not values:
|
|
199
|
+
continue # IN (subquery): no literal values to compare
|
|
200
|
+
value = ", ".join(v.sql(dialect=DIALECT) for v in values)
|
|
201
|
+
elif isinstance(predicate, exp.Binary):
|
|
202
|
+
column, other = _column_and_value(predicate)
|
|
203
|
+
if column is None or other is None:
|
|
204
|
+
continue
|
|
205
|
+
value = other.sql(dialect=DIALECT)
|
|
206
|
+
else:
|
|
207
|
+
continue
|
|
208
|
+
found.append(
|
|
209
|
+
Condition(
|
|
210
|
+
column=column.name.lower(),
|
|
211
|
+
operator=type(predicate).__name__.upper(),
|
|
212
|
+
value=value,
|
|
213
|
+
text=predicate.sql(dialect=DIALECT),
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
return found
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _column_and_value(
|
|
220
|
+
predicate: exp.Binary,
|
|
221
|
+
) -> tuple[exp.Column | None, Expr | None]:
|
|
222
|
+
"""Split ``col op literal`` (either order); ``None`` if it is not that shape."""
|
|
223
|
+
left, right = predicate.left, predicate.right
|
|
224
|
+
for column_side, value_side in ((left, right), (right, left)):
|
|
225
|
+
column = _column_inside(column_side)
|
|
226
|
+
if (
|
|
227
|
+
column is not None
|
|
228
|
+
and _has_literal(value_side)
|
|
229
|
+
and not _has_column(value_side)
|
|
230
|
+
):
|
|
231
|
+
return column, value_side
|
|
232
|
+
return None, None
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _column_inside(node: Expr) -> exp.Column | None:
|
|
236
|
+
"""A column, possibly wrapped in a function such as ``LOWER(col)``."""
|
|
237
|
+
if isinstance(node, exp.Column):
|
|
238
|
+
return node
|
|
239
|
+
columns = list(node.find_all(exp.Column))
|
|
240
|
+
return columns[0] if len(columns) == 1 and not _has_subquery(node) else None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _has_literal(node: Expr) -> bool:
|
|
244
|
+
return node.find(exp.Literal) is not None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _has_column(node: Expr) -> bool:
|
|
248
|
+
return node.find(exp.Column) is not None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _has_subquery(node: Expr) -> bool:
|
|
252
|
+
return node.find(exp.Select) is not None
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def normalize(query: Expr) -> Expr:
|
|
256
|
+
"""Resolve table aliases to table names and lower-case identifiers."""
|
|
257
|
+
aliases = {
|
|
258
|
+
table.alias.lower(): table.name.lower()
|
|
259
|
+
for table in query.find_all(exp.Table)
|
|
260
|
+
if table.alias
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
def rewrite(node: Expr) -> Expr:
|
|
264
|
+
if isinstance(node, exp.Column) and node.table:
|
|
265
|
+
real = aliases.get(node.table.lower())
|
|
266
|
+
if real is not None:
|
|
267
|
+
node.set("table", exp.to_identifier(real))
|
|
268
|
+
if isinstance(node, exp.Table) and node.alias:
|
|
269
|
+
node.set("alias", None)
|
|
270
|
+
if isinstance(node, exp.Identifier):
|
|
271
|
+
node.set("this", node.this.lower())
|
|
272
|
+
return node
|
|
273
|
+
|
|
274
|
+
return query.copy().transform(rewrite)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def unqualified(node: Expr) -> str:
|
|
278
|
+
"""SQL of ``node`` with table qualifiers dropped from columns.
|
|
279
|
+
|
|
280
|
+
``COUNT(bond_id)`` and ``COUNT(bond.bond_id)`` name the same thing in the
|
|
281
|
+
aggregate and ORDER BY comparisons; a qualifier is not a difference.
|
|
282
|
+
"""
|
|
283
|
+
|
|
284
|
+
def strip(child: Expr) -> Expr:
|
|
285
|
+
if isinstance(child, exp.Column):
|
|
286
|
+
child.set("table", None)
|
|
287
|
+
return child
|
|
288
|
+
|
|
289
|
+
return node.copy().transform(strip).sql(dialect=DIALECT)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _describe_tables(shape: Shape) -> str:
|
|
293
|
+
tables = ", ".join(sorted(shape.tables)) or "(none)"
|
|
294
|
+
return f"tables: {tables}; joins: {shape.joins}"
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _describe_order(shape: Shape) -> str:
|
|
298
|
+
parts = [part for part in (shape.order, shape.limit) if part]
|
|
299
|
+
return " ".join(parts) if parts else "(none)"
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Judge execution accuracy (EX) exactly as BIRD's official script does.
|
|
2
|
+
|
|
3
|
+
The official rule is ``set(predicted_rows) == set(gold_rows)``: row order and
|
|
4
|
+
duplicate rows are ignored, the order of values inside a row matters, column
|
|
5
|
+
names are ignored, and values compare with Python ``==`` (so ``1 == 1.0``).
|
|
6
|
+
A query that fails scores zero (ADR 0022).
|
|
7
|
+
|
|
8
|
+
Weaknesses of that rule are surfaced as flags, never by changing the verdict
|
|
9
|
+
(ADR 0023, ADR 0024).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Literal, assert_never
|
|
17
|
+
|
|
18
|
+
from text2sql_eval.execution.executor import QueryFailure, QueryResult, Rows, SqlValue
|
|
19
|
+
from text2sql_eval.facts.output import (
|
|
20
|
+
concatenates_gold_columns,
|
|
21
|
+
gold_rows_in_projection,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
FlagKind = Literal[
|
|
25
|
+
"empty_prediction_correct",
|
|
26
|
+
"gold_empty_correct",
|
|
27
|
+
"matches_tied_alternative_only",
|
|
28
|
+
"gold_rows_with_extra_columns",
|
|
29
|
+
"gold_columns_concatenated",
|
|
30
|
+
"tied_alternative_exists",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Flag:
|
|
36
|
+
"""A reason to doubt or double-check a verdict. It never changes the verdict."""
|
|
37
|
+
|
|
38
|
+
kind: FlagKind
|
|
39
|
+
detail: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Correct:
|
|
44
|
+
"""The prediction's rows equal the gold rows as sets."""
|
|
45
|
+
|
|
46
|
+
flags: tuple[Flag, ...] = ()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class Incorrect:
|
|
51
|
+
"""Both queries ran, and their rows differ as sets."""
|
|
52
|
+
|
|
53
|
+
flags: tuple[Flag, ...] = ()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class ExecError:
|
|
58
|
+
"""A query failed, so the case scores zero.
|
|
59
|
+
|
|
60
|
+
``side`` says whose query failed. When both fail, the prediction is
|
|
61
|
+
reported, because the official script runs it first.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
side: Literal["pred", "gold"]
|
|
65
|
+
kind: Literal["error", "timeout", "row_limit"]
|
|
66
|
+
message: str
|
|
67
|
+
flags: tuple[Flag, ...] = ()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
Verdict = Correct | Incorrect | ExecError
|
|
71
|
+
|
|
72
|
+
_EMPTY_PREDICTION_CORRECT = Flag(
|
|
73
|
+
"empty_prediction_correct",
|
|
74
|
+
"the model gave no SQL; it counts as correct only because the gold result "
|
|
75
|
+
"is empty too",
|
|
76
|
+
)
|
|
77
|
+
_GOLD_EMPTY_CORRECT = Flag(
|
|
78
|
+
"gold_empty_correct",
|
|
79
|
+
"the gold result is empty, so any query returning no rows counts as correct",
|
|
80
|
+
)
|
|
81
|
+
_GOLD_ROWS_WITH_EXTRA_COLUMNS = Flag(
|
|
82
|
+
"gold_rows_with_extra_columns",
|
|
83
|
+
"the prediction returns every gold row plus extra columns; it may be a "
|
|
84
|
+
"correct answer judged wrong",
|
|
85
|
+
)
|
|
86
|
+
_GOLD_COLUMNS_CONCATENATED = Flag(
|
|
87
|
+
"gold_columns_concatenated",
|
|
88
|
+
"the prediction joins the gold columns into one text column; it may be a "
|
|
89
|
+
"correct answer in a different format",
|
|
90
|
+
)
|
|
91
|
+
_TIED_ALTERNATIVE_EXISTS = Flag(
|
|
92
|
+
"tied_alternative_exists",
|
|
93
|
+
"the dataset has an alternative gold SQL that returns all tied rows; "
|
|
94
|
+
"the original gold picks only some of them",
|
|
95
|
+
)
|
|
96
|
+
_MATCHES_TIED_ALTERNATIVE_ONLY = Flag(
|
|
97
|
+
"matches_tied_alternative_only",
|
|
98
|
+
"the prediction matches the tied-rows alternative gold but not the "
|
|
99
|
+
"original gold; it may be a correct answer judged wrong",
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def official_match(
|
|
104
|
+
pred_rows: Sequence[tuple[SqlValue, ...]], gold_rows: Sequence[tuple[SqlValue, ...]]
|
|
105
|
+
) -> bool:
|
|
106
|
+
"""BIRD's comparison, verbatim: ``set(predicted_res) == set(ground_truth_res)``."""
|
|
107
|
+
return set(pred_rows) == set(gold_rows)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def judge_case(
|
|
111
|
+
gold: QueryResult,
|
|
112
|
+
pred: QueryResult,
|
|
113
|
+
*,
|
|
114
|
+
pred_is_empty: bool,
|
|
115
|
+
tied: QueryResult | None = None,
|
|
116
|
+
) -> Verdict:
|
|
117
|
+
"""Judge one case from the executed gold and predicted queries.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
gold: Result of the gold SQL.
|
|
121
|
+
pred: Result of the predicted SQL. An empty or missing prediction is
|
|
122
|
+
executed as empty SQL, like the official script does.
|
|
123
|
+
pred_is_empty: Whether the model gave no SQL (ADR 0023).
|
|
124
|
+
tied: Result of the tied-rows alternative gold SQL, if the dataset has
|
|
125
|
+
one for this case (ADR 0024).
|
|
126
|
+
|
|
127
|
+
Example:
|
|
128
|
+
>>> gold = Rows(columns=("n",), rows=((1,), (2,)))
|
|
129
|
+
>>> judge_case(gold, Rows(columns=("x",), rows=((2,), (1,), (1,))), pred_is_empty=False)
|
|
130
|
+
Correct(flags=())
|
|
131
|
+
"""
|
|
132
|
+
flags: list[Flag] = []
|
|
133
|
+
if tied is not None:
|
|
134
|
+
flags.append(_TIED_ALTERNATIVE_EXISTS)
|
|
135
|
+
|
|
136
|
+
# The official script runs the prediction first, so its failure wins.
|
|
137
|
+
if isinstance(pred, QueryFailure):
|
|
138
|
+
return ExecError("pred", pred.kind, pred.message, tuple(flags))
|
|
139
|
+
if isinstance(gold, QueryFailure):
|
|
140
|
+
return ExecError("gold", gold.kind, gold.message, tuple(flags))
|
|
141
|
+
|
|
142
|
+
if official_match(pred.rows, gold.rows):
|
|
143
|
+
if pred_is_empty:
|
|
144
|
+
flags.append(_EMPTY_PREDICTION_CORRECT)
|
|
145
|
+
elif not gold.rows:
|
|
146
|
+
flags.append(_GOLD_EMPTY_CORRECT)
|
|
147
|
+
return Correct(flags=tuple(flags))
|
|
148
|
+
|
|
149
|
+
if isinstance(tied, Rows) and official_match(pred.rows, tied.rows):
|
|
150
|
+
flags.append(_MATCHES_TIED_ALTERNATIVE_ONLY)
|
|
151
|
+
gold_set = set(gold.rows)
|
|
152
|
+
if (
|
|
153
|
+
gold_set
|
|
154
|
+
and len(pred.columns) > len(gold.columns)
|
|
155
|
+
and gold_rows_in_projection(gold_set, pred) is True
|
|
156
|
+
):
|
|
157
|
+
flags.append(_GOLD_ROWS_WITH_EXTRA_COLUMNS)
|
|
158
|
+
if concatenates_gold_columns(gold, pred):
|
|
159
|
+
flags.append(_GOLD_COLUMNS_CONCATENATED)
|
|
160
|
+
return Incorrect(flags=tuple(flags))
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def status_of(verdict: Verdict) -> Literal["correct", "incorrect", "error"]:
|
|
164
|
+
"""The verdict's status name, as written to run files and reports.
|
|
165
|
+
|
|
166
|
+
Example:
|
|
167
|
+
>>> status_of(Correct()), status_of(ExecError("pred", "timeout", "30 s"))
|
|
168
|
+
('correct', 'error')
|
|
169
|
+
"""
|
|
170
|
+
match verdict:
|
|
171
|
+
case Correct():
|
|
172
|
+
return "correct"
|
|
173
|
+
case Incorrect():
|
|
174
|
+
return "incorrect"
|
|
175
|
+
case ExecError():
|
|
176
|
+
return "error"
|
|
177
|
+
case _:
|
|
178
|
+
# pyright proves this unreachable; a new Verdict type fails here.
|
|
179
|
+
assert_never(verdict)
|
text2sql_eval/py.typed
ADDED
|
File without changes
|