text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,81 @@
1
+ """One CSV row per case of a run, for spreadsheet analysis (ADR 0007).
2
+
3
+ Researchers used to classify wrong cases by hand in a spreadsheet. This file
4
+ starts that sheet with the verdict, the differences and the flags already
5
+ filled in, and leaves ``my_category`` and ``notes`` empty for the user.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import csv
11
+ from pathlib import Path
12
+
13
+ from text2sql_eval.csv_safety import excel_safe
14
+ from text2sql_eval.judge.ex import ExecError, Verdict, status_of
15
+ from text2sql_eval.runfile import CaseRecord, Run
16
+
17
+ CSV_COLUMNS = (
18
+ "id",
19
+ "db_id",
20
+ "difficulty",
21
+ "status",
22
+ "prediction",
23
+ "output_differences",
24
+ "sql_structure_differences",
25
+ "flags",
26
+ "question",
27
+ "evidence",
28
+ "gold_sql",
29
+ "pred_sql",
30
+ "error",
31
+ "my_category",
32
+ "notes",
33
+ )
34
+
35
+
36
+ def write_report_csv(run: Run, path: Path) -> None:
37
+ """Write every case of ``run`` as one CSV row, in the run's order.
38
+
39
+ Written as UTF-8 with a byte order mark so Excel shows Korean text and
40
+ other non-ASCII characters correctly. Multiple differences or flags are
41
+ joined with ``;``. ``prediction`` is ``given``, ``empty`` or ``missing``.
42
+ A cell that starts like a formula (``=``, ``-`` ...) gets a leading ``'``
43
+ so Excel shows it as text (ADR 0040).
44
+
45
+ Example:
46
+ >>> write_report_csv(read_run(Path("runs/v3.json")), Path("runs/v3.csv"))
47
+ """
48
+ path.parent.mkdir(parents=True, exist_ok=True)
49
+ with path.open("w", encoding="utf-8-sig", newline="") as file:
50
+ writer = csv.writer(file)
51
+ writer.writerow(CSV_COLUMNS)
52
+ for case in run.cases:
53
+ writer.writerow(excel_safe(cell) for cell in _row(case))
54
+
55
+
56
+ def _row(case: CaseRecord) -> tuple[str, ...]:
57
+ outputs = () if case.diff is None else case.diff.output.differences
58
+ facts = () if case.diff is None else case.diff.structure.facts
59
+ return (
60
+ case.id,
61
+ case.db_id,
62
+ case.difficulty or "",
63
+ status_of(case.verdict),
64
+ case.prediction_state,
65
+ ";".join(outputs),
66
+ ";".join(fact.kind for fact in facts),
67
+ ";".join(flag.kind for flag in case.verdict.flags),
68
+ case.question,
69
+ case.evidence,
70
+ case.gold_sql,
71
+ case.pred_sql,
72
+ _error(case.verdict),
73
+ "",
74
+ "",
75
+ )
76
+
77
+
78
+ def _error(verdict: Verdict) -> str:
79
+ if isinstance(verdict, ExecError):
80
+ return f"{verdict.side} {verdict.kind}: {verdict.message}"
81
+ return ""
@@ -0,0 +1,238 @@
1
+ """What to highlight in the report: differing SQL clauses and result rows (ADR 0039).
2
+
3
+ Highlighting is display only. It never changes a verdict, an output
4
+ difference, a SQL structure difference or a flag.
5
+
6
+ Clauses are compared after the same normalization as the SQL structure
7
+ differences (table aliases resolved, identifiers lower-cased, column
8
+ qualifiers dropped), so ``T1.name`` and ``s.name`` are not a difference.
9
+ Writing order that SQLite ignores is not a difference either: the order of
10
+ conditions joined by AND, the two sides of ``=``, the order of inner joins,
11
+ and ``INNER JOIN`` versus ``JOIN``. Only the outer query's clauses are
12
+ compared; a subquery belongs to the clause it sits in.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from collections.abc import Callable
18
+ from typing import Literal
19
+
20
+ from sqlglot import exp
21
+
22
+ # sqlglot 30's node base class; see the note in facts/structure.py.
23
+ from sqlglot.expressions.core import Expr
24
+
25
+ from text2sql_eval.facts.structure import DIALECT, normalize, unqualified
26
+ from text2sql_eval.runfile import QuerySummary, ResultSample
27
+
28
+ Clause = Literal["with", "select", "from", "where", "group", "having", "order", "limit"]
29
+
30
+ # First words of the lines that start each clause in sqlglot's pretty output.
31
+ # FROM and every kind of JOIN form one clause, as do LIMIT and OFFSET.
32
+ _CLAUSE_STARTS: tuple[tuple[str, Clause], ...] = (
33
+ ("WITH", "with"),
34
+ ("SELECT", "select"),
35
+ ("FROM", "from"),
36
+ ("WHERE", "where"),
37
+ ("GROUP BY", "group"),
38
+ ("HAVING", "having"),
39
+ ("ORDER BY", "order"),
40
+ ("LIMIT", "limit"),
41
+ ("OFFSET", "limit"),
42
+ )
43
+ # Clauses held in one node under the same name in sqlglot's Select.args;
44
+ # WHERE and HAVING are compared as conditions instead.
45
+ _SINGLE_NODE_CLAUSES: tuple[Clause, ...] = ("group", "order")
46
+ _JOIN_WORDS = frozenset({"JOIN", "INNER", "LEFT", "RIGHT", "FULL", "CROSS", "NATURAL"})
47
+
48
+
49
+ def differing_clauses(gold: Expr, pred: Expr) -> frozenset[Clause]:
50
+ """Clauses whose normalized SQL differs, or that only one query has.
51
+
52
+ Empty when either query is not a plain SELECT (for example a UNION),
53
+ because its clauses cannot be lined up with the other query's.
54
+
55
+ Example:
56
+ >>> differing_clauses(
57
+ ... parse("SELECT T1.name FROM hero AS T1 WHERE T1.power = 'Fly'"),
58
+ ... parse("SELECT h.name FROM hero AS h WHERE h.power = 'fly'"),
59
+ ... )
60
+ frozenset({'where'})
61
+ """
62
+ if not isinstance(gold, exp.Select) or not isinstance(pred, exp.Select):
63
+ return frozenset()
64
+ gold_parts = _clause_texts(gold)
65
+ pred_parts = _clause_texts(pred)
66
+ return frozenset(
67
+ clause
68
+ for clause in gold_parts.keys() | pred_parts.keys()
69
+ if gold_parts.get(clause) != pred_parts.get(clause)
70
+ )
71
+
72
+
73
+ def clause_blocks(pretty_sql: str) -> list[tuple[Clause | None, str]]:
74
+ """Split sqlglot's pretty output into one block per outer clause.
75
+
76
+ sqlglot starts each outer clause at the beginning of a line and indents
77
+ everything inside it. Lines before the first known clause get ``None``.
78
+
79
+ Example:
80
+ >>> clause_blocks("SELECT\n name\nFROM hero\nJOIN power\n ON hero.id = power.id")
81
+ [('select', 'SELECT\n name'), ('from', 'FROM hero\nJOIN power\n ON hero.id = power.id')]
82
+ """
83
+ blocks: list[tuple[Clause | None, list[str]]] = []
84
+ for line in pretty_sql.split("\n"):
85
+ clause = _clause_starting(line)
86
+ if clause is not None and (not blocks or blocks[-1][0] != clause):
87
+ blocks.append((clause, [line]))
88
+ elif blocks:
89
+ blocks[-1][1].append(line)
90
+ else:
91
+ blocks.append((None, [line]))
92
+ return [(clause, "\n".join(lines)) for clause, lines in blocks]
93
+
94
+
95
+ def differing_rows(
96
+ gold: QuerySummary, pred: QuerySummary
97
+ ) -> tuple[tuple[bool, ...], tuple[bool, ...]]:
98
+ """For each stored row of each side, whether the other result lacks it.
99
+
100
+ The run file keeps only the first rows of each result. A row missing from
101
+ the other side's sample may still be in its full result, so a side's rows
102
+ are marked only when the other side's sample is the complete result. Rows
103
+ are compared like EX does (as tuples, ``1 == 1.0``), and only when both
104
+ results have the same number of columns.
105
+
106
+ Example:
107
+ >>> gold = ResultSample(("name",), (("Surtur",), ("Ymir",)), row_count=2)
108
+ >>> pred = ResultSample(("name",), (("Surtur",), ("Thor",)), row_count=2)
109
+ >>> differing_rows(gold, pred)
110
+ ((False, True), (False, True))
111
+ """
112
+ gold_marks = tuple(False for _ in _rows(gold))
113
+ pred_marks = tuple(False for _ in _rows(pred))
114
+ if not isinstance(gold, ResultSample) or not isinstance(pred, ResultSample):
115
+ return gold_marks, pred_marks
116
+ if len(gold.columns) != len(pred.columns) or not gold.columns:
117
+ return gold_marks, pred_marks
118
+ if _complete(pred):
119
+ pred_rows = set(pred.rows)
120
+ gold_marks = tuple(row not in pred_rows for row in gold.rows)
121
+ if _complete(gold):
122
+ gold_rows = set(gold.rows)
123
+ pred_marks = tuple(row not in gold_rows for row in pred.rows)
124
+ return gold_marks, pred_marks
125
+
126
+
127
+ def _clause_texts(select: exp.Select) -> dict[Clause, str]:
128
+ """Normalized SQL of each clause the query has."""
129
+ query = normalize(select)
130
+ parts: dict[Clause, str] = {}
131
+ with_ = query.args.get("with_")
132
+ if isinstance(with_, Expr):
133
+ parts["with"] = unqualified(with_)
134
+ # Output column names (AS ...) do not matter to EX, so aliases are dropped.
135
+ columns = ", ".join(unqualified(e.unalias()) for e in query.expressions)
136
+ distinct = "DISTINCT " if query.args.get("distinct") else ""
137
+ parts["select"] = distinct + columns
138
+ sources = _sources_text(query)
139
+ if sources is not None:
140
+ parts["from"] = sources
141
+ for clause in ("where", "having"):
142
+ node = query.args.get(clause)
143
+ if isinstance(node, Expr):
144
+ parts[clause] = _condition_text(node.this, unqualified)
145
+ for clause in _SINGLE_NODE_CLAUSES:
146
+ node = query.args.get(clause)
147
+ if isinstance(node, Expr):
148
+ parts[clause] = unqualified(node)
149
+ limits = [query.args.get("limit"), query.args.get("offset")]
150
+ if any(isinstance(node, Expr) for node in limits):
151
+ parts["limit"] = " ".join(
152
+ unqualified(node) for node in limits if isinstance(node, Expr)
153
+ )
154
+ return parts
155
+
156
+
157
+ def _sources_text(query: Expr) -> str | None:
158
+ """FROM and JOINs, where the order of inner joins does not matter.
159
+
160
+ Join conditions keep their table qualifiers: in ``ON`` they say which
161
+ tables are joined. LEFT, RIGHT and FULL joins keep their written order.
162
+ """
163
+ from_ = query.args.get("from_")
164
+ joins = [j for j in query.args.get("joins") or () if isinstance(j, exp.Join)]
165
+ if not isinstance(from_, exp.From) and not joins:
166
+ return None
167
+ tables = [_qualified(from_.this)] if isinstance(from_, exp.From) else []
168
+ conditions: list[str] = []
169
+ outer: list[str] = []
170
+ for join in joins:
171
+ on = join.args.get("on")
172
+ using = [u for u in join.args.get("using") or () if isinstance(u, Expr)]
173
+ join_conditions = (
174
+ _conjunct_texts(on, _qualified) if isinstance(on, Expr) else []
175
+ )
176
+ join_conditions.extend(f"USING {_qualified(u)}" for u in using)
177
+ side = str(join.args.get("side") or "").upper()
178
+ if side:
179
+ outer.append(
180
+ f"{side} JOIN {_qualified(join.this)} ON "
181
+ + " AND ".join(sorted(join_conditions))
182
+ )
183
+ else: # inner, CROSS or plain JOIN: all the same to SQLite
184
+ tables.append(_qualified(join.this))
185
+ conditions.extend(join_conditions)
186
+ inner = ", ".join(sorted(tables)) + " ON " + " AND ".join(sorted(conditions))
187
+ return "; ".join([inner, *outer])
188
+
189
+
190
+ def _condition_text(node: Expr, text: Callable[[Expr], str]) -> str:
191
+ return " AND ".join(sorted(_conjunct_texts(node, text)))
192
+
193
+
194
+ def _conjunct_texts(node: Expr, text: Callable[[Expr], str]) -> list[str]:
195
+ """Each condition joined by AND, as text, with ``=`` sides in a fixed order."""
196
+ return [text(_symmetric(part, text)) for part in _conjuncts(node)]
197
+
198
+
199
+ def _conjuncts(node: Expr) -> list[Expr]:
200
+ node = node.unnest()
201
+ if isinstance(node, exp.And):
202
+ return [*_conjuncts(node.left), *_conjuncts(node.right)]
203
+ return [node]
204
+
205
+
206
+ def _symmetric(node: Expr, text: Callable[[Expr], str]) -> Expr:
207
+ """A copy in which the two sides of every ``=`` are ordered by their text."""
208
+
209
+ def order(child: Expr) -> Expr:
210
+ if isinstance(child, exp.EQ) and text(child.left) > text(child.right):
211
+ return exp.EQ(this=child.right.copy(), expression=child.left.copy())
212
+ return child
213
+
214
+ return node.copy().transform(order)
215
+
216
+
217
+ def _qualified(node: Expr) -> str:
218
+ return node.sql(dialect=DIALECT)
219
+
220
+
221
+ def _clause_starting(line: str) -> Clause | None:
222
+ """The clause a line starts, if it is an outer clause line."""
223
+ if not line or line[0].isspace():
224
+ return None
225
+ for word, clause in _CLAUSE_STARTS:
226
+ if line == word or line.startswith(word + " "):
227
+ return clause
228
+ if line.split(" ", 1)[0] in _JOIN_WORDS and " JOIN" in f" {line}":
229
+ return "from"
230
+ return None
231
+
232
+
233
+ def _rows(result: QuerySummary) -> tuple[tuple[object, ...], ...]:
234
+ return result.rows if isinstance(result, ResultSample) else ()
235
+
236
+
237
+ def _complete(result: ResultSample) -> bool:
238
+ return result.row_count <= len(result.rows)