text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,204 @@
1
+ """Output differences of a wrong case: how its result differs from gold (ADR 0033).
2
+
3
+ Each check looks at one aspect of the executed results, so a case can have
4
+ several differences at once (for example a different row count and different
5
+ rows). Decided from executed rows alone, so no SQL parser is needed.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+ from itertools import permutations
12
+ from typing import Literal
13
+
14
+ from text2sql_eval.execution.executor import QueryFailure, QueryResult, Rows, SqlValue
15
+
16
+ OutputDifference = Literal[
17
+ "execution_failed",
18
+ "empty_result",
19
+ "column_mismatch",
20
+ "row_count_differs",
21
+ "single_value_differs",
22
+ "row_set_differs",
23
+ ]
24
+
25
+ Relation = Literal["subset", "superset", "disjoint", "overlap"]
26
+
27
+ OUTPUT_LABELS_KO: dict[OutputDifference, str] = {
28
+ "execution_failed": "실행 실패",
29
+ "empty_result": "빈 결과",
30
+ "column_mismatch": "컬럼 수가 다름",
31
+ "row_count_differs": "행 수가 다름",
32
+ "single_value_differs": "값 하나가 다름",
33
+ "row_set_differs": "행 내용이 다름",
34
+ }
35
+
36
+ # Beyond this many predicted columns, trying every column subset to find the
37
+ # gold rows gets expensive (n! / (n-k)!); the hint is then left unknown.
38
+ _MAX_PROJECTION_COLUMNS = 8
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class OutputDiff:
43
+ """All output differences of one wrong case, plus hints for later flags.
44
+
45
+ Row counts are distinct rows, because EX compares rows as sets.
46
+ ``gold_contained`` says whether the gold rows appear unchanged inside the
47
+ predicted rows (after projecting onto some subset and order of the
48
+ predicted columns when the widths differ); ``None`` means not applicable
49
+ or not computed. ``relation`` compares the row sets when widths match.
50
+ """
51
+
52
+ differences: tuple[OutputDifference, ...]
53
+ gold_columns: int
54
+ pred_columns: int
55
+ gold_distinct_rows: int
56
+ pred_distinct_rows: int
57
+ gold_contained: bool | None = None
58
+ relation: Relation | None = None
59
+
60
+
61
+ def compare_outputs(gold: QueryResult, pred: QueryResult) -> OutputDiff:
62
+ """Every output difference of a case already judged not correct.
63
+
64
+ - ``execution_failed``: a query failed; no other check is possible.
65
+ - ``empty_result``: the prediction returned no rows but gold did; checks of
66
+ row content are then moot, only ``column_mismatch`` can join it.
67
+ - ``column_mismatch``: the number of columns differs.
68
+ - ``row_count_differs``: the number of distinct rows differs.
69
+ - ``single_value_differs``: both results are one value (one row, one
70
+ column) and the values differ.
71
+ - ``row_set_differs``: the rows differ. With equal widths, the row sets
72
+ differ; with different widths, the gold rows are not found inside the
73
+ predicted rows under any column projection.
74
+
75
+ Example:
76
+ >>> gold = Rows(("name",), (("Surtur",),))
77
+ >>> pred = Rows(("name",), (("Surtur",), ("Ymir",)))
78
+ >>> compare_outputs(gold, pred).differences
79
+ ('row_count_differs', 'row_set_differs')
80
+ """
81
+ if isinstance(pred, QueryFailure) or isinstance(gold, QueryFailure):
82
+ return OutputDiff(
83
+ ("execution_failed",),
84
+ gold_columns=_width(gold),
85
+ pred_columns=_width(pred),
86
+ gold_distinct_rows=_distinct(gold),
87
+ pred_distinct_rows=_distinct(pred),
88
+ )
89
+ gold_set, pred_set = set(gold.rows), set(pred.rows)
90
+ gold_width, pred_width = len(gold.columns), len(pred.columns)
91
+ found: list[OutputDifference] = []
92
+ contained: bool | None = None
93
+ relation: Relation | None = None
94
+
95
+ if gold_set and not pred_set:
96
+ found.append("empty_result")
97
+ # Zero columns only comes from empty SQL (no prediction); comparing
98
+ # column counts would call a missing answer a "column" problem.
99
+ if pred_width and gold_width != pred_width:
100
+ found.append("column_mismatch")
101
+ else:
102
+ if gold_width != pred_width:
103
+ found.append("column_mismatch")
104
+ contained = gold_rows_in_projection(gold_set, pred)
105
+ else:
106
+ relation = _relation(gold_set, pred_set)
107
+ contained = relation == "superset"
108
+ if not gold_set:
109
+ contained = (
110
+ None # "gold rows are contained" says nothing when there are none
111
+ )
112
+ if len(gold_set) != len(pred_set):
113
+ found.append("row_count_differs")
114
+ if _single_value(gold, gold_set) and _single_value(pred, pred_set):
115
+ found.append("single_value_differs")
116
+ elif gold_width == pred_width and gold_set != pred_set:
117
+ found.append("row_set_differs")
118
+ elif gold_width != pred_width and contained is False:
119
+ # None (too many columns to check) claims nothing about the rows.
120
+ found.append("row_set_differs")
121
+
122
+ return OutputDiff(
123
+ tuple(found),
124
+ gold_columns=gold_width,
125
+ pred_columns=pred_width,
126
+ gold_distinct_rows=len(gold_set),
127
+ pred_distinct_rows=len(pred_set),
128
+ gold_contained=contained,
129
+ relation=relation,
130
+ )
131
+
132
+
133
+ def _single_value(result: Rows, distinct: set[tuple[SqlValue, ...]]) -> bool:
134
+ return len(result.columns) == 1 and len(distinct) == 1
135
+
136
+
137
+ def _relation(
138
+ gold_set: set[tuple[SqlValue, ...]], pred_set: set[tuple[SqlValue, ...]]
139
+ ) -> Relation | None:
140
+ if pred_set == gold_set:
141
+ return None
142
+ if pred_set < gold_set:
143
+ return "subset"
144
+ if pred_set > gold_set:
145
+ return "superset"
146
+ if pred_set & gold_set:
147
+ return "overlap"
148
+ return "disjoint"
149
+
150
+
151
+ def gold_rows_in_projection(
152
+ gold_set: set[tuple[SqlValue, ...]], pred: Rows
153
+ ) -> bool | None:
154
+ """Whether some subset and order of ``pred``'s columns gives exactly the gold rows.
155
+
156
+ ``None`` when ``pred`` is too wide to try every projection.
157
+ """
158
+ width = len(pred.columns)
159
+ gold_width = len(next(iter(gold_set), ()))
160
+ if gold_width == 0 or gold_width > width:
161
+ return False
162
+ if width > _MAX_PROJECTION_COLUMNS:
163
+ return None
164
+ for columns in permutations(range(width), gold_width):
165
+ projected = {tuple(row[index] for index in columns) for row in pred.rows}
166
+ if projected == gold_set:
167
+ return True
168
+ return False
169
+
170
+
171
+ # Separators tried when checking whether one predicted column joins several
172
+ # gold columns, as in first_name || ' ' || last_name.
173
+ _CONCAT_SEPARATORS = (" ", ", ", "")
174
+
175
+
176
+ def concatenates_gold_columns(gold: Rows, pred: Rows) -> bool:
177
+ """Whether ``pred``'s single column is each gold row's values joined into one string.
178
+
179
+ The same separator must work for every row; gold rows with NULLs are not
180
+ considered, since SQL concatenation with NULL yields NULL.
181
+
182
+ Example:
183
+ >>> gold = Rows(("first_name", "last_name"), (("Grant", "Gilmour"),))
184
+ >>> concatenates_gold_columns(gold, Rows(("full_name",), (("Grant Gilmour",),)))
185
+ True
186
+ """
187
+ if len(gold.columns) < 2 or len(pred.columns) != 1 or not gold.rows:
188
+ return False
189
+ if any(value is None for row in gold.rows for value in row):
190
+ return False
191
+ pred_values = {str(row[0]) for row in pred.rows}
192
+ return any(
193
+ pred_values
194
+ == {separator.join(str(value) for value in row) for row in gold.rows}
195
+ for separator in _CONCAT_SEPARATORS
196
+ )
197
+
198
+
199
+ def _width(result: QueryResult) -> int:
200
+ return len(result.columns) if isinstance(result, Rows) else 0
201
+
202
+
203
+ def _distinct(result: QueryResult) -> int:
204
+ return len(set(result.rows)) if isinstance(result, Rows) else 0
@@ -0,0 +1,299 @@
1
+ """SQL structure differences between gold and predicted SQL (ADR 0031, 0033).
2
+
3
+ Both queries are parsed with sqlglot (ADR 0032) and reduced to a small
4
+ ``Shape``; the differences come from comparing shapes. Normalization is
5
+ deliberately minimal: identifier case, whitespace, a trailing semicolon and
6
+ table aliases. String literals keep their case, because a value that differs
7
+ only by case is exactly the kind of fact worth reporting.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Literal
14
+
15
+ import sqlglot
16
+ from sqlglot import exp
17
+ from sqlglot.errors import SqlglotError
18
+
19
+ # sqlglot 30's node base class, defined in expressions.core and star-imported
20
+ # without __all__, which pyright does not treat as a re-export.
21
+ from sqlglot.expressions.core import Expr
22
+
23
+ DIALECT = "sqlite"
24
+
25
+ FactKind = Literal[
26
+ "where_value", "tables_joins", "columns", "aggregation", "order_limit"
27
+ ]
28
+
29
+ STRUCTURE_LABELS_KO: dict[FactKind, str] = {
30
+ "where_value": "WHERE 조건의 값이 다름",
31
+ "tables_joins": "테이블·JOIN 구성이 다름",
32
+ "columns": "사용한 컬럼이 다름",
33
+ "aggregation": "집계 함수가 다름",
34
+ "order_limit": "ORDER BY/LIMIT이 다름",
35
+ }
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class Fact:
40
+ """One difference, with the evidence from each side as readable SQL fragments."""
41
+
42
+ kind: FactKind
43
+ gold: str
44
+ pred: str
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class StructureReport:
49
+ """Facts for one case, or the reason none could be produced."""
50
+
51
+ facts: tuple[Fact, ...]
52
+ parse_failed: Literal["gold", "pred"] | None = None
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class Condition:
57
+ """A WHERE comparison between a column and literal value(s)."""
58
+
59
+ column: str # unqualified, lower-cased
60
+ operator: str
61
+ value: str # the literal side as SQL, case preserved
62
+ text: str # the whole condition, normalized, for evidence
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Shape:
67
+ tables: frozenset[str]
68
+ joins: int
69
+ columns: frozenset[str] # unqualified, lower-cased, select aliases removed
70
+ conditions: tuple[Condition, ...]
71
+ aggregates: frozenset[str]
72
+ order: str | None
73
+ limit: str | None
74
+
75
+
76
+ def structural_facts(gold_sql: str, pred_sql: str) -> StructureReport:
77
+ """Compare two queries and report the v1 structural facts.
78
+
79
+ Example:
80
+ >>> report = structural_facts(
81
+ ... "SELECT id FROM superpower WHERE power_name = 'Cryokinesis'",
82
+ ... "SELECT id FROM superpower WHERE power_name = 'cryokinesis';",
83
+ ... )
84
+ >>> report.facts
85
+ (Fact(kind='where_value', gold="power_name = 'Cryokinesis'", pred="power_name = 'cryokinesis'"),)
86
+ """
87
+ gold = parse(gold_sql)
88
+ if gold is None:
89
+ return StructureReport((), parse_failed="gold")
90
+ pred = parse(pred_sql)
91
+ if pred is None:
92
+ return StructureReport((), parse_failed="pred")
93
+ return StructureReport(tuple(compare(shape_of(gold), shape_of(pred))))
94
+
95
+
96
+ def parse(sql: str) -> Expr | None:
97
+ """Parse one statement, or ``None`` when sqlglot cannot."""
98
+ try:
99
+ return sqlglot.parse_one(sql, read=DIALECT)
100
+ except SqlglotError:
101
+ return None
102
+
103
+
104
+ def shape_of(query: Expr) -> Shape:
105
+ """Reduce a parsed query to the parts the facts compare."""
106
+ query = normalize(query)
107
+ cte_names = {cte.alias_or_name.lower() for cte in query.find_all(exp.CTE)}
108
+ tables = frozenset(
109
+ table.name.lower()
110
+ for table in query.find_all(exp.Table)
111
+ if table.name and table.name.lower() not in cte_names
112
+ )
113
+ joins = sum(1 for _ in query.find_all(exp.Join))
114
+ # A select alias used later (ORDER BY percentage) parses as a column; it
115
+ # names an output, not a table column, so it is not counted.
116
+ aliases = {alias.alias.lower() for alias in query.find_all(exp.Alias)}
117
+ columns = frozenset(
118
+ column.name.lower()
119
+ for column in query.find_all(exp.Column)
120
+ if column.name and column.name.lower() not in aliases
121
+ )
122
+ conditions = tuple(
123
+ condition
124
+ for where in query.find_all(exp.Where)
125
+ for condition in _conditions(where)
126
+ )
127
+ aggregates = frozenset(unqualified(agg) for agg in query.find_all(exp.AggFunc))
128
+ order = query.args.get("order")
129
+ limit = query.args.get("limit")
130
+ return Shape(
131
+ tables=tables,
132
+ joins=joins,
133
+ columns=columns,
134
+ conditions=conditions,
135
+ aggregates=aggregates,
136
+ order=unqualified(order) if isinstance(order, Expr) else None,
137
+ limit=unqualified(limit) if isinstance(limit, Expr) else None,
138
+ )
139
+
140
+
141
+ def compare(gold: Shape, pred: Shape) -> list[Fact]:
142
+ """The facts that distinguish two shapes, in a fixed order."""
143
+ facts: list[Fact] = []
144
+ facts.extend(_where_value_facts(gold, pred))
145
+ if gold.tables != pred.tables or gold.joins != pred.joins:
146
+ facts.append(
147
+ Fact(
148
+ "tables_joins",
149
+ gold=_describe_tables(gold),
150
+ pred=_describe_tables(pred),
151
+ )
152
+ )
153
+ if gold.columns != pred.columns:
154
+ # Only the columns one side uses and the other does not.
155
+ facts.append(
156
+ Fact(
157
+ "columns",
158
+ gold=", ".join(sorted(gold.columns - pred.columns)) or "(none)",
159
+ pred=", ".join(sorted(pred.columns - gold.columns)) or "(none)",
160
+ )
161
+ )
162
+ if gold.aggregates != pred.aggregates:
163
+ facts.append(
164
+ Fact(
165
+ "aggregation",
166
+ gold=", ".join(sorted(gold.aggregates)) or "(none)",
167
+ pred=", ".join(sorted(pred.aggregates)) or "(none)",
168
+ )
169
+ )
170
+ if (gold.order, gold.limit) != (pred.order, pred.limit):
171
+ facts.append(
172
+ Fact(
173
+ "order_limit",
174
+ gold=_describe_order(gold),
175
+ pred=_describe_order(pred),
176
+ )
177
+ )
178
+ return facts
179
+
180
+
181
+ def _where_value_facts(gold: Shape, pred: Shape) -> list[Fact]:
182
+ """Same column and operator on both sides, different value."""
183
+ facts: list[Fact] = []
184
+ pred_by_key = {(c.column, c.operator): c for c in pred.conditions}
185
+ for condition in gold.conditions:
186
+ other = pred_by_key.get((condition.column, condition.operator))
187
+ if other is not None and other.value != condition.value:
188
+ facts.append(Fact("where_value", gold=condition.text, pred=other.text))
189
+ return facts
190
+
191
+
192
+ def _conditions(where: exp.Where) -> list[Condition]:
193
+ found: list[Condition] = []
194
+ for predicate in where.this.find_all(exp.Predicate):
195
+ if isinstance(predicate, exp.In):
196
+ column = predicate.this
197
+ values = predicate.expressions
198
+ if not isinstance(column, exp.Column) or not values:
199
+ continue # IN (subquery): no literal values to compare
200
+ value = ", ".join(v.sql(dialect=DIALECT) for v in values)
201
+ elif isinstance(predicate, exp.Binary):
202
+ column, other = _column_and_value(predicate)
203
+ if column is None or other is None:
204
+ continue
205
+ value = other.sql(dialect=DIALECT)
206
+ else:
207
+ continue
208
+ found.append(
209
+ Condition(
210
+ column=column.name.lower(),
211
+ operator=type(predicate).__name__.upper(),
212
+ value=value,
213
+ text=predicate.sql(dialect=DIALECT),
214
+ )
215
+ )
216
+ return found
217
+
218
+
219
+ def _column_and_value(
220
+ predicate: exp.Binary,
221
+ ) -> tuple[exp.Column | None, Expr | None]:
222
+ """Split ``col op literal`` (either order); ``None`` if it is not that shape."""
223
+ left, right = predicate.left, predicate.right
224
+ for column_side, value_side in ((left, right), (right, left)):
225
+ column = _column_inside(column_side)
226
+ if (
227
+ column is not None
228
+ and _has_literal(value_side)
229
+ and not _has_column(value_side)
230
+ ):
231
+ return column, value_side
232
+ return None, None
233
+
234
+
235
+ def _column_inside(node: Expr) -> exp.Column | None:
236
+ """A column, possibly wrapped in a function such as ``LOWER(col)``."""
237
+ if isinstance(node, exp.Column):
238
+ return node
239
+ columns = list(node.find_all(exp.Column))
240
+ return columns[0] if len(columns) == 1 and not _has_subquery(node) else None
241
+
242
+
243
+ def _has_literal(node: Expr) -> bool:
244
+ return node.find(exp.Literal) is not None
245
+
246
+
247
+ def _has_column(node: Expr) -> bool:
248
+ return node.find(exp.Column) is not None
249
+
250
+
251
+ def _has_subquery(node: Expr) -> bool:
252
+ return node.find(exp.Select) is not None
253
+
254
+
255
+ def normalize(query: Expr) -> Expr:
256
+ """Resolve table aliases to table names and lower-case identifiers."""
257
+ aliases = {
258
+ table.alias.lower(): table.name.lower()
259
+ for table in query.find_all(exp.Table)
260
+ if table.alias
261
+ }
262
+
263
+ def rewrite(node: Expr) -> Expr:
264
+ if isinstance(node, exp.Column) and node.table:
265
+ real = aliases.get(node.table.lower())
266
+ if real is not None:
267
+ node.set("table", exp.to_identifier(real))
268
+ if isinstance(node, exp.Table) and node.alias:
269
+ node.set("alias", None)
270
+ if isinstance(node, exp.Identifier):
271
+ node.set("this", node.this.lower())
272
+ return node
273
+
274
+ return query.copy().transform(rewrite)
275
+
276
+
277
+ def unqualified(node: Expr) -> str:
278
+ """SQL of ``node`` with table qualifiers dropped from columns.
279
+
280
+ ``COUNT(bond_id)`` and ``COUNT(bond.bond_id)`` name the same thing in the
281
+ aggregate and ORDER BY comparisons; a qualifier is not a difference.
282
+ """
283
+
284
+ def strip(child: Expr) -> Expr:
285
+ if isinstance(child, exp.Column):
286
+ child.set("table", None)
287
+ return child
288
+
289
+ return node.copy().transform(strip).sql(dialect=DIALECT)
290
+
291
+
292
+ def _describe_tables(shape: Shape) -> str:
293
+ tables = ", ".join(sorted(shape.tables)) or "(none)"
294
+ return f"tables: {tables}; joins: {shape.joins}"
295
+
296
+
297
+ def _describe_order(shape: Shape) -> str:
298
+ parts = [part for part in (shape.order, shape.limit) if part]
299
+ return " ".join(parts) if parts else "(none)"
@@ -0,0 +1,5 @@
1
+ """Execution-accuracy judging and reliability flags.
2
+
3
+ Not part of the public API yet; the public surface is decided once ``eval``
4
+ works end to end.
5
+ """
@@ -0,0 +1,179 @@
1
+ """Judge execution accuracy (EX) exactly as BIRD's official script does.
2
+
3
+ The official rule is ``set(predicted_rows) == set(gold_rows)``: row order and
4
+ duplicate rows are ignored, the order of values inside a row matters, column
5
+ names are ignored, and values compare with Python ``==`` (so ``1 == 1.0``).
6
+ A query that fails scores zero (ADR 0022).
7
+
8
+ Weaknesses of that rule are surfaced as flags, never by changing the verdict
9
+ (ADR 0023, ADR 0024).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from collections.abc import Sequence
15
+ from dataclasses import dataclass
16
+ from typing import Literal, assert_never
17
+
18
+ from text2sql_eval.execution.executor import QueryFailure, QueryResult, Rows, SqlValue
19
+ from text2sql_eval.facts.output import (
20
+ concatenates_gold_columns,
21
+ gold_rows_in_projection,
22
+ )
23
+
24
+ FlagKind = Literal[
25
+ "empty_prediction_correct",
26
+ "gold_empty_correct",
27
+ "matches_tied_alternative_only",
28
+ "gold_rows_with_extra_columns",
29
+ "gold_columns_concatenated",
30
+ "tied_alternative_exists",
31
+ ]
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class Flag:
36
+ """A reason to doubt or double-check a verdict. It never changes the verdict."""
37
+
38
+ kind: FlagKind
39
+ detail: str
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class Correct:
44
+ """The prediction's rows equal the gold rows as sets."""
45
+
46
+ flags: tuple[Flag, ...] = ()
47
+
48
+
49
+ @dataclass(frozen=True)
50
+ class Incorrect:
51
+ """Both queries ran, and their rows differ as sets."""
52
+
53
+ flags: tuple[Flag, ...] = ()
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class ExecError:
58
+ """A query failed, so the case scores zero.
59
+
60
+ ``side`` says whose query failed. When both fail, the prediction is
61
+ reported, because the official script runs it first.
62
+ """
63
+
64
+ side: Literal["pred", "gold"]
65
+ kind: Literal["error", "timeout", "row_limit"]
66
+ message: str
67
+ flags: tuple[Flag, ...] = ()
68
+
69
+
70
+ Verdict = Correct | Incorrect | ExecError
71
+
72
+ _EMPTY_PREDICTION_CORRECT = Flag(
73
+ "empty_prediction_correct",
74
+ "the model gave no SQL; it counts as correct only because the gold result "
75
+ "is empty too",
76
+ )
77
+ _GOLD_EMPTY_CORRECT = Flag(
78
+ "gold_empty_correct",
79
+ "the gold result is empty, so any query returning no rows counts as correct",
80
+ )
81
+ _GOLD_ROWS_WITH_EXTRA_COLUMNS = Flag(
82
+ "gold_rows_with_extra_columns",
83
+ "the prediction returns every gold row plus extra columns; it may be a "
84
+ "correct answer judged wrong",
85
+ )
86
+ _GOLD_COLUMNS_CONCATENATED = Flag(
87
+ "gold_columns_concatenated",
88
+ "the prediction joins the gold columns into one text column; it may be a "
89
+ "correct answer in a different format",
90
+ )
91
+ _TIED_ALTERNATIVE_EXISTS = Flag(
92
+ "tied_alternative_exists",
93
+ "the dataset has an alternative gold SQL that returns all tied rows; "
94
+ "the original gold picks only some of them",
95
+ )
96
+ _MATCHES_TIED_ALTERNATIVE_ONLY = Flag(
97
+ "matches_tied_alternative_only",
98
+ "the prediction matches the tied-rows alternative gold but not the "
99
+ "original gold; it may be a correct answer judged wrong",
100
+ )
101
+
102
+
103
+ def official_match(
104
+ pred_rows: Sequence[tuple[SqlValue, ...]], gold_rows: Sequence[tuple[SqlValue, ...]]
105
+ ) -> bool:
106
+ """BIRD's comparison, verbatim: ``set(predicted_res) == set(ground_truth_res)``."""
107
+ return set(pred_rows) == set(gold_rows)
108
+
109
+
110
+ def judge_case(
111
+ gold: QueryResult,
112
+ pred: QueryResult,
113
+ *,
114
+ pred_is_empty: bool,
115
+ tied: QueryResult | None = None,
116
+ ) -> Verdict:
117
+ """Judge one case from the executed gold and predicted queries.
118
+
119
+ Args:
120
+ gold: Result of the gold SQL.
121
+ pred: Result of the predicted SQL. An empty or missing prediction is
122
+ executed as empty SQL, like the official script does.
123
+ pred_is_empty: Whether the model gave no SQL (ADR 0023).
124
+ tied: Result of the tied-rows alternative gold SQL, if the dataset has
125
+ one for this case (ADR 0024).
126
+
127
+ Example:
128
+ >>> gold = Rows(columns=("n",), rows=((1,), (2,)))
129
+ >>> judge_case(gold, Rows(columns=("x",), rows=((2,), (1,), (1,))), pred_is_empty=False)
130
+ Correct(flags=())
131
+ """
132
+ flags: list[Flag] = []
133
+ if tied is not None:
134
+ flags.append(_TIED_ALTERNATIVE_EXISTS)
135
+
136
+ # The official script runs the prediction first, so its failure wins.
137
+ if isinstance(pred, QueryFailure):
138
+ return ExecError("pred", pred.kind, pred.message, tuple(flags))
139
+ if isinstance(gold, QueryFailure):
140
+ return ExecError("gold", gold.kind, gold.message, tuple(flags))
141
+
142
+ if official_match(pred.rows, gold.rows):
143
+ if pred_is_empty:
144
+ flags.append(_EMPTY_PREDICTION_CORRECT)
145
+ elif not gold.rows:
146
+ flags.append(_GOLD_EMPTY_CORRECT)
147
+ return Correct(flags=tuple(flags))
148
+
149
+ if isinstance(tied, Rows) and official_match(pred.rows, tied.rows):
150
+ flags.append(_MATCHES_TIED_ALTERNATIVE_ONLY)
151
+ gold_set = set(gold.rows)
152
+ if (
153
+ gold_set
154
+ and len(pred.columns) > len(gold.columns)
155
+ and gold_rows_in_projection(gold_set, pred) is True
156
+ ):
157
+ flags.append(_GOLD_ROWS_WITH_EXTRA_COLUMNS)
158
+ if concatenates_gold_columns(gold, pred):
159
+ flags.append(_GOLD_COLUMNS_CONCATENATED)
160
+ return Incorrect(flags=tuple(flags))
161
+
162
+
163
+ def status_of(verdict: Verdict) -> Literal["correct", "incorrect", "error"]:
164
+ """The verdict's status name, as written to run files and reports.
165
+
166
+ Example:
167
+ >>> status_of(Correct()), status_of(ExecError("pred", "timeout", "30 s"))
168
+ ('correct', 'error')
169
+ """
170
+ match verdict:
171
+ case Correct():
172
+ return "correct"
173
+ case Incorrect():
174
+ return "incorrect"
175
+ case ExecError():
176
+ return "error"
177
+ case _:
178
+ # pyright proves this unreachable; a new Verdict type fails here.
179
+ assert_never(verdict)
text2sql_eval/py.typed ADDED
File without changes
@@ -0,0 +1,4 @@
1
+ """Reports of one run: an HTML page and a CSV file (ADR 0007).
2
+
3
+ Not part of the public API yet.
4
+ """