text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
|
@@ -0,0 +1,595 @@
|
|
|
1
|
+
"""A single-file HTML report of one run (ADR 0007, 0038, 0039).
|
|
2
|
+
|
|
3
|
+
The page works offline: the CSS and the small filter script are inline and
|
|
4
|
+
nothing is loaded from the network. Every value goes through Jinja2 with
|
|
5
|
+
autoescaping on, so text from questions, SQL and results is shown as text,
|
|
6
|
+
never interpreted as HTML.
|
|
7
|
+
|
|
8
|
+
The template only displays. Everything it shows is prepared here as strings
|
|
9
|
+
in small dataclasses, which pyright checks; the template has no logic beyond
|
|
10
|
+
loops and ``if``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Literal, TypeVar
|
|
20
|
+
|
|
21
|
+
import jinja2
|
|
22
|
+
|
|
23
|
+
# sqlglot 30's node base class; see the note in facts/structure.py.
|
|
24
|
+
from sqlglot.expressions.core import Expr
|
|
25
|
+
|
|
26
|
+
from text2sql_eval.execution.executor import QueryFailure, SqlValue
|
|
27
|
+
from text2sql_eval.facts.output import OUTPUT_LABELS_KO, OutputDiff, Relation
|
|
28
|
+
from text2sql_eval.facts.structure import DIALECT, STRUCTURE_LABELS_KO, parse
|
|
29
|
+
from text2sql_eval.judge.ex import ExecError, FlagKind, Verdict, status_of
|
|
30
|
+
from text2sql_eval.report.highlight import (
|
|
31
|
+
Clause,
|
|
32
|
+
clause_blocks,
|
|
33
|
+
differing_clauses,
|
|
34
|
+
differing_rows,
|
|
35
|
+
)
|
|
36
|
+
from text2sql_eval.runfile import CaseRecord, QuerySummary, Run
|
|
37
|
+
|
|
38
|
+
# A bound TypeVar instead of 3.12's ``def f[K: str]`` (ADR 0009).
|
|
39
|
+
_Kind = TypeVar("_Kind", bound=str)
|
|
40
|
+
|
|
41
|
+
FlagGroup = Literal["suspect_correct", "suspect_incorrect", "info"]
|
|
42
|
+
|
|
43
|
+
FLAG_LABELS_KO: dict[FlagKind, str] = {
|
|
44
|
+
"empty_prediction_correct": "빈 예측이 정답 처리됨",
|
|
45
|
+
"gold_empty_correct": "정답 결과가 비어, 결과가 비기만 하면 정답 처리됨",
|
|
46
|
+
"matches_tied_alternative_only": "동점 대체 정답과만 일치",
|
|
47
|
+
"gold_rows_with_extra_columns": "정답 행을 그대로 포함하고 컬럼만 더 있음",
|
|
48
|
+
"gold_columns_concatenated": "정답의 여러 컬럼을 한 문자열로 이어 붙임",
|
|
49
|
+
"tied_alternative_exists": "동점 대체 정답이 있는 문제",
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
# The three groups of ADR 0034.
|
|
53
|
+
FLAG_GROUPS: dict[FlagKind, FlagGroup] = {
|
|
54
|
+
"empty_prediction_correct": "suspect_correct",
|
|
55
|
+
"gold_empty_correct": "suspect_correct",
|
|
56
|
+
"matches_tied_alternative_only": "suspect_incorrect",
|
|
57
|
+
"gold_rows_with_extra_columns": "suspect_incorrect",
|
|
58
|
+
"gold_columns_concatenated": "suspect_incorrect",
|
|
59
|
+
"tied_alternative_exists": "info",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
FLAG_GROUP_LABELS_KO: dict[FlagGroup, str] = {
|
|
63
|
+
"suspect_correct": "거짓 정답 의심",
|
|
64
|
+
"suspect_incorrect": "거짓 오답 의심",
|
|
65
|
+
"info": "정보",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
_STATUS_LABELS_KO = {"correct": "정답", "incorrect": "오답", "error": "오류"}
|
|
69
|
+
|
|
70
|
+
_PREDICTION_LABELS_KO = {
|
|
71
|
+
"given": "",
|
|
72
|
+
"empty": "빈 예측",
|
|
73
|
+
"missing": "예측 없음 (빈 SQL로 판정)",
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
_RELATION_LABELS_KO: dict[Relation, str] = {
|
|
77
|
+
"subset": "예측 행이 모두 정답에 있음 (정답의 일부만 반환)",
|
|
78
|
+
"superset": "정답 행을 모두 포함하고 행이 더 있음",
|
|
79
|
+
"overlap": "일부 행만 겹침",
|
|
80
|
+
"disjoint": "겹치는 행이 없음",
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
_DIFFICULTY_ORDER = {"simple": 0, "moderate": 1, "challenging": 2}
|
|
84
|
+
|
|
85
|
+
# Bytes values are shown as a SQL hex literal, cut after this many bytes.
|
|
86
|
+
_MAX_BYTES_SHOWN = 16
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# --- what the template shows ------------------------------------------------
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(frozen=True)
|
|
93
|
+
class Labeled:
|
|
94
|
+
"""A code such as ``empty_result`` and its Korean description."""
|
|
95
|
+
|
|
96
|
+
code: str
|
|
97
|
+
label: str
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class CountRow:
|
|
102
|
+
code: str
|
|
103
|
+
label: str
|
|
104
|
+
count: int
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True)
|
|
108
|
+
class GroupRow:
|
|
109
|
+
"""EX of one difficulty or database."""
|
|
110
|
+
|
|
111
|
+
name: str
|
|
112
|
+
cases: int
|
|
113
|
+
correct: int
|
|
114
|
+
ex: str
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(frozen=True)
|
|
118
|
+
class SqlBlock:
|
|
119
|
+
"""Lines of formatted SQL; ``differs`` marks a clause to highlight."""
|
|
120
|
+
|
|
121
|
+
text: str
|
|
122
|
+
differs: bool = False
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass(frozen=True)
|
|
126
|
+
class SqlView:
|
|
127
|
+
"""SQL formatted for reading, with the original text kept alongside.
|
|
128
|
+
|
|
129
|
+
``blocks`` is empty when there is no SQL. ``formatted`` is false when
|
|
130
|
+
sqlglot could not parse the SQL, so it is shown as written.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
blocks: tuple[SqlBlock, ...]
|
|
134
|
+
original: str
|
|
135
|
+
formatted: bool
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass(frozen=True)
|
|
139
|
+
class Cell:
|
|
140
|
+
text: str
|
|
141
|
+
null: bool = False
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class RowView:
|
|
146
|
+
cells: tuple[Cell, ...]
|
|
147
|
+
differs: bool = False
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True)
|
|
151
|
+
class TableView:
|
|
152
|
+
"""The stored sample of one query result, or why the query failed."""
|
|
153
|
+
|
|
154
|
+
columns: tuple[str, ...]
|
|
155
|
+
rows: tuple[RowView, ...]
|
|
156
|
+
caption: str
|
|
157
|
+
failure: str | None = None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass(frozen=True)
|
|
161
|
+
class FactView:
|
|
162
|
+
code: str
|
|
163
|
+
label: str
|
|
164
|
+
gold: str
|
|
165
|
+
pred: str
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass(frozen=True)
|
|
169
|
+
class FlagView:
|
|
170
|
+
code: str
|
|
171
|
+
label: str
|
|
172
|
+
group: FlagGroup
|
|
173
|
+
group_label: str
|
|
174
|
+
detail: str
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass(frozen=True)
|
|
178
|
+
class NumberRow:
|
|
179
|
+
"""One measured quantity of both results, e.g. the number of columns."""
|
|
180
|
+
|
|
181
|
+
label: str
|
|
182
|
+
gold: int
|
|
183
|
+
pred: int
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@dataclass(frozen=True)
|
|
187
|
+
class CaseView:
|
|
188
|
+
id: str
|
|
189
|
+
db_id: str
|
|
190
|
+
difficulty: str
|
|
191
|
+
status: str
|
|
192
|
+
status_label: str
|
|
193
|
+
prediction_label: str
|
|
194
|
+
question: str
|
|
195
|
+
evidence: str
|
|
196
|
+
gold_sql: SqlView
|
|
197
|
+
pred_sql: SqlView
|
|
198
|
+
error: str
|
|
199
|
+
outputs: tuple[Labeled, ...]
|
|
200
|
+
numbers: tuple[NumberRow, ...]
|
|
201
|
+
relation: str
|
|
202
|
+
facts: tuple[FactView, ...]
|
|
203
|
+
structure_note: str
|
|
204
|
+
flags: tuple[FlagView, ...]
|
|
205
|
+
gold_result: TableView
|
|
206
|
+
pred_result: TableView
|
|
207
|
+
filters: dict[str, str]
|
|
208
|
+
"""Space-separated tokens per filter key, read by the filter script."""
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@dataclass(frozen=True)
|
|
212
|
+
class FilterOption:
|
|
213
|
+
value: str
|
|
214
|
+
label: str
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclass(frozen=True)
|
|
218
|
+
class Filter:
|
|
219
|
+
key: str
|
|
220
|
+
label: str
|
|
221
|
+
options: tuple[FilterOption, ...]
|
|
222
|
+
default: str = ""
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@dataclass(frozen=True)
|
|
226
|
+
class Page:
|
|
227
|
+
title: str
|
|
228
|
+
info: tuple[tuple[str, str], ...]
|
|
229
|
+
ex: str
|
|
230
|
+
status_counts: tuple[CountRow, ...]
|
|
231
|
+
by_difficulty: tuple[GroupRow, ...]
|
|
232
|
+
by_db: tuple[GroupRow, ...]
|
|
233
|
+
output_counts: tuple[CountRow, ...]
|
|
234
|
+
structure_counts: tuple[CountRow, ...]
|
|
235
|
+
flag_counts: tuple[CountRow, ...]
|
|
236
|
+
filters: tuple[Filter, ...]
|
|
237
|
+
cases: tuple[CaseView, ...]
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# --- rendering ----------------------------------------------------------------
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def write_html_report(run: Run, path: Path, *, run_name: str) -> None:
|
|
244
|
+
"""Write the report of ``run`` to ``path`` as one UTF-8 HTML file.
|
|
245
|
+
|
|
246
|
+
``run_name`` names the run in the page title, usually the run file name.
|
|
247
|
+
|
|
248
|
+
Example:
|
|
249
|
+
>>> run_path = Path("runs/v3.json")
|
|
250
|
+
>>> write_html_report(read_run(run_path), Path("runs/v3.html"), run_name=run_path.name)
|
|
251
|
+
"""
|
|
252
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
253
|
+
path.write_text(render_html(run, run_name=run_name), encoding="utf-8")
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def render_html(run: Run, *, run_name: str) -> str:
|
|
257
|
+
"""The report of ``run`` as an HTML document."""
|
|
258
|
+
environment = jinja2.Environment(
|
|
259
|
+
loader=jinja2.PackageLoader("text2sql_eval.report", "templates"),
|
|
260
|
+
autoescape=True,
|
|
261
|
+
undefined=jinja2.StrictUndefined,
|
|
262
|
+
trim_blocks=True,
|
|
263
|
+
lstrip_blocks=True,
|
|
264
|
+
)
|
|
265
|
+
template = environment.get_template("report.html.j2")
|
|
266
|
+
return template.render(page=build_page(run, run_name=run_name))
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def build_page(run: Run, *, run_name: str) -> Page:
|
|
270
|
+
"""Everything the template shows, computed from the run."""
|
|
271
|
+
cases = tuple(_case_view(case) for case in run.cases)
|
|
272
|
+
summary = run.summary
|
|
273
|
+
return Page(
|
|
274
|
+
title=f"text2sql-eval report: {run_name}",
|
|
275
|
+
info=_info(run, run_name),
|
|
276
|
+
ex=f"{summary.ex:.1%}",
|
|
277
|
+
status_counts=(
|
|
278
|
+
CountRow("total", "전체", summary.total),
|
|
279
|
+
CountRow("correct", "정답", summary.correct),
|
|
280
|
+
CountRow("incorrect", "오답", summary.incorrect),
|
|
281
|
+
CountRow("error", "오류", summary.error),
|
|
282
|
+
),
|
|
283
|
+
by_difficulty=_groups(
|
|
284
|
+
run.cases, lambda c: c.difficulty or "(없음)", _difficulty_key
|
|
285
|
+
),
|
|
286
|
+
by_db=_groups(run.cases, lambda c: c.db_id, lambda name: (0, name)),
|
|
287
|
+
output_counts=_count_rows(_output_kinds(run.cases), OUTPUT_LABELS_KO),
|
|
288
|
+
structure_counts=_count_rows(_structure_kinds(run.cases), STRUCTURE_LABELS_KO),
|
|
289
|
+
flag_counts=_flag_counts(run.cases),
|
|
290
|
+
filters=_filters(run.cases),
|
|
291
|
+
cases=cases,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# --- summary ------------------------------------------------------------------
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _info(run: Run, run_name: str) -> tuple[tuple[str, str], ...]:
|
|
299
|
+
rows = [
|
|
300
|
+
("실행 파일", run_name),
|
|
301
|
+
("만든 시각", run.created_at),
|
|
302
|
+
("도구 버전", run.tool_version),
|
|
303
|
+
("데이터셋", run.dataset.path),
|
|
304
|
+
("예측 파일", f"{run.predictions.path} ({run.prediction_format})"),
|
|
305
|
+
("동점 대체 정답", "없음" if run.tied is None else run.tied.path),
|
|
306
|
+
("시간 제한", f"쿼리당 {run.settings.timeout_seconds:g}초"),
|
|
307
|
+
]
|
|
308
|
+
rows.extend((f"meta: {key}", value) for key, value in sorted(run.meta.items()))
|
|
309
|
+
return tuple(rows)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _difficulty_key(name: str) -> tuple[int, str]:
|
|
313
|
+
return (_DIFFICULTY_ORDER.get(name, len(_DIFFICULTY_ORDER)), name)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _groups(
|
|
317
|
+
cases: Iterable[CaseRecord],
|
|
318
|
+
name_of: Callable[[CaseRecord], str],
|
|
319
|
+
sort_key: Callable[[str], tuple[int, str]],
|
|
320
|
+
) -> tuple[GroupRow, ...]:
|
|
321
|
+
totals: Counter[str] = Counter()
|
|
322
|
+
correct: Counter[str] = Counter()
|
|
323
|
+
for case in cases:
|
|
324
|
+
name = name_of(case)
|
|
325
|
+
totals[name] += 1
|
|
326
|
+
correct[name] += status_of(case.verdict) == "correct"
|
|
327
|
+
return tuple(
|
|
328
|
+
GroupRow(
|
|
329
|
+
name, totals[name], correct[name], f"{correct[name] / totals[name]:.1%}"
|
|
330
|
+
)
|
|
331
|
+
for name in sorted(totals, key=sort_key)
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _output_kinds(cases: Iterable[CaseRecord]) -> Counter[str]:
|
|
336
|
+
return Counter(
|
|
337
|
+
kind for case in cases if case.diff for kind in case.diff.output.differences
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _structure_kinds(cases: Iterable[CaseRecord]) -> Counter[str]:
|
|
342
|
+
return Counter(
|
|
343
|
+
fact.kind for case in cases if case.diff for fact in case.diff.structure.facts
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _count_rows(
|
|
348
|
+
counts: Counter[str], labels: Mapping[_Kind, str]
|
|
349
|
+
) -> tuple[CountRow, ...]:
|
|
350
|
+
"""Every kind in ``labels``, most frequent first, ties in ``labels`` order.
|
|
351
|
+
|
|
352
|
+
Kinds that never occur are kept, with count 0, at the end.
|
|
353
|
+
"""
|
|
354
|
+
kinds = list(labels)
|
|
355
|
+
ordered = sorted(kinds, key=lambda kind: (-counts[kind], kinds.index(kind)))
|
|
356
|
+
return tuple(CountRow(kind, labels[kind], counts[kind]) for kind in ordered)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _flag_counts(cases: Iterable[CaseRecord]) -> tuple[CountRow, ...]:
|
|
360
|
+
"""Every flag kind in the order of ADR 0034, including those never raised."""
|
|
361
|
+
counts = Counter(flag.kind for case in cases for flag in case.verdict.flags)
|
|
362
|
+
return tuple(
|
|
363
|
+
CountRow(
|
|
364
|
+
kind,
|
|
365
|
+
f"{FLAG_GROUP_LABELS_KO[FLAG_GROUPS[kind]]}: {FLAG_LABELS_KO[kind]}",
|
|
366
|
+
counts[kind],
|
|
367
|
+
)
|
|
368
|
+
for kind in FLAG_LABELS_KO
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _filters(cases: tuple[CaseRecord, ...]) -> tuple[Filter, ...]:
|
|
373
|
+
databases = sorted({case.db_id for case in cases})
|
|
374
|
+
difficulties = sorted(
|
|
375
|
+
{case.difficulty or "none" for case in cases}, key=_difficulty_key
|
|
376
|
+
)
|
|
377
|
+
flags = Counter(flag.kind for case in cases for flag in case.verdict.flags)
|
|
378
|
+
flag_groups = tuple(
|
|
379
|
+
FilterOption(group, f"{label} (모두)")
|
|
380
|
+
for group, label in FLAG_GROUP_LABELS_KO.items()
|
|
381
|
+
if group != "info"
|
|
382
|
+
)
|
|
383
|
+
flag_kinds = tuple(
|
|
384
|
+
FilterOption(kind, f"{kind} {flags[kind]}")
|
|
385
|
+
for kind in FLAG_LABELS_KO
|
|
386
|
+
if flags[kind]
|
|
387
|
+
)
|
|
388
|
+
return (
|
|
389
|
+
_filter(
|
|
390
|
+
"status",
|
|
391
|
+
"판정",
|
|
392
|
+
(
|
|
393
|
+
FilterOption("wrong", "오답·오류"),
|
|
394
|
+
FilterOption("correct", "정답"),
|
|
395
|
+
FilterOption("incorrect", "오답"),
|
|
396
|
+
FilterOption("error", "오류"),
|
|
397
|
+
),
|
|
398
|
+
default="wrong",
|
|
399
|
+
),
|
|
400
|
+
_filter("db", "데이터베이스", tuple(FilterOption(db, db) for db in databases)),
|
|
401
|
+
_filter(
|
|
402
|
+
"difficulty",
|
|
403
|
+
"난이도",
|
|
404
|
+
tuple(
|
|
405
|
+
FilterOption(d, "(없음)" if d == "none" else d) for d in difficulties
|
|
406
|
+
),
|
|
407
|
+
),
|
|
408
|
+
_filter(
|
|
409
|
+
"output",
|
|
410
|
+
"출력 차이",
|
|
411
|
+
_kind_options(_output_kinds(cases), OUTPUT_LABELS_KO),
|
|
412
|
+
),
|
|
413
|
+
_filter(
|
|
414
|
+
"structure",
|
|
415
|
+
"SQL 구조 차이",
|
|
416
|
+
_kind_options(_structure_kinds(cases), STRUCTURE_LABELS_KO),
|
|
417
|
+
),
|
|
418
|
+
_filter("flag", "플래그", flag_groups + flag_kinds),
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _filter(
|
|
423
|
+
key: str, label: str, options: tuple[FilterOption, ...], default: str = ""
|
|
424
|
+
) -> Filter:
|
|
425
|
+
"""A filter whose first option, 전체 (all), turns it off."""
|
|
426
|
+
return Filter(key, label, (FilterOption("", "전체"), *options), default)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _kind_options(
|
|
430
|
+
counts: Counter[str], labels: Mapping[_Kind, str]
|
|
431
|
+
) -> tuple[FilterOption, ...]:
|
|
432
|
+
"""One option per kind that occurs, with its description and count."""
|
|
433
|
+
return tuple(
|
|
434
|
+
FilterOption(row.code, f"{row.code} ({row.label}) {row.count}")
|
|
435
|
+
for row in _count_rows(counts, labels)
|
|
436
|
+
if row.count
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# --- one case -----------------------------------------------------------------
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _case_view(case: CaseRecord) -> CaseView:
|
|
444
|
+
status = status_of(case.verdict)
|
|
445
|
+
diff = case.diff
|
|
446
|
+
flags = tuple(
|
|
447
|
+
FlagView(
|
|
448
|
+
code=flag.kind,
|
|
449
|
+
label=FLAG_LABELS_KO[flag.kind],
|
|
450
|
+
group=FLAG_GROUPS[flag.kind],
|
|
451
|
+
group_label=FLAG_GROUP_LABELS_KO[FLAG_GROUPS[flag.kind]],
|
|
452
|
+
detail=flag.detail,
|
|
453
|
+
)
|
|
454
|
+
for flag in case.verdict.flags
|
|
455
|
+
)
|
|
456
|
+
outputs = () if diff is None else diff.output.differences
|
|
457
|
+
facts = () if diff is None else diff.structure.facts
|
|
458
|
+
# Highlight where the differences are reported: cases that are not
|
|
459
|
+
# correct and whose gold query ran (ADR 0031, 0039).
|
|
460
|
+
highlight = diff is not None
|
|
461
|
+
gold_sql, pred_sql = sql_views(case.gold_sql, case.pred_sql, highlight=highlight)
|
|
462
|
+
gold_rows, pred_rows = (
|
|
463
|
+
differing_rows(case.gold, case.pred) if highlight else ((), ())
|
|
464
|
+
)
|
|
465
|
+
return CaseView(
|
|
466
|
+
id=case.id,
|
|
467
|
+
db_id=case.db_id,
|
|
468
|
+
difficulty=case.difficulty or "",
|
|
469
|
+
status=status,
|
|
470
|
+
status_label=_STATUS_LABELS_KO[status],
|
|
471
|
+
prediction_label=_PREDICTION_LABELS_KO[case.prediction_state],
|
|
472
|
+
question=case.question,
|
|
473
|
+
evidence=case.evidence,
|
|
474
|
+
gold_sql=gold_sql,
|
|
475
|
+
pred_sql=pred_sql,
|
|
476
|
+
error=_error(case.verdict),
|
|
477
|
+
outputs=tuple(Labeled(kind, OUTPUT_LABELS_KO[kind]) for kind in outputs),
|
|
478
|
+
numbers=() if diff is None else _numbers(diff.output),
|
|
479
|
+
relation=""
|
|
480
|
+
if diff is None or diff.output.relation is None
|
|
481
|
+
else _RELATION_LABELS_KO[diff.output.relation],
|
|
482
|
+
facts=tuple(
|
|
483
|
+
FactView(fact.kind, STRUCTURE_LABELS_KO[fact.kind], fact.gold, fact.pred)
|
|
484
|
+
for fact in facts
|
|
485
|
+
),
|
|
486
|
+
structure_note=_structure_note(case),
|
|
487
|
+
flags=flags,
|
|
488
|
+
gold_result=table_view(case.gold, gold_rows),
|
|
489
|
+
pred_result=table_view(case.pred, pred_rows),
|
|
490
|
+
filters={
|
|
491
|
+
"status": status if status == "correct" else f"{status} wrong",
|
|
492
|
+
"db": case.db_id,
|
|
493
|
+
"difficulty": case.difficulty or "none",
|
|
494
|
+
"output": " ".join(outputs),
|
|
495
|
+
"structure": " ".join(fact.kind for fact in facts),
|
|
496
|
+
"flag": " ".join(
|
|
497
|
+
sorted({flag.code for flag in flags} | {flag.group for flag in flags})
|
|
498
|
+
),
|
|
499
|
+
},
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def sql_views(
|
|
504
|
+
gold_sql: str, pred_sql: str, *, highlight: bool
|
|
505
|
+
) -> tuple[SqlView, SqlView]:
|
|
506
|
+
"""Both queries reformatted by sqlglot, one block per outer clause.
|
|
507
|
+
|
|
508
|
+
With ``highlight``, clauses that differ between the two are marked. SQL
|
|
509
|
+
that is empty or that sqlglot cannot parse is shown as written, and then
|
|
510
|
+
nothing is marked on either side.
|
|
511
|
+
"""
|
|
512
|
+
gold_tree = parse(gold_sql) if gold_sql.strip() else None
|
|
513
|
+
pred_tree = parse(pred_sql) if pred_sql.strip() else None
|
|
514
|
+
differs: frozenset[Clause] = frozenset()
|
|
515
|
+
if highlight and gold_tree is not None and pred_tree is not None:
|
|
516
|
+
differs = differing_clauses(gold_tree, pred_tree)
|
|
517
|
+
return (
|
|
518
|
+
_sql_view(gold_sql, gold_tree, differs),
|
|
519
|
+
_sql_view(pred_sql, pred_tree, differs),
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _sql_view(sql: str, tree: Expr | None, differs: frozenset[Clause]) -> SqlView:
|
|
524
|
+
if not sql.strip():
|
|
525
|
+
return SqlView((), sql, formatted=False)
|
|
526
|
+
if tree is None:
|
|
527
|
+
return SqlView((SqlBlock(sql.strip()),), sql, formatted=False)
|
|
528
|
+
pretty = tree.sql(dialect=DIALECT, pretty=True)
|
|
529
|
+
return SqlView(
|
|
530
|
+
tuple(
|
|
531
|
+
SqlBlock(text, differs=clause in differs)
|
|
532
|
+
for clause, text in clause_blocks(pretty)
|
|
533
|
+
),
|
|
534
|
+
sql,
|
|
535
|
+
formatted=True,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def table_view(result: QuerySummary, differs: tuple[bool, ...] = ()) -> TableView:
|
|
540
|
+
"""A result sample as a table; ``differs`` marks rows, one flag per row."""
|
|
541
|
+
if isinstance(result, QueryFailure):
|
|
542
|
+
return TableView((), (), caption="", failure=f"{result.kind}: {result.message}")
|
|
543
|
+
if result.row_count > len(result.rows):
|
|
544
|
+
caption = f"앞 {len(result.rows)}행 / 전체 {result.row_count}행"
|
|
545
|
+
else:
|
|
546
|
+
caption = f"{result.row_count}행"
|
|
547
|
+
return TableView(
|
|
548
|
+
columns=result.columns,
|
|
549
|
+
rows=tuple(
|
|
550
|
+
RowView(
|
|
551
|
+
tuple(_cell(value) for value in row),
|
|
552
|
+
differs=index < len(differs) and differs[index],
|
|
553
|
+
)
|
|
554
|
+
for index, row in enumerate(result.rows)
|
|
555
|
+
),
|
|
556
|
+
caption=caption,
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _cell(value: SqlValue) -> Cell:
|
|
561
|
+
if value is None:
|
|
562
|
+
return Cell("NULL", null=True)
|
|
563
|
+
if isinstance(value, bytes):
|
|
564
|
+
shown = value[:_MAX_BYTES_SHOWN].hex()
|
|
565
|
+
more = "…" if len(value) > _MAX_BYTES_SHOWN else ""
|
|
566
|
+
return Cell(f"x'{shown}{more}' ({len(value)} bytes)")
|
|
567
|
+
return Cell(str(value))
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _numbers(output: OutputDiff) -> tuple[NumberRow, ...]:
|
|
571
|
+
return (
|
|
572
|
+
NumberRow("컬럼 수", output.gold_columns, output.pred_columns),
|
|
573
|
+
NumberRow(
|
|
574
|
+
"서로 다른 행 수", output.gold_distinct_rows, output.pred_distinct_rows
|
|
575
|
+
),
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _structure_note(case: CaseRecord) -> str:
|
|
580
|
+
if case.diff is None:
|
|
581
|
+
return ""
|
|
582
|
+
failed = case.diff.structure.parse_failed
|
|
583
|
+
if failed is not None:
|
|
584
|
+
side = "정답" if failed == "gold" else "예측"
|
|
585
|
+
return f"{side} SQL을 파싱하지 못해 구조 차이를 구하지 않았습니다."
|
|
586
|
+
if not case.diff.structure.facts:
|
|
587
|
+
return "비교하는 다섯 가지 구조에서는 차이를 찾지 못했습니다."
|
|
588
|
+
return ""
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _error(verdict: Verdict) -> str:
|
|
592
|
+
if not isinstance(verdict, ExecError):
|
|
593
|
+
return ""
|
|
594
|
+
side = "예측" if verdict.side == "pred" else "정답"
|
|
595
|
+
return f"{side} SQL 실행 실패 ({verdict.kind}): {verdict.message}"
|