text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,595 @@
1
+ """A single-file HTML report of one run (ADR 0007, 0038, 0039).
2
+
3
+ The page works offline: the CSS and the small filter script are inline and
4
+ nothing is loaded from the network. Every value goes through Jinja2 with
5
+ autoescaping on, so text from questions, SQL and results is shown as text,
6
+ never interpreted as HTML.
7
+
8
+ The template only displays. Everything it shows is prepared here as strings
9
+ in small dataclasses, which pyright checks; the template has no logic beyond
10
+ loops and ``if``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections import Counter
16
+ from collections.abc import Callable, Iterable, Mapping
17
+ from dataclasses import dataclass
18
+ from pathlib import Path
19
+ from typing import Literal, TypeVar
20
+
21
+ import jinja2
22
+
23
+ # sqlglot 30's node base class; see the note in facts/structure.py.
24
+ from sqlglot.expressions.core import Expr
25
+
26
+ from text2sql_eval.execution.executor import QueryFailure, SqlValue
27
+ from text2sql_eval.facts.output import OUTPUT_LABELS_KO, OutputDiff, Relation
28
+ from text2sql_eval.facts.structure import DIALECT, STRUCTURE_LABELS_KO, parse
29
+ from text2sql_eval.judge.ex import ExecError, FlagKind, Verdict, status_of
30
+ from text2sql_eval.report.highlight import (
31
+ Clause,
32
+ clause_blocks,
33
+ differing_clauses,
34
+ differing_rows,
35
+ )
36
+ from text2sql_eval.runfile import CaseRecord, QuerySummary, Run
37
+
38
+ # A bound TypeVar instead of 3.12's ``def f[K: str]`` (ADR 0009).
39
+ _Kind = TypeVar("_Kind", bound=str)
40
+
41
+ FlagGroup = Literal["suspect_correct", "suspect_incorrect", "info"]
42
+
43
+ FLAG_LABELS_KO: dict[FlagKind, str] = {
44
+ "empty_prediction_correct": "빈 예측이 정답 처리됨",
45
+ "gold_empty_correct": "정답 결과가 비어, 결과가 비기만 하면 정답 처리됨",
46
+ "matches_tied_alternative_only": "동점 대체 정답과만 일치",
47
+ "gold_rows_with_extra_columns": "정답 행을 그대로 포함하고 컬럼만 더 있음",
48
+ "gold_columns_concatenated": "정답의 여러 컬럼을 한 문자열로 이어 붙임",
49
+ "tied_alternative_exists": "동점 대체 정답이 있는 문제",
50
+ }
51
+
52
+ # The three groups of ADR 0034.
53
+ FLAG_GROUPS: dict[FlagKind, FlagGroup] = {
54
+ "empty_prediction_correct": "suspect_correct",
55
+ "gold_empty_correct": "suspect_correct",
56
+ "matches_tied_alternative_only": "suspect_incorrect",
57
+ "gold_rows_with_extra_columns": "suspect_incorrect",
58
+ "gold_columns_concatenated": "suspect_incorrect",
59
+ "tied_alternative_exists": "info",
60
+ }
61
+
62
+ FLAG_GROUP_LABELS_KO: dict[FlagGroup, str] = {
63
+ "suspect_correct": "거짓 정답 의심",
64
+ "suspect_incorrect": "거짓 오답 의심",
65
+ "info": "정보",
66
+ }
67
+
68
+ _STATUS_LABELS_KO = {"correct": "정답", "incorrect": "오답", "error": "오류"}
69
+
70
+ _PREDICTION_LABELS_KO = {
71
+ "given": "",
72
+ "empty": "빈 예측",
73
+ "missing": "예측 없음 (빈 SQL로 판정)",
74
+ }
75
+
76
+ _RELATION_LABELS_KO: dict[Relation, str] = {
77
+ "subset": "예측 행이 모두 정답에 있음 (정답의 일부만 반환)",
78
+ "superset": "정답 행을 모두 포함하고 행이 더 있음",
79
+ "overlap": "일부 행만 겹침",
80
+ "disjoint": "겹치는 행이 없음",
81
+ }
82
+
83
+ _DIFFICULTY_ORDER = {"simple": 0, "moderate": 1, "challenging": 2}
84
+
85
+ # Bytes values are shown as a SQL hex literal, cut after this many bytes.
86
+ _MAX_BYTES_SHOWN = 16
87
+
88
+
89
+ # --- what the template shows ------------------------------------------------
90
+
91
+
92
+ @dataclass(frozen=True)
93
+ class Labeled:
94
+ """A code such as ``empty_result`` and its Korean description."""
95
+
96
+ code: str
97
+ label: str
98
+
99
+
100
+ @dataclass(frozen=True)
101
+ class CountRow:
102
+ code: str
103
+ label: str
104
+ count: int
105
+
106
+
107
+ @dataclass(frozen=True)
108
+ class GroupRow:
109
+ """EX of one difficulty or database."""
110
+
111
+ name: str
112
+ cases: int
113
+ correct: int
114
+ ex: str
115
+
116
+
117
+ @dataclass(frozen=True)
118
+ class SqlBlock:
119
+ """Lines of formatted SQL; ``differs`` marks a clause to highlight."""
120
+
121
+ text: str
122
+ differs: bool = False
123
+
124
+
125
+ @dataclass(frozen=True)
126
+ class SqlView:
127
+ """SQL formatted for reading, with the original text kept alongside.
128
+
129
+ ``blocks`` is empty when there is no SQL. ``formatted`` is false when
130
+ sqlglot could not parse the SQL, so it is shown as written.
131
+ """
132
+
133
+ blocks: tuple[SqlBlock, ...]
134
+ original: str
135
+ formatted: bool
136
+
137
+
138
+ @dataclass(frozen=True)
139
+ class Cell:
140
+ text: str
141
+ null: bool = False
142
+
143
+
144
+ @dataclass(frozen=True)
145
+ class RowView:
146
+ cells: tuple[Cell, ...]
147
+ differs: bool = False
148
+
149
+
150
+ @dataclass(frozen=True)
151
+ class TableView:
152
+ """The stored sample of one query result, or why the query failed."""
153
+
154
+ columns: tuple[str, ...]
155
+ rows: tuple[RowView, ...]
156
+ caption: str
157
+ failure: str | None = None
158
+
159
+
160
+ @dataclass(frozen=True)
161
+ class FactView:
162
+ code: str
163
+ label: str
164
+ gold: str
165
+ pred: str
166
+
167
+
168
+ @dataclass(frozen=True)
169
+ class FlagView:
170
+ code: str
171
+ label: str
172
+ group: FlagGroup
173
+ group_label: str
174
+ detail: str
175
+
176
+
177
+ @dataclass(frozen=True)
178
+ class NumberRow:
179
+ """One measured quantity of both results, e.g. the number of columns."""
180
+
181
+ label: str
182
+ gold: int
183
+ pred: int
184
+
185
+
186
+ @dataclass(frozen=True)
187
+ class CaseView:
188
+ id: str
189
+ db_id: str
190
+ difficulty: str
191
+ status: str
192
+ status_label: str
193
+ prediction_label: str
194
+ question: str
195
+ evidence: str
196
+ gold_sql: SqlView
197
+ pred_sql: SqlView
198
+ error: str
199
+ outputs: tuple[Labeled, ...]
200
+ numbers: tuple[NumberRow, ...]
201
+ relation: str
202
+ facts: tuple[FactView, ...]
203
+ structure_note: str
204
+ flags: tuple[FlagView, ...]
205
+ gold_result: TableView
206
+ pred_result: TableView
207
+ filters: dict[str, str]
208
+ """Space-separated tokens per filter key, read by the filter script."""
209
+
210
+
211
+ @dataclass(frozen=True)
212
+ class FilterOption:
213
+ value: str
214
+ label: str
215
+
216
+
217
+ @dataclass(frozen=True)
218
+ class Filter:
219
+ key: str
220
+ label: str
221
+ options: tuple[FilterOption, ...]
222
+ default: str = ""
223
+
224
+
225
+ @dataclass(frozen=True)
226
+ class Page:
227
+ title: str
228
+ info: tuple[tuple[str, str], ...]
229
+ ex: str
230
+ status_counts: tuple[CountRow, ...]
231
+ by_difficulty: tuple[GroupRow, ...]
232
+ by_db: tuple[GroupRow, ...]
233
+ output_counts: tuple[CountRow, ...]
234
+ structure_counts: tuple[CountRow, ...]
235
+ flag_counts: tuple[CountRow, ...]
236
+ filters: tuple[Filter, ...]
237
+ cases: tuple[CaseView, ...]
238
+
239
+
240
+ # --- rendering ----------------------------------------------------------------
241
+
242
+
243
+ def write_html_report(run: Run, path: Path, *, run_name: str) -> None:
244
+ """Write the report of ``run`` to ``path`` as one UTF-8 HTML file.
245
+
246
+ ``run_name`` names the run in the page title, usually the run file name.
247
+
248
+ Example:
249
+ >>> run_path = Path("runs/v3.json")
250
+ >>> write_html_report(read_run(run_path), Path("runs/v3.html"), run_name=run_path.name)
251
+ """
252
+ path.parent.mkdir(parents=True, exist_ok=True)
253
+ path.write_text(render_html(run, run_name=run_name), encoding="utf-8")
254
+
255
+
256
+ def render_html(run: Run, *, run_name: str) -> str:
257
+ """The report of ``run`` as an HTML document."""
258
+ environment = jinja2.Environment(
259
+ loader=jinja2.PackageLoader("text2sql_eval.report", "templates"),
260
+ autoescape=True,
261
+ undefined=jinja2.StrictUndefined,
262
+ trim_blocks=True,
263
+ lstrip_blocks=True,
264
+ )
265
+ template = environment.get_template("report.html.j2")
266
+ return template.render(page=build_page(run, run_name=run_name))
267
+
268
+
269
+ def build_page(run: Run, *, run_name: str) -> Page:
270
+ """Everything the template shows, computed from the run."""
271
+ cases = tuple(_case_view(case) for case in run.cases)
272
+ summary = run.summary
273
+ return Page(
274
+ title=f"text2sql-eval report: {run_name}",
275
+ info=_info(run, run_name),
276
+ ex=f"{summary.ex:.1%}",
277
+ status_counts=(
278
+ CountRow("total", "전체", summary.total),
279
+ CountRow("correct", "정답", summary.correct),
280
+ CountRow("incorrect", "오답", summary.incorrect),
281
+ CountRow("error", "오류", summary.error),
282
+ ),
283
+ by_difficulty=_groups(
284
+ run.cases, lambda c: c.difficulty or "(없음)", _difficulty_key
285
+ ),
286
+ by_db=_groups(run.cases, lambda c: c.db_id, lambda name: (0, name)),
287
+ output_counts=_count_rows(_output_kinds(run.cases), OUTPUT_LABELS_KO),
288
+ structure_counts=_count_rows(_structure_kinds(run.cases), STRUCTURE_LABELS_KO),
289
+ flag_counts=_flag_counts(run.cases),
290
+ filters=_filters(run.cases),
291
+ cases=cases,
292
+ )
293
+
294
+
295
+ # --- summary ------------------------------------------------------------------
296
+
297
+
298
+ def _info(run: Run, run_name: str) -> tuple[tuple[str, str], ...]:
299
+ rows = [
300
+ ("실행 파일", run_name),
301
+ ("만든 시각", run.created_at),
302
+ ("도구 버전", run.tool_version),
303
+ ("데이터셋", run.dataset.path),
304
+ ("예측 파일", f"{run.predictions.path} ({run.prediction_format})"),
305
+ ("동점 대체 정답", "없음" if run.tied is None else run.tied.path),
306
+ ("시간 제한", f"쿼리당 {run.settings.timeout_seconds:g}초"),
307
+ ]
308
+ rows.extend((f"meta: {key}", value) for key, value in sorted(run.meta.items()))
309
+ return tuple(rows)
310
+
311
+
312
+ def _difficulty_key(name: str) -> tuple[int, str]:
313
+ return (_DIFFICULTY_ORDER.get(name, len(_DIFFICULTY_ORDER)), name)
314
+
315
+
316
+ def _groups(
317
+ cases: Iterable[CaseRecord],
318
+ name_of: Callable[[CaseRecord], str],
319
+ sort_key: Callable[[str], tuple[int, str]],
320
+ ) -> tuple[GroupRow, ...]:
321
+ totals: Counter[str] = Counter()
322
+ correct: Counter[str] = Counter()
323
+ for case in cases:
324
+ name = name_of(case)
325
+ totals[name] += 1
326
+ correct[name] += status_of(case.verdict) == "correct"
327
+ return tuple(
328
+ GroupRow(
329
+ name, totals[name], correct[name], f"{correct[name] / totals[name]:.1%}"
330
+ )
331
+ for name in sorted(totals, key=sort_key)
332
+ )
333
+
334
+
335
+ def _output_kinds(cases: Iterable[CaseRecord]) -> Counter[str]:
336
+ return Counter(
337
+ kind for case in cases if case.diff for kind in case.diff.output.differences
338
+ )
339
+
340
+
341
+ def _structure_kinds(cases: Iterable[CaseRecord]) -> Counter[str]:
342
+ return Counter(
343
+ fact.kind for case in cases if case.diff for fact in case.diff.structure.facts
344
+ )
345
+
346
+
347
+ def _count_rows(
348
+ counts: Counter[str], labels: Mapping[_Kind, str]
349
+ ) -> tuple[CountRow, ...]:
350
+ """Every kind in ``labels``, most frequent first, ties in ``labels`` order.
351
+
352
+ Kinds that never occur are kept, with count 0, at the end.
353
+ """
354
+ kinds = list(labels)
355
+ ordered = sorted(kinds, key=lambda kind: (-counts[kind], kinds.index(kind)))
356
+ return tuple(CountRow(kind, labels[kind], counts[kind]) for kind in ordered)
357
+
358
+
359
+ def _flag_counts(cases: Iterable[CaseRecord]) -> tuple[CountRow, ...]:
360
+ """Every flag kind in the order of ADR 0034, including those never raised."""
361
+ counts = Counter(flag.kind for case in cases for flag in case.verdict.flags)
362
+ return tuple(
363
+ CountRow(
364
+ kind,
365
+ f"{FLAG_GROUP_LABELS_KO[FLAG_GROUPS[kind]]}: {FLAG_LABELS_KO[kind]}",
366
+ counts[kind],
367
+ )
368
+ for kind in FLAG_LABELS_KO
369
+ )
370
+
371
+
372
+ def _filters(cases: tuple[CaseRecord, ...]) -> tuple[Filter, ...]:
373
+ databases = sorted({case.db_id for case in cases})
374
+ difficulties = sorted(
375
+ {case.difficulty or "none" for case in cases}, key=_difficulty_key
376
+ )
377
+ flags = Counter(flag.kind for case in cases for flag in case.verdict.flags)
378
+ flag_groups = tuple(
379
+ FilterOption(group, f"{label} (모두)")
380
+ for group, label in FLAG_GROUP_LABELS_KO.items()
381
+ if group != "info"
382
+ )
383
+ flag_kinds = tuple(
384
+ FilterOption(kind, f"{kind} {flags[kind]}")
385
+ for kind in FLAG_LABELS_KO
386
+ if flags[kind]
387
+ )
388
+ return (
389
+ _filter(
390
+ "status",
391
+ "판정",
392
+ (
393
+ FilterOption("wrong", "오답·오류"),
394
+ FilterOption("correct", "정답"),
395
+ FilterOption("incorrect", "오답"),
396
+ FilterOption("error", "오류"),
397
+ ),
398
+ default="wrong",
399
+ ),
400
+ _filter("db", "데이터베이스", tuple(FilterOption(db, db) for db in databases)),
401
+ _filter(
402
+ "difficulty",
403
+ "난이도",
404
+ tuple(
405
+ FilterOption(d, "(없음)" if d == "none" else d) for d in difficulties
406
+ ),
407
+ ),
408
+ _filter(
409
+ "output",
410
+ "출력 차이",
411
+ _kind_options(_output_kinds(cases), OUTPUT_LABELS_KO),
412
+ ),
413
+ _filter(
414
+ "structure",
415
+ "SQL 구조 차이",
416
+ _kind_options(_structure_kinds(cases), STRUCTURE_LABELS_KO),
417
+ ),
418
+ _filter("flag", "플래그", flag_groups + flag_kinds),
419
+ )
420
+
421
+
422
+ def _filter(
423
+ key: str, label: str, options: tuple[FilterOption, ...], default: str = ""
424
+ ) -> Filter:
425
+ """A filter whose first option, 전체 (all), turns it off."""
426
+ return Filter(key, label, (FilterOption("", "전체"), *options), default)
427
+
428
+
429
+ def _kind_options(
430
+ counts: Counter[str], labels: Mapping[_Kind, str]
431
+ ) -> tuple[FilterOption, ...]:
432
+ """One option per kind that occurs, with its description and count."""
433
+ return tuple(
434
+ FilterOption(row.code, f"{row.code} ({row.label}) {row.count}")
435
+ for row in _count_rows(counts, labels)
436
+ if row.count
437
+ )
438
+
439
+
440
+ # --- one case -----------------------------------------------------------------
441
+
442
+
443
+ def _case_view(case: CaseRecord) -> CaseView:
444
+ status = status_of(case.verdict)
445
+ diff = case.diff
446
+ flags = tuple(
447
+ FlagView(
448
+ code=flag.kind,
449
+ label=FLAG_LABELS_KO[flag.kind],
450
+ group=FLAG_GROUPS[flag.kind],
451
+ group_label=FLAG_GROUP_LABELS_KO[FLAG_GROUPS[flag.kind]],
452
+ detail=flag.detail,
453
+ )
454
+ for flag in case.verdict.flags
455
+ )
456
+ outputs = () if diff is None else diff.output.differences
457
+ facts = () if diff is None else diff.structure.facts
458
+ # Highlight where the differences are reported: cases that are not
459
+ # correct and whose gold query ran (ADR 0031, 0039).
460
+ highlight = diff is not None
461
+ gold_sql, pred_sql = sql_views(case.gold_sql, case.pred_sql, highlight=highlight)
462
+ gold_rows, pred_rows = (
463
+ differing_rows(case.gold, case.pred) if highlight else ((), ())
464
+ )
465
+ return CaseView(
466
+ id=case.id,
467
+ db_id=case.db_id,
468
+ difficulty=case.difficulty or "",
469
+ status=status,
470
+ status_label=_STATUS_LABELS_KO[status],
471
+ prediction_label=_PREDICTION_LABELS_KO[case.prediction_state],
472
+ question=case.question,
473
+ evidence=case.evidence,
474
+ gold_sql=gold_sql,
475
+ pred_sql=pred_sql,
476
+ error=_error(case.verdict),
477
+ outputs=tuple(Labeled(kind, OUTPUT_LABELS_KO[kind]) for kind in outputs),
478
+ numbers=() if diff is None else _numbers(diff.output),
479
+ relation=""
480
+ if diff is None or diff.output.relation is None
481
+ else _RELATION_LABELS_KO[diff.output.relation],
482
+ facts=tuple(
483
+ FactView(fact.kind, STRUCTURE_LABELS_KO[fact.kind], fact.gold, fact.pred)
484
+ for fact in facts
485
+ ),
486
+ structure_note=_structure_note(case),
487
+ flags=flags,
488
+ gold_result=table_view(case.gold, gold_rows),
489
+ pred_result=table_view(case.pred, pred_rows),
490
+ filters={
491
+ "status": status if status == "correct" else f"{status} wrong",
492
+ "db": case.db_id,
493
+ "difficulty": case.difficulty or "none",
494
+ "output": " ".join(outputs),
495
+ "structure": " ".join(fact.kind for fact in facts),
496
+ "flag": " ".join(
497
+ sorted({flag.code for flag in flags} | {flag.group for flag in flags})
498
+ ),
499
+ },
500
+ )
501
+
502
+
503
+ def sql_views(
504
+ gold_sql: str, pred_sql: str, *, highlight: bool
505
+ ) -> tuple[SqlView, SqlView]:
506
+ """Both queries reformatted by sqlglot, one block per outer clause.
507
+
508
+ With ``highlight``, clauses that differ between the two are marked. SQL
509
+ that is empty or that sqlglot cannot parse is shown as written, and then
510
+ nothing is marked on either side.
511
+ """
512
+ gold_tree = parse(gold_sql) if gold_sql.strip() else None
513
+ pred_tree = parse(pred_sql) if pred_sql.strip() else None
514
+ differs: frozenset[Clause] = frozenset()
515
+ if highlight and gold_tree is not None and pred_tree is not None:
516
+ differs = differing_clauses(gold_tree, pred_tree)
517
+ return (
518
+ _sql_view(gold_sql, gold_tree, differs),
519
+ _sql_view(pred_sql, pred_tree, differs),
520
+ )
521
+
522
+
523
+ def _sql_view(sql: str, tree: Expr | None, differs: frozenset[Clause]) -> SqlView:
524
+ if not sql.strip():
525
+ return SqlView((), sql, formatted=False)
526
+ if tree is None:
527
+ return SqlView((SqlBlock(sql.strip()),), sql, formatted=False)
528
+ pretty = tree.sql(dialect=DIALECT, pretty=True)
529
+ return SqlView(
530
+ tuple(
531
+ SqlBlock(text, differs=clause in differs)
532
+ for clause, text in clause_blocks(pretty)
533
+ ),
534
+ sql,
535
+ formatted=True,
536
+ )
537
+
538
+
539
+ def table_view(result: QuerySummary, differs: tuple[bool, ...] = ()) -> TableView:
540
+ """A result sample as a table; ``differs`` marks rows, one flag per row."""
541
+ if isinstance(result, QueryFailure):
542
+ return TableView((), (), caption="", failure=f"{result.kind}: {result.message}")
543
+ if result.row_count > len(result.rows):
544
+ caption = f"앞 {len(result.rows)}행 / 전체 {result.row_count}행"
545
+ else:
546
+ caption = f"{result.row_count}행"
547
+ return TableView(
548
+ columns=result.columns,
549
+ rows=tuple(
550
+ RowView(
551
+ tuple(_cell(value) for value in row),
552
+ differs=index < len(differs) and differs[index],
553
+ )
554
+ for index, row in enumerate(result.rows)
555
+ ),
556
+ caption=caption,
557
+ )
558
+
559
+
560
+ def _cell(value: SqlValue) -> Cell:
561
+ if value is None:
562
+ return Cell("NULL", null=True)
563
+ if isinstance(value, bytes):
564
+ shown = value[:_MAX_BYTES_SHOWN].hex()
565
+ more = "…" if len(value) > _MAX_BYTES_SHOWN else ""
566
+ return Cell(f"x'{shown}{more}' ({len(value)} bytes)")
567
+ return Cell(str(value))
568
+
569
+
570
+ def _numbers(output: OutputDiff) -> tuple[NumberRow, ...]:
571
+ return (
572
+ NumberRow("컬럼 수", output.gold_columns, output.pred_columns),
573
+ NumberRow(
574
+ "서로 다른 행 수", output.gold_distinct_rows, output.pred_distinct_rows
575
+ ),
576
+ )
577
+
578
+
579
+ def _structure_note(case: CaseRecord) -> str:
580
+ if case.diff is None:
581
+ return ""
582
+ failed = case.diff.structure.parse_failed
583
+ if failed is not None:
584
+ side = "정답" if failed == "gold" else "예측"
585
+ return f"{side} SQL을 파싱하지 못해 구조 차이를 구하지 않았습니다."
586
+ if not case.diff.structure.facts:
587
+ return "비교하는 다섯 가지 구조에서는 차이를 찾지 못했습니다."
588
+ return ""
589
+
590
+
591
+ def _error(verdict: Verdict) -> str:
592
+ if not isinstance(verdict, ExecError):
593
+ return ""
594
+ side = "예측" if verdict.side == "pred" else "정답"
595
+ return f"{side} SQL 실행 실패 ({verdict.kind}): {verdict.message}"