text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
{#
|
|
2
|
+
HTML report of one run (ADR 0007, 0038, 0039).
|
|
3
|
+
Rendered with autoescaping on: every {{ value }} is escaped.
|
|
4
|
+
Everything shown is prepared in html_report.py; keep logic out of here.
|
|
5
|
+
No external resources: the page must open offline.
|
|
6
|
+
#}
|
|
7
|
+
{% macro sql(view) %}
|
|
8
|
+
{% if view.blocks %}
|
|
9
|
+
<pre class="sql">{% for block in view.blocks %}<span class="clause{% if block.differs %} differs{% endif %}">{{ block.text }}</span>{% endfor %}</pre>
|
|
10
|
+
{% if view.formatted %}
|
|
11
|
+
<details class="original"><summary>원래 SQL</summary><pre class="sql">{{ view.original }}</pre></details>
|
|
12
|
+
{% endif %}
|
|
13
|
+
{% else %}
|
|
14
|
+
<p class="muted">(SQL 없음)</p>
|
|
15
|
+
{% endif %}
|
|
16
|
+
{% endmacro %}
|
|
17
|
+
{% macro table(view) %}
|
|
18
|
+
{% if view.failure %}
|
|
19
|
+
<p class="error">{{ view.failure }}</p>
|
|
20
|
+
{% elif not view.columns %}
|
|
21
|
+
<p class="muted">(컬럼 없음)</p>
|
|
22
|
+
{% else %}
|
|
23
|
+
<div class="scroll"><table class="result">
|
|
24
|
+
<thead><tr>{% for column in view.columns %}<th>{{ column }}</th>{% endfor %}</tr></thead>
|
|
25
|
+
<tbody>
|
|
26
|
+
{% for row in view.rows %}
|
|
27
|
+
<tr{% if row.differs %} class="differs"{% endif %}>{% for cell in row.cells %}<td{% if cell.null %} class="null"{% endif %}>{{ cell.text }}</td>{% endfor %}</tr>
|
|
28
|
+
{% endfor %}
|
|
29
|
+
</tbody>
|
|
30
|
+
</table></div>
|
|
31
|
+
{% endif %}
|
|
32
|
+
{% endmacro %}
|
|
33
|
+
{% macro counts(title, rows) %}
|
|
34
|
+
<section class="card">
|
|
35
|
+
<h3>{{ title }}</h3>
|
|
36
|
+
<table class="counts">
|
|
37
|
+
{% for row in rows %}
|
|
38
|
+
<tr{% if not row.count %} class="zero"{% endif %}><td><code>{{ row.code }}</code><br><span class="muted">{{ row.label }}</span></td><td class="num">{{ row.count }}</td></tr>
|
|
39
|
+
{% endfor %}
|
|
40
|
+
</table>
|
|
41
|
+
</section>
|
|
42
|
+
{% endmacro %}
|
|
43
|
+
{% macro groups(title, rows) %}
|
|
44
|
+
<section class="card">
|
|
45
|
+
<h3>{{ title }}</h3>
|
|
46
|
+
<table class="counts">
|
|
47
|
+
<tr><th></th><th class="num">문제</th><th class="num">정답</th><th class="num">EX</th></tr>
|
|
48
|
+
{% for row in rows %}
|
|
49
|
+
<tr><td>{{ row.name }}</td><td class="num">{{ row.cases }}</td><td class="num">{{ row.correct }}</td><td class="num">{{ row.ex }}</td></tr>
|
|
50
|
+
{% endfor %}
|
|
51
|
+
</table>
|
|
52
|
+
</section>
|
|
53
|
+
{% endmacro %}
|
|
54
|
+
<!DOCTYPE html>
|
|
55
|
+
<html lang="ko">
|
|
56
|
+
<head>
|
|
57
|
+
<meta charset="utf-8">
|
|
58
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
59
|
+
<title>{{ page.title }}</title>
|
|
60
|
+
<style>
|
|
61
|
+
:root {
|
|
62
|
+
--bg: #ffffff; --fg: #1f2328; --muted: #656d76; --line: #d0d7de; --panel: #f6f8fa;
|
|
63
|
+
--differs: #fff1b8; --differs-edge: #d4a72c;
|
|
64
|
+
--correct: #1a7f37; --incorrect: #cf222e; --error: #bc4c00;
|
|
65
|
+
--suspect: #8250df; --info: #57606a;
|
|
66
|
+
}
|
|
67
|
+
@media (prefers-color-scheme: dark) {
|
|
68
|
+
:root {
|
|
69
|
+
--bg: #0d1117; --fg: #e6edf3; --muted: #8d96a0; --line: #30363d; --panel: #161b22;
|
|
70
|
+
--differs: #4d3c00; --differs-edge: #bb8009;
|
|
71
|
+
--correct: #3fb950; --incorrect: #f85149; --error: #db6d28;
|
|
72
|
+
--suspect: #a371f7; --info: #8d96a0;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
* { box-sizing: border-box; }
|
|
76
|
+
body { margin: 0; background: var(--bg); color: var(--fg); font: 14px/1.5 system-ui, -apple-system, "Segoe UI", "Apple SD Gothic Neo", "Malgun Gothic", sans-serif; }
|
|
77
|
+
main { max-width: 1280px; margin: 0 auto; padding: 16px; }
|
|
78
|
+
h1 { font-size: 20px; margin: 8px 0 16px; overflow-wrap: anywhere; }
|
|
79
|
+
h2 { font-size: 16px; margin: 24px 0 8px; }
|
|
80
|
+
h3 { font-size: 14px; margin: 0 0 8px; }
|
|
81
|
+
h4 { font-size: 13px; margin: 16px 0 4px; color: var(--muted); font-weight: 600; }
|
|
82
|
+
code, pre { font: 12.5px/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
|
|
83
|
+
[hidden] { display: none !important; }
|
|
84
|
+
.muted { color: var(--muted); }
|
|
85
|
+
.num { text-align: right; font-variant-numeric: tabular-nums; }
|
|
86
|
+
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 12px; align-items: start; }
|
|
87
|
+
.card { border: 1px solid var(--line); border-radius: 8px; padding: 12px; background: var(--bg); }
|
|
88
|
+
.ex { font-size: 32px; font-weight: 700; }
|
|
89
|
+
table { border-collapse: collapse; }
|
|
90
|
+
.counts { width: 100%; }
|
|
91
|
+
.counts.numbers { width: auto; min-width: 240px; }
|
|
92
|
+
.counts td, .counts th { padding: 4px 6px; border-top: 1px solid var(--line); vertical-align: top; }
|
|
93
|
+
.counts tr:first-child td, .counts tr:first-child th { border-top: 0; }
|
|
94
|
+
.counts .zero { color: var(--muted); }
|
|
95
|
+
.info { display: grid; grid-template-columns: max-content minmax(0, 1fr) max-content minmax(0, 1fr); gap: 2px 12px; margin: 0; }
|
|
96
|
+
.info dt { color: var(--muted); white-space: nowrap; }
|
|
97
|
+
.info dd { margin: 0; overflow-wrap: anywhere; }
|
|
98
|
+
@media (max-width: 720px) { .info { grid-template-columns: max-content minmax(0, 1fr); } }
|
|
99
|
+
.run-info { margin-bottom: 12px; }
|
|
100
|
+
.filters { position: sticky; top: 0; z-index: 1; display: flex; flex-wrap: wrap; gap: 8px 16px; align-items: center; padding: 10px 12px; margin: 0 -12px; background: var(--panel); border-bottom: 1px solid var(--line); }
|
|
101
|
+
.filters label { display: flex; gap: 6px; align-items: center; }
|
|
102
|
+
.filters select, .filters button { font: inherit; color: inherit; background: var(--bg); border: 1px solid var(--line); border-radius: 6px; padding: 2px 6px; max-width: 100%; }
|
|
103
|
+
.case { border: 1px solid var(--line); border-radius: 8px; margin: 8px 0; }
|
|
104
|
+
.case > summary { cursor: pointer; padding: 8px 12px; display: flex; flex-wrap: wrap; gap: 4px 8px; align-items: baseline; }
|
|
105
|
+
.case[open] > summary { border-bottom: 1px solid var(--line); background: var(--panel); border-radius: 8px 8px 0 0; }
|
|
106
|
+
.case .id { font-weight: 700; }
|
|
107
|
+
.case .question { flex: 1 1 320px; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
108
|
+
.body { padding: 12px; }
|
|
109
|
+
.badge { border-radius: 10px; padding: 0 8px; color: #fff; font-size: 12px; font-weight: 600; }
|
|
110
|
+
.status-correct { background: var(--correct); }
|
|
111
|
+
.status-incorrect { background: var(--incorrect); }
|
|
112
|
+
.status-error { background: var(--error); }
|
|
113
|
+
.tag { border: 1px solid var(--line); border-radius: 6px; padding: 0 4px; font-size: 11.5px; }
|
|
114
|
+
.flag-suspect_incorrect, .flag-suspect_correct { border-color: var(--suspect); color: var(--suspect); }
|
|
115
|
+
.flag-info { color: var(--info); }
|
|
116
|
+
.pair > section > h4:first-child { margin-top: 0; }
|
|
117
|
+
.pair { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 12px; margin: 12px 0; }
|
|
118
|
+
.pair > section { min-width: 0; }
|
|
119
|
+
pre.sql { margin: 0; padding: 8px; background: var(--panel); border: 1px solid var(--line); border-radius: 6px; white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
120
|
+
.clause { display: block; }
|
|
121
|
+
.differs { background: var(--differs); box-shadow: inset 3px 0 var(--differs-edge); }
|
|
122
|
+
.original { margin-top: 4px; }
|
|
123
|
+
.original > summary { cursor: pointer; color: var(--muted); font-size: 12px; }
|
|
124
|
+
.scroll { overflow-x: auto; max-width: 100%; }
|
|
125
|
+
.result td, .result th { border: 1px solid var(--line); padding: 2px 6px; text-align: left; vertical-align: top; max-width: 360px; overflow-wrap: anywhere; }
|
|
126
|
+
.result th { background: var(--panel); }
|
|
127
|
+
.result .null { color: var(--muted); font-style: italic; }
|
|
128
|
+
.error { color: var(--error); overflow-wrap: anywhere; }
|
|
129
|
+
.facts { width: 100%; }
|
|
130
|
+
.facts td, .facts th { border-top: 1px solid var(--line); padding: 4px 6px; text-align: left; vertical-align: top; overflow-wrap: anywhere; }
|
|
131
|
+
.flags { margin: 0; padding-left: 18px; }
|
|
132
|
+
.legend { font-size: 12px; }
|
|
133
|
+
.legend .differs { padding: 0 4px; }
|
|
134
|
+
</style>
|
|
135
|
+
</head>
|
|
136
|
+
<body>
|
|
137
|
+
<main>
|
|
138
|
+
<h1>{{ page.title }}</h1>
|
|
139
|
+
|
|
140
|
+
<section class="card run-info">
|
|
141
|
+
<h3>실행 정보</h3>
|
|
142
|
+
<dl class="info">
|
|
143
|
+
{% for label, value in page.info %}
|
|
144
|
+
<dt>{{ label }}</dt><dd>{{ value }}</dd>
|
|
145
|
+
{% endfor %}
|
|
146
|
+
</dl>
|
|
147
|
+
</section>
|
|
148
|
+
|
|
149
|
+
<section class="grid">
|
|
150
|
+
<section class="card">
|
|
151
|
+
<h3>EX (실행 정확도)</h3>
|
|
152
|
+
<div class="ex">{{ page.ex }}</div>
|
|
153
|
+
<table class="counts">
|
|
154
|
+
{% for row in page.status_counts %}
|
|
155
|
+
<tr><td>{{ row.label }}</td><td class="num">{{ row.count }}</td></tr>
|
|
156
|
+
{% endfor %}
|
|
157
|
+
</table>
|
|
158
|
+
</section>
|
|
159
|
+
{{ groups("난이도별 EX", page.by_difficulty) }}
|
|
160
|
+
{{ groups("데이터베이스별 EX", page.by_db) }}
|
|
161
|
+
</section>
|
|
162
|
+
|
|
163
|
+
<h2>오답 분석</h2>
|
|
164
|
+
<p class="muted">한 오답에 여러 차이가 붙을 수 있어 합계가 오답 수보다 클 수 있습니다. 플래그는 판정을 바꾸지 않고, 공식 규칙이 놓칠 수 있는 경우를 표시합니다.</p>
|
|
165
|
+
<section class="grid">
|
|
166
|
+
{{ counts("출력 차이 (Output difference)", page.output_counts) }}
|
|
167
|
+
{{ counts("SQL 구조 차이 (SQL structure difference)", page.structure_counts) }}
|
|
168
|
+
{{ counts("판정 신뢰도 플래그", page.flag_counts) }}
|
|
169
|
+
</section>
|
|
170
|
+
|
|
171
|
+
<h2>케이스</h2>
|
|
172
|
+
<p class="muted legend"><span class="differs">색칠한 부분</span>은 정답과 예측에서 다른 SQL 절과 결과 행입니다.</p>
|
|
173
|
+
<form class="filters" onsubmit="return false">
|
|
174
|
+
{% for filter in page.filters %}
|
|
175
|
+
<label>{{ filter.label }}
|
|
176
|
+
<select data-filter="{{ filter.key }}">
|
|
177
|
+
{% for option in filter.options %}
|
|
178
|
+
<option value="{{ option.value }}"{% if option.value == filter.default %} selected{% endif %}>{{ option.label }}</option>
|
|
179
|
+
{% endfor %}
|
|
180
|
+
</select>
|
|
181
|
+
</label>
|
|
182
|
+
{% endfor %}
|
|
183
|
+
<span><output id="shown">{{ page.cases|length }}</output> / {{ page.cases|length }}개</span>
|
|
184
|
+
<button type="button" id="open-all">모두 펼치기</button>
|
|
185
|
+
<button type="button" id="close-all">모두 접기</button>
|
|
186
|
+
</form>
|
|
187
|
+
|
|
188
|
+
{% for case in page.cases %}
|
|
189
|
+
<details class="case"{% for key, value in case.filters.items() %} data-{{ key }}="{{ value }}"{% endfor %}>
|
|
190
|
+
<summary>
|
|
191
|
+
<span class="id">#{{ case.id }}</span>
|
|
192
|
+
<span class="badge status-{{ case.status }}">{{ case.status_label }}</span>
|
|
193
|
+
<span class="muted">{{ case.db_id }}{% if case.difficulty %} · {{ case.difficulty }}{% endif %}</span>
|
|
194
|
+
{% for output in case.outputs %}<code class="tag">{{ output.code }}</code> {% endfor %}
|
|
195
|
+
{% for flag in case.flags %}<code class="tag flag-{{ flag.group }}">{{ flag.code }}</code> {% endfor %}
|
|
196
|
+
<span class="question">{{ case.question }}</span>
|
|
197
|
+
</summary>
|
|
198
|
+
<div class="body">
|
|
199
|
+
<p><strong>질문</strong> {{ case.question }}</p>
|
|
200
|
+
{% if case.evidence %}
|
|
201
|
+
<p><strong>evidence</strong> {{ case.evidence }}</p>
|
|
202
|
+
{% endif %}
|
|
203
|
+
{% if case.prediction_label %}
|
|
204
|
+
<p class="muted">{{ case.prediction_label }}</p>
|
|
205
|
+
{% endif %}
|
|
206
|
+
{% if case.error %}
|
|
207
|
+
<p class="error">{{ case.error }}</p>
|
|
208
|
+
{% endif %}
|
|
209
|
+
<div class="pair">
|
|
210
|
+
<section><h4>정답 SQL</h4>{{ sql(case.gold_sql) }}</section>
|
|
211
|
+
<section><h4>예측 SQL</h4>{{ sql(case.pred_sql) }}</section>
|
|
212
|
+
</div>
|
|
213
|
+
{% if case.flags %}
|
|
214
|
+
<h4>판정 신뢰도 플래그</h4>
|
|
215
|
+
<ul class="flags">
|
|
216
|
+
{% for flag in case.flags %}
|
|
217
|
+
<li><code class="tag flag-{{ flag.group }}">{{ flag.code }}</code> {{ flag.group_label }}: {{ flag.label }} <span class="muted">({{ flag.detail }})</span></li>
|
|
218
|
+
{% endfor %}
|
|
219
|
+
</ul>
|
|
220
|
+
{% endif %}
|
|
221
|
+
{% if case.outputs %}
|
|
222
|
+
<h4>출력 차이</h4>
|
|
223
|
+
<p>{% for output in case.outputs %}<code class="tag">{{ output.code }}</code> {{ output.label }}{% if not loop.last %} · {% endif %}{% endfor %}</p>
|
|
224
|
+
<table class="counts numbers">
|
|
225
|
+
<tr><th></th><th class="num">정답</th><th class="num">예측</th></tr>
|
|
226
|
+
{% for row in case.numbers %}
|
|
227
|
+
<tr><td>{{ row.label }}</td><td class="num">{{ row.gold }}</td><td class="num">{{ row.pred }}</td></tr>
|
|
228
|
+
{% endfor %}
|
|
229
|
+
</table>
|
|
230
|
+
{% if case.relation %}
|
|
231
|
+
<p class="muted">행 집합 관계: {{ case.relation }}</p>
|
|
232
|
+
{% endif %}
|
|
233
|
+
{% endif %}
|
|
234
|
+
{% if case.facts or case.structure_note %}
|
|
235
|
+
<h4>SQL 구조 차이</h4>
|
|
236
|
+
{% if case.facts %}
|
|
237
|
+
<table class="facts">
|
|
238
|
+
<tr><th>차이</th><th>정답 SQL</th><th>예측 SQL</th></tr>
|
|
239
|
+
{% for fact in case.facts %}
|
|
240
|
+
<tr><td><code>{{ fact.code }}</code><br><span class="muted">{{ fact.label }}</span></td><td><code>{{ fact.gold }}</code></td><td><code>{{ fact.pred }}</code></td></tr>
|
|
241
|
+
{% endfor %}
|
|
242
|
+
</table>
|
|
243
|
+
{% endif %}
|
|
244
|
+
{% if case.structure_note %}
|
|
245
|
+
<p class="muted">{{ case.structure_note }}</p>
|
|
246
|
+
{% endif %}
|
|
247
|
+
{% endif %}
|
|
248
|
+
<div class="pair">
|
|
249
|
+
<section><h4>정답 결과 <span class="muted">{{ case.gold_result.caption }}</span></h4>{{ table(case.gold_result) }}</section>
|
|
250
|
+
<section><h4>예측 결과 <span class="muted">{{ case.pred_result.caption }}</span></h4>{{ table(case.pred_result) }}</section>
|
|
251
|
+
</div>
|
|
252
|
+
</div>
|
|
253
|
+
</details>
|
|
254
|
+
{% endfor %}
|
|
255
|
+
</main>
|
|
256
|
+
<script>
|
|
257
|
+
// Filters only show or hide cases; all content is already in the page.
|
|
258
|
+
(() => {
|
|
259
|
+
"use strict";
|
|
260
|
+
const selects = [...document.querySelectorAll("select[data-filter]")];
|
|
261
|
+
const cases = [...document.querySelectorAll(".case")];
|
|
262
|
+
const shown = document.getElementById("shown");
|
|
263
|
+
const matches = (element, select) =>
|
|
264
|
+
!select.value ||
|
|
265
|
+
(element.dataset[select.dataset.filter] || "").split(" ").includes(select.value);
|
|
266
|
+
const apply = () => {
|
|
267
|
+
let count = 0;
|
|
268
|
+
for (const element of cases) {
|
|
269
|
+
element.hidden = !selects.every((select) => matches(element, select));
|
|
270
|
+
if (!element.hidden) count += 1;
|
|
271
|
+
}
|
|
272
|
+
shown.textContent = String(count);
|
|
273
|
+
};
|
|
274
|
+
const setOpen = (open) => {
|
|
275
|
+
for (const element of cases) if (!element.hidden) element.open = open;
|
|
276
|
+
};
|
|
277
|
+
for (const select of selects) select.addEventListener("change", apply);
|
|
278
|
+
document.getElementById("open-all").addEventListener("click", () => setOpen(true));
|
|
279
|
+
document.getElementById("close-all").addEventListener("click", () => setOpen(false));
|
|
280
|
+
apply();
|
|
281
|
+
})();
|
|
282
|
+
</script>
|
|
283
|
+
</body>
|
|
284
|
+
</html>
|