text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
text2sql_eval/errors.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Errors raised when user-supplied input cannot be used."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class InputError(Exception):
|
|
9
|
+
"""An input file is malformed or does not match the dataset.
|
|
10
|
+
|
|
11
|
+
The message says where the problem is (file, and line or item when known),
|
|
12
|
+
what is wrong, and how to fix it, so it can be shown to users as is.
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
>>> err = InputError(Path("preds.jsonl"), "missing field 'sql'",
|
|
16
|
+
... location="line 3", hint="add an 'sql' string")
|
|
17
|
+
>>> print(err)
|
|
18
|
+
preds.jsonl, line 3: missing field 'sql'
|
|
19
|
+
hint: add an 'sql' string
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
path: Path,
|
|
25
|
+
problem: str,
|
|
26
|
+
*,
|
|
27
|
+
location: str | None = None,
|
|
28
|
+
hint: str | None = None,
|
|
29
|
+
) -> None:
|
|
30
|
+
self.path = path
|
|
31
|
+
self.problem = problem
|
|
32
|
+
self.location = location
|
|
33
|
+
self.hint = hint
|
|
34
|
+
super().__init__(self._format())
|
|
35
|
+
|
|
36
|
+
def _format(self) -> str:
|
|
37
|
+
where = (
|
|
38
|
+
str(self.path) if self.location is None else f"{self.path}, {self.location}"
|
|
39
|
+
)
|
|
40
|
+
message = f"{where}: {self.problem}"
|
|
41
|
+
if self.hint is not None:
|
|
42
|
+
message += f"\n hint: {self.hint}"
|
|
43
|
+
return message
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Execute and judge every case of a dataset against a set of predictions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from datetime import UTC, datetime
|
|
8
|
+
from importlib.metadata import version
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from text2sql_eval.datasets.bird import Case, Dataset, load_bird, load_bird_tied
|
|
12
|
+
from text2sql_eval.datasets.predictions import (
|
|
13
|
+
Prediction,
|
|
14
|
+
load_jsonl_predictions,
|
|
15
|
+
load_text_predictions,
|
|
16
|
+
)
|
|
17
|
+
from text2sql_eval.errors import InputError
|
|
18
|
+
from text2sql_eval.execution.executor import (
|
|
19
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
20
|
+
Query,
|
|
21
|
+
QueryResult,
|
|
22
|
+
execute_all,
|
|
23
|
+
)
|
|
24
|
+
from text2sql_eval.facts import CaseDiff
|
|
25
|
+
from text2sql_eval.facts.output import compare_outputs
|
|
26
|
+
from text2sql_eval.facts.structure import structural_facts
|
|
27
|
+
from text2sql_eval.judge.ex import Correct, ExecError, Verdict, judge_case
|
|
28
|
+
from text2sql_eval.runfile import (
|
|
29
|
+
CaseRecord,
|
|
30
|
+
FileRef,
|
|
31
|
+
PredictionFormat,
|
|
32
|
+
PredictionState,
|
|
33
|
+
Run,
|
|
34
|
+
RunSettings,
|
|
35
|
+
sample_of,
|
|
36
|
+
sha256_of,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
_FORMAT_BY_SUFFIX: dict[str, PredictionFormat] = {".txt": "text", ".jsonl": "jsonl"}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class _Plan:
|
|
44
|
+
"""Where each case's queries sit in the flat list sent to ``execute_all``."""
|
|
45
|
+
|
|
46
|
+
case: Case
|
|
47
|
+
prediction: Prediction | None
|
|
48
|
+
gold_index: int
|
|
49
|
+
pred_index: int
|
|
50
|
+
tied_index: int | None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def evaluate_files(
|
|
54
|
+
dataset: Path,
|
|
55
|
+
predictions: Path,
|
|
56
|
+
*,
|
|
57
|
+
db_dir: Path,
|
|
58
|
+
prediction_format: PredictionFormat | None = None,
|
|
59
|
+
allow_missing: bool = False,
|
|
60
|
+
tied: Path | None = None,
|
|
61
|
+
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
|
62
|
+
max_rows: int | None = None,
|
|
63
|
+
workers: int | None = None,
|
|
64
|
+
meta: Mapping[str, str] | None = None,
|
|
65
|
+
) -> Run:
|
|
66
|
+
"""Do what ``text2sql-eval eval`` does and return the run.
|
|
67
|
+
|
|
68
|
+
Reads the dataset and the predictions, executes and judges every case,
|
|
69
|
+
and records the inputs (with SHA-256 hashes), the settings and ``meta``.
|
|
70
|
+
Write the result with :func:`write_run`.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
dataset: BIRD question file.
|
|
74
|
+
predictions: Predictions, one SQL per line (``.txt``) or JSONL.
|
|
75
|
+
db_dir: Folder with ``<db_id>/<db_id>.sqlite`` databases.
|
|
76
|
+
prediction_format: ``"text"`` or ``"jsonl"``; ``None`` reads it from
|
|
77
|
+
the file extension (ADR 0027).
|
|
78
|
+
allow_missing: JSONL only: judge cases without a line as empty
|
|
79
|
+
predictions instead of failing.
|
|
80
|
+
tied: BIRD's tied-rows alternative gold file, used only for flags.
|
|
81
|
+
timeout: Seconds allowed per query.
|
|
82
|
+
max_rows: Row limit per query, or ``None`` for no limit.
|
|
83
|
+
workers: Worker threads, or ``None`` for the default.
|
|
84
|
+
meta: Facts about the run to record, such as the model name.
|
|
85
|
+
|
|
86
|
+
Raises:
|
|
87
|
+
InputError: if an input file is malformed, does not match the
|
|
88
|
+
dataset, or a database file is missing.
|
|
89
|
+
|
|
90
|
+
Example:
|
|
91
|
+
>>> run = evaluate_files(
|
|
92
|
+
... Path("examples/bird-mini/dev.json"),
|
|
93
|
+
... Path("examples/results/gpt-4o-mini-evidence.jsonl"),
|
|
94
|
+
... db_dir=Path("examples/bird-mini/dev_databases"),
|
|
95
|
+
... meta={"model": "gpt-4o-mini"},
|
|
96
|
+
... )
|
|
97
|
+
>>> round(run.summary.ex, 4)
|
|
98
|
+
0.6181
|
|
99
|
+
"""
|
|
100
|
+
pred_format = prediction_format or _format_from_extension(predictions)
|
|
101
|
+
if allow_missing and pred_format != "jsonl":
|
|
102
|
+
raise InputError(
|
|
103
|
+
predictions,
|
|
104
|
+
"--allow-missing only applies to JSONL predictions",
|
|
105
|
+
hint="text predictions need one line per case; leave a line empty instead",
|
|
106
|
+
)
|
|
107
|
+
loaded = load_bird(dataset)
|
|
108
|
+
if pred_format == "text":
|
|
109
|
+
by_id = load_text_predictions(predictions, loaded)
|
|
110
|
+
else:
|
|
111
|
+
by_id = load_jsonl_predictions(predictions, loaded, allow_missing=allow_missing)
|
|
112
|
+
tied_sql = None if tied is None else load_bird_tied(tied, loaded)
|
|
113
|
+
records = evaluate(
|
|
114
|
+
loaded,
|
|
115
|
+
by_id,
|
|
116
|
+
db_dir=db_dir,
|
|
117
|
+
tied=tied_sql,
|
|
118
|
+
timeout=timeout,
|
|
119
|
+
max_rows=max_rows,
|
|
120
|
+
workers=workers,
|
|
121
|
+
)
|
|
122
|
+
return Run(
|
|
123
|
+
tool_version=version("text2sql-eval"),
|
|
124
|
+
created_at=datetime.now(UTC)
|
|
125
|
+
.isoformat(timespec="seconds")
|
|
126
|
+
.replace("+00:00", "Z"),
|
|
127
|
+
dataset=FileRef(str(dataset), sha256_of(dataset)),
|
|
128
|
+
predictions=FileRef(str(predictions), sha256_of(predictions)),
|
|
129
|
+
prediction_format=pred_format,
|
|
130
|
+
tied=None if tied is None else FileRef(str(tied), sha256_of(tied)),
|
|
131
|
+
db_dir=str(db_dir),
|
|
132
|
+
settings=RunSettings(
|
|
133
|
+
timeout_seconds=timeout, max_rows=max_rows, allow_missing=allow_missing
|
|
134
|
+
),
|
|
135
|
+
meta=dict(meta or {}),
|
|
136
|
+
cases=records,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _format_from_extension(path: Path) -> PredictionFormat:
|
|
141
|
+
detected = _FORMAT_BY_SUFFIX.get(path.suffix.lower())
|
|
142
|
+
if detected is None:
|
|
143
|
+
raise InputError(
|
|
144
|
+
path,
|
|
145
|
+
f"cannot tell the prediction format from the extension '{path.suffix}'",
|
|
146
|
+
hint="use a .txt or .jsonl file, or pass --pred-format text|jsonl "
|
|
147
|
+
"(prediction_format= in Python)",
|
|
148
|
+
)
|
|
149
|
+
return detected
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def database_path(db_dir: Path, db_id: str) -> Path:
|
|
153
|
+
"""BIRD's layout: ``<db_dir>/<db_id>/<db_id>.sqlite``."""
|
|
154
|
+
return db_dir / db_id / f"{db_id}.sqlite"
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def evaluate(
|
|
158
|
+
dataset: Dataset,
|
|
159
|
+
predictions: Mapping[str, Prediction],
|
|
160
|
+
*,
|
|
161
|
+
db_dir: Path,
|
|
162
|
+
tied: Mapping[str, str] | None = None,
|
|
163
|
+
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
|
164
|
+
max_rows: int | None = None,
|
|
165
|
+
workers: int | None = None,
|
|
166
|
+
) -> tuple[CaseRecord, ...]:
|
|
167
|
+
"""Run gold, predicted and tied-alternative SQL for every case and judge it.
|
|
168
|
+
|
|
169
|
+
A case with no entry in ``predictions`` is judged as an empty prediction
|
|
170
|
+
and recorded as ``"missing"`` (ADR 0023, ADR 0027). All queries go to one
|
|
171
|
+
thread pool, so slow cases do not hold up the rest.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
dataset: The cases, in the order the records are returned.
|
|
175
|
+
predictions: Predicted SQL by case id.
|
|
176
|
+
db_dir: Folder with ``<db_id>/<db_id>.sqlite`` databases.
|
|
177
|
+
tied: Tied-rows alternative gold SQL by case id, used only for flags.
|
|
178
|
+
timeout: Seconds allowed per query.
|
|
179
|
+
max_rows: Row limit per query, or ``None`` for no limit.
|
|
180
|
+
workers: Worker threads, or ``None`` for the default.
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
InputError: if a database file is missing.
|
|
184
|
+
|
|
185
|
+
Example:
|
|
186
|
+
Using the gold SQL as predictions judges every case correct:
|
|
187
|
+
|
|
188
|
+
>>> gold_as_pred = {c.id: Prediction(c.id, c.gold_sql) for c in dataset.cases}
|
|
189
|
+
>>> records = evaluate(dataset, gold_as_pred, db_dir=Path("dev_databases"))
|
|
190
|
+
>>> {status_of(record.verdict) for record in records}
|
|
191
|
+
{'correct'}
|
|
192
|
+
"""
|
|
193
|
+
tied = tied or {}
|
|
194
|
+
queries: list[Query] = []
|
|
195
|
+
plans: list[_Plan] = []
|
|
196
|
+
|
|
197
|
+
def add(case: Case, sql: str) -> int:
|
|
198
|
+
queries.append(Query(database_path(db_dir, case.db_id), sql))
|
|
199
|
+
return len(queries) - 1
|
|
200
|
+
|
|
201
|
+
for case in dataset.cases:
|
|
202
|
+
prediction = predictions.get(case.id)
|
|
203
|
+
tied_sql = tied.get(case.id)
|
|
204
|
+
plans.append(
|
|
205
|
+
_Plan(
|
|
206
|
+
case=case,
|
|
207
|
+
prediction=prediction,
|
|
208
|
+
gold_index=add(case, case.gold_sql),
|
|
209
|
+
pred_index=add(case, "" if prediction is None else prediction.sql),
|
|
210
|
+
tied_index=None if tied_sql is None else add(case, tied_sql),
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
results = execute_all(queries, timeout=timeout, max_rows=max_rows, workers=workers)
|
|
215
|
+
return tuple(_record(plan, results) for plan in plans)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _record(plan: _Plan, results: list[QueryResult]) -> CaseRecord:
|
|
219
|
+
case, prediction = plan.case, plan.prediction
|
|
220
|
+
gold = results[plan.gold_index]
|
|
221
|
+
pred = results[plan.pred_index]
|
|
222
|
+
tied = None if plan.tied_index is None else results[plan.tied_index]
|
|
223
|
+
state: PredictionState
|
|
224
|
+
if prediction is None:
|
|
225
|
+
state = "missing"
|
|
226
|
+
elif prediction.sql == "":
|
|
227
|
+
state = "empty"
|
|
228
|
+
else:
|
|
229
|
+
state = "given"
|
|
230
|
+
pred_sql = "" if prediction is None else prediction.sql
|
|
231
|
+
verdict = judge_case(gold, pred, pred_is_empty=state != "given", tied=tied)
|
|
232
|
+
return CaseRecord(
|
|
233
|
+
id=case.id,
|
|
234
|
+
db_id=case.db_id,
|
|
235
|
+
difficulty=case.difficulty,
|
|
236
|
+
question=case.question,
|
|
237
|
+
evidence=case.evidence,
|
|
238
|
+
gold_sql=case.gold_sql,
|
|
239
|
+
pred_sql=pred_sql,
|
|
240
|
+
prediction_state=state,
|
|
241
|
+
verdict=verdict,
|
|
242
|
+
gold=sample_of(gold),
|
|
243
|
+
pred=sample_of(pred),
|
|
244
|
+
pred_meta={} if prediction is None else prediction.meta,
|
|
245
|
+
diff=_diff(verdict, gold, pred, case.gold_sql, pred_sql),
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _diff(
|
|
250
|
+
verdict: Verdict,
|
|
251
|
+
gold: QueryResult,
|
|
252
|
+
pred: QueryResult,
|
|
253
|
+
gold_sql: str,
|
|
254
|
+
pred_sql: str,
|
|
255
|
+
) -> CaseDiff | None:
|
|
256
|
+
"""Facts for cases the model got wrong (ADR 0031).
|
|
257
|
+
|
|
258
|
+
None for correct cases, and for a gold query that failed: that is the
|
|
259
|
+
dataset's problem, not the prediction's.
|
|
260
|
+
"""
|
|
261
|
+
if isinstance(verdict, Correct):
|
|
262
|
+
return None
|
|
263
|
+
if isinstance(verdict, ExecError) and verdict.side == "gold":
|
|
264
|
+
return None
|
|
265
|
+
return CaseDiff(
|
|
266
|
+
output=compare_outputs(gold, pred),
|
|
267
|
+
structure=structural_facts(gold_sql, pred_sql),
|
|
268
|
+
)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Run SQL read-only against SQLite databases, with a per-query deadline.
|
|
2
|
+
|
|
3
|
+
The deadline is enforced by SQLite's progress handler (ADR 0020); defaults
|
|
4
|
+
follow ADR 0021.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sqlite3
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Literal
|
|
17
|
+
|
|
18
|
+
from text2sql_eval.errors import InputError
|
|
19
|
+
|
|
20
|
+
DEFAULT_TIMEOUT_SECONDS = 30.0
|
|
21
|
+
"""Same value as BIRD's official script, but applied to each query separately."""
|
|
22
|
+
|
|
23
|
+
# SQLite virtual machine instructions between deadline checks. Small enough to
|
|
24
|
+
# stop within milliseconds, large enough that the check costs nothing visible.
|
|
25
|
+
_PROGRESS_INTERVAL = 10_000
|
|
26
|
+
_FETCH_BATCH = 1_000
|
|
27
|
+
|
|
28
|
+
SqlValue = int | float | str | bytes | None
|
|
29
|
+
"""The Python types sqlite3 returns for SQLite values (no type conversion)."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Rows:
|
|
34
|
+
"""A query that finished. ``columns`` are the result column names."""
|
|
35
|
+
|
|
36
|
+
columns: tuple[str, ...]
|
|
37
|
+
rows: tuple[tuple[SqlValue, ...], ...]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class QueryFailure:
|
|
42
|
+
"""A query that did not produce rows.
|
|
43
|
+
|
|
44
|
+
``kind`` is ``"error"`` (SQLite rejected or failed the query),
|
|
45
|
+
``"timeout"`` (the deadline passed) or ``"row_limit"`` (more rows than
|
|
46
|
+
allowed).
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
kind: Literal["error", "timeout", "row_limit"]
|
|
50
|
+
message: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
QueryResult = Rows | QueryFailure
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class Query:
|
|
58
|
+
"""One SQL statement to run against one database file."""
|
|
59
|
+
|
|
60
|
+
db_path: Path
|
|
61
|
+
sql: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def default_workers() -> int:
|
|
65
|
+
"""Number of worker threads used when the caller does not choose."""
|
|
66
|
+
return min(8, os.cpu_count() or 1)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def execute_query(
|
|
70
|
+
db_path: Path,
|
|
71
|
+
sql: str,
|
|
72
|
+
*,
|
|
73
|
+
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
|
74
|
+
max_rows: int | None = None,
|
|
75
|
+
) -> QueryResult:
|
|
76
|
+
"""Run one statement on a read-only connection and collect its rows.
|
|
77
|
+
|
|
78
|
+
Failures of the SQL itself (syntax errors, unknown tables, writes, more
|
|
79
|
+
than one statement, the deadline, the row limit) are returned as
|
|
80
|
+
``QueryFailure`` rather than raised, because they are results to judge.
|
|
81
|
+
|
|
82
|
+
Raises:
|
|
83
|
+
InputError: if ``db_path`` is not an existing file.
|
|
84
|
+
|
|
85
|
+
Example:
|
|
86
|
+
>>> result = execute_query(Path("superhero.sqlite"), "SELECT COUNT(*) FROM superhero")
|
|
87
|
+
>>> result
|
|
88
|
+
Rows(columns=('COUNT(*)',), rows=((750,),))
|
|
89
|
+
"""
|
|
90
|
+
_require_database(db_path)
|
|
91
|
+
deadline = time.monotonic() + timeout
|
|
92
|
+
timed_out = False
|
|
93
|
+
|
|
94
|
+
def stop_after_deadline() -> int:
|
|
95
|
+
nonlocal timed_out
|
|
96
|
+
if time.monotonic() > deadline:
|
|
97
|
+
timed_out = True
|
|
98
|
+
return 1 # a non-zero return makes SQLite abort the statement
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
# as_uri() percent-encodes the path, so names with spaces or '#' work.
|
|
102
|
+
connection = sqlite3.connect(db_path.resolve().as_uri() + "?mode=ro", uri=True)
|
|
103
|
+
try:
|
|
104
|
+
connection.set_progress_handler(stop_after_deadline, _PROGRESS_INTERVAL)
|
|
105
|
+
cursor = connection.execute(sql)
|
|
106
|
+
columns = tuple(str(column[0]) for column in cursor.description or ())
|
|
107
|
+
rows: list[tuple[SqlValue, ...]] = []
|
|
108
|
+
while batch := cursor.fetchmany(_FETCH_BATCH):
|
|
109
|
+
rows.extend(batch)
|
|
110
|
+
if max_rows is not None and len(rows) > max_rows:
|
|
111
|
+
return QueryFailure("row_limit", f"more than {max_rows} rows")
|
|
112
|
+
return Rows(columns=columns, rows=tuple(rows))
|
|
113
|
+
except sqlite3.Error as exc:
|
|
114
|
+
if timed_out:
|
|
115
|
+
return QueryFailure("timeout", f"did not finish within {timeout:g} s")
|
|
116
|
+
return QueryFailure("error", str(exc))
|
|
117
|
+
finally:
|
|
118
|
+
connection.close()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def execute_all(
|
|
122
|
+
queries: Sequence[Query],
|
|
123
|
+
*,
|
|
124
|
+
timeout: float = DEFAULT_TIMEOUT_SECONDS,
|
|
125
|
+
max_rows: int | None = None,
|
|
126
|
+
workers: int | None = None,
|
|
127
|
+
) -> list[QueryResult]:
|
|
128
|
+
"""Run many queries in parallel threads; results come back in input order.
|
|
129
|
+
|
|
130
|
+
Each query opens its own connection. sqlite3 releases the GIL while a
|
|
131
|
+
statement runs, so threads execute queries truly in parallel.
|
|
132
|
+
|
|
133
|
+
Raises:
|
|
134
|
+
InputError: if any database file is missing (checked before running).
|
|
135
|
+
ValueError: if ``workers`` is less than 1.
|
|
136
|
+
|
|
137
|
+
Example:
|
|
138
|
+
>>> db = Path("superhero.sqlite")
|
|
139
|
+
>>> execute_all([Query(db, "SELECT 1"), Query(db, "SELEC")], workers=2)
|
|
140
|
+
[Rows(columns=('1',), rows=((1,),)), QueryFailure(kind='error', message=...)]
|
|
141
|
+
"""
|
|
142
|
+
if workers is not None and workers < 1:
|
|
143
|
+
raise ValueError(f"workers must be at least 1, got {workers}")
|
|
144
|
+
for db_path in sorted({query.db_path for query in queries}):
|
|
145
|
+
_require_database(db_path)
|
|
146
|
+
|
|
147
|
+
def run(query: Query) -> QueryResult:
|
|
148
|
+
return execute_query(
|
|
149
|
+
query.db_path, query.sql, timeout=timeout, max_rows=max_rows
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
with ThreadPoolExecutor(max_workers=workers or default_workers()) as pool:
|
|
153
|
+
return list(pool.map(run, queries))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _require_database(db_path: Path) -> None:
|
|
157
|
+
if not db_path.is_file():
|
|
158
|
+
raise InputError(
|
|
159
|
+
db_path,
|
|
160
|
+
"database file not found",
|
|
161
|
+
hint="databases are expected at <db-dir>/<db_id>/<db_id>.sqlite",
|
|
162
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""What differs between a wrong prediction and the gold answer (ADR 0031, 0033).
|
|
2
|
+
|
|
3
|
+
``output`` compares executed results; ``structure`` compares parsed SQL.
|
|
4
|
+
Not part of the public API yet.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from text2sql_eval.facts.output import OutputDiff
|
|
12
|
+
from text2sql_eval.facts.structure import StructureReport
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class CaseDiff:
|
|
17
|
+
"""Everything the tool can say about why one case is not correct.
|
|
18
|
+
|
|
19
|
+
``output`` is the output difference, ``structure`` the SQL structure
|
|
20
|
+
difference.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
output: OutputDiff
|
|
24
|
+
structure: StructureReport
|