text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,43 @@
1
+ """Errors raised when user-supplied input cannot be used."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+
8
+ class InputError(Exception):
9
+ """An input file is malformed or does not match the dataset.
10
+
11
+ The message says where the problem is (file, and line or item when known),
12
+ what is wrong, and how to fix it, so it can be shown to users as is.
13
+
14
+ Example:
15
+ >>> err = InputError(Path("preds.jsonl"), "missing field 'sql'",
16
+ ... location="line 3", hint="add an 'sql' string")
17
+ >>> print(err)
18
+ preds.jsonl, line 3: missing field 'sql'
19
+ hint: add an 'sql' string
20
+ """
21
+
22
+ def __init__(
23
+ self,
24
+ path: Path,
25
+ problem: str,
26
+ *,
27
+ location: str | None = None,
28
+ hint: str | None = None,
29
+ ) -> None:
30
+ self.path = path
31
+ self.problem = problem
32
+ self.location = location
33
+ self.hint = hint
34
+ super().__init__(self._format())
35
+
36
+ def _format(self) -> str:
37
+ where = (
38
+ str(self.path) if self.location is None else f"{self.path}, {self.location}"
39
+ )
40
+ message = f"{where}: {self.problem}"
41
+ if self.hint is not None:
42
+ message += f"\n hint: {self.hint}"
43
+ return message
@@ -0,0 +1,268 @@
1
+ """Execute and judge every case of a dataset against a set of predictions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from dataclasses import dataclass
7
+ from datetime import UTC, datetime
8
+ from importlib.metadata import version
9
+ from pathlib import Path
10
+
11
+ from text2sql_eval.datasets.bird import Case, Dataset, load_bird, load_bird_tied
12
+ from text2sql_eval.datasets.predictions import (
13
+ Prediction,
14
+ load_jsonl_predictions,
15
+ load_text_predictions,
16
+ )
17
+ from text2sql_eval.errors import InputError
18
+ from text2sql_eval.execution.executor import (
19
+ DEFAULT_TIMEOUT_SECONDS,
20
+ Query,
21
+ QueryResult,
22
+ execute_all,
23
+ )
24
+ from text2sql_eval.facts import CaseDiff
25
+ from text2sql_eval.facts.output import compare_outputs
26
+ from text2sql_eval.facts.structure import structural_facts
27
+ from text2sql_eval.judge.ex import Correct, ExecError, Verdict, judge_case
28
+ from text2sql_eval.runfile import (
29
+ CaseRecord,
30
+ FileRef,
31
+ PredictionFormat,
32
+ PredictionState,
33
+ Run,
34
+ RunSettings,
35
+ sample_of,
36
+ sha256_of,
37
+ )
38
+
39
+ _FORMAT_BY_SUFFIX: dict[str, PredictionFormat] = {".txt": "text", ".jsonl": "jsonl"}
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class _Plan:
44
+ """Where each case's queries sit in the flat list sent to ``execute_all``."""
45
+
46
+ case: Case
47
+ prediction: Prediction | None
48
+ gold_index: int
49
+ pred_index: int
50
+ tied_index: int | None
51
+
52
+
53
+ def evaluate_files(
54
+ dataset: Path,
55
+ predictions: Path,
56
+ *,
57
+ db_dir: Path,
58
+ prediction_format: PredictionFormat | None = None,
59
+ allow_missing: bool = False,
60
+ tied: Path | None = None,
61
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
62
+ max_rows: int | None = None,
63
+ workers: int | None = None,
64
+ meta: Mapping[str, str] | None = None,
65
+ ) -> Run:
66
+ """Do what ``text2sql-eval eval`` does and return the run.
67
+
68
+ Reads the dataset and the predictions, executes and judges every case,
69
+ and records the inputs (with SHA-256 hashes), the settings and ``meta``.
70
+ Write the result with :func:`write_run`.
71
+
72
+ Args:
73
+ dataset: BIRD question file.
74
+ predictions: Predictions, one SQL per line (``.txt``) or JSONL.
75
+ db_dir: Folder with ``<db_id>/<db_id>.sqlite`` databases.
76
+ prediction_format: ``"text"`` or ``"jsonl"``; ``None`` reads it from
77
+ the file extension (ADR 0027).
78
+ allow_missing: JSONL only: judge cases without a line as empty
79
+ predictions instead of failing.
80
+ tied: BIRD's tied-rows alternative gold file, used only for flags.
81
+ timeout: Seconds allowed per query.
82
+ max_rows: Row limit per query, or ``None`` for no limit.
83
+ workers: Worker threads, or ``None`` for the default.
84
+ meta: Facts about the run to record, such as the model name.
85
+
86
+ Raises:
87
+ InputError: if an input file is malformed, does not match the
88
+ dataset, or a database file is missing.
89
+
90
+ Example:
91
+ >>> run = evaluate_files(
92
+ ... Path("examples/bird-mini/dev.json"),
93
+ ... Path("examples/results/gpt-4o-mini-evidence.jsonl"),
94
+ ... db_dir=Path("examples/bird-mini/dev_databases"),
95
+ ... meta={"model": "gpt-4o-mini"},
96
+ ... )
97
+ >>> round(run.summary.ex, 4)
98
+ 0.6181
99
+ """
100
+ pred_format = prediction_format or _format_from_extension(predictions)
101
+ if allow_missing and pred_format != "jsonl":
102
+ raise InputError(
103
+ predictions,
104
+ "--allow-missing only applies to JSONL predictions",
105
+ hint="text predictions need one line per case; leave a line empty instead",
106
+ )
107
+ loaded = load_bird(dataset)
108
+ if pred_format == "text":
109
+ by_id = load_text_predictions(predictions, loaded)
110
+ else:
111
+ by_id = load_jsonl_predictions(predictions, loaded, allow_missing=allow_missing)
112
+ tied_sql = None if tied is None else load_bird_tied(tied, loaded)
113
+ records = evaluate(
114
+ loaded,
115
+ by_id,
116
+ db_dir=db_dir,
117
+ tied=tied_sql,
118
+ timeout=timeout,
119
+ max_rows=max_rows,
120
+ workers=workers,
121
+ )
122
+ return Run(
123
+ tool_version=version("text2sql-eval"),
124
+ created_at=datetime.now(UTC)
125
+ .isoformat(timespec="seconds")
126
+ .replace("+00:00", "Z"),
127
+ dataset=FileRef(str(dataset), sha256_of(dataset)),
128
+ predictions=FileRef(str(predictions), sha256_of(predictions)),
129
+ prediction_format=pred_format,
130
+ tied=None if tied is None else FileRef(str(tied), sha256_of(tied)),
131
+ db_dir=str(db_dir),
132
+ settings=RunSettings(
133
+ timeout_seconds=timeout, max_rows=max_rows, allow_missing=allow_missing
134
+ ),
135
+ meta=dict(meta or {}),
136
+ cases=records,
137
+ )
138
+
139
+
140
+ def _format_from_extension(path: Path) -> PredictionFormat:
141
+ detected = _FORMAT_BY_SUFFIX.get(path.suffix.lower())
142
+ if detected is None:
143
+ raise InputError(
144
+ path,
145
+ f"cannot tell the prediction format from the extension '{path.suffix}'",
146
+ hint="use a .txt or .jsonl file, or pass --pred-format text|jsonl "
147
+ "(prediction_format= in Python)",
148
+ )
149
+ return detected
150
+
151
+
152
+ def database_path(db_dir: Path, db_id: str) -> Path:
153
+ """BIRD's layout: ``<db_dir>/<db_id>/<db_id>.sqlite``."""
154
+ return db_dir / db_id / f"{db_id}.sqlite"
155
+
156
+
157
+ def evaluate(
158
+ dataset: Dataset,
159
+ predictions: Mapping[str, Prediction],
160
+ *,
161
+ db_dir: Path,
162
+ tied: Mapping[str, str] | None = None,
163
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
164
+ max_rows: int | None = None,
165
+ workers: int | None = None,
166
+ ) -> tuple[CaseRecord, ...]:
167
+ """Run gold, predicted and tied-alternative SQL for every case and judge it.
168
+
169
+ A case with no entry in ``predictions`` is judged as an empty prediction
170
+ and recorded as ``"missing"`` (ADR 0023, ADR 0027). All queries go to one
171
+ thread pool, so slow cases do not hold up the rest.
172
+
173
+ Args:
174
+ dataset: The cases, in the order the records are returned.
175
+ predictions: Predicted SQL by case id.
176
+ db_dir: Folder with ``<db_id>/<db_id>.sqlite`` databases.
177
+ tied: Tied-rows alternative gold SQL by case id, used only for flags.
178
+ timeout: Seconds allowed per query.
179
+ max_rows: Row limit per query, or ``None`` for no limit.
180
+ workers: Worker threads, or ``None`` for the default.
181
+
182
+ Raises:
183
+ InputError: if a database file is missing.
184
+
185
+ Example:
186
+ Using the gold SQL as predictions judges every case correct:
187
+
188
+ >>> gold_as_pred = {c.id: Prediction(c.id, c.gold_sql) for c in dataset.cases}
189
+ >>> records = evaluate(dataset, gold_as_pred, db_dir=Path("dev_databases"))
190
+ >>> {status_of(record.verdict) for record in records}
191
+ {'correct'}
192
+ """
193
+ tied = tied or {}
194
+ queries: list[Query] = []
195
+ plans: list[_Plan] = []
196
+
197
+ def add(case: Case, sql: str) -> int:
198
+ queries.append(Query(database_path(db_dir, case.db_id), sql))
199
+ return len(queries) - 1
200
+
201
+ for case in dataset.cases:
202
+ prediction = predictions.get(case.id)
203
+ tied_sql = tied.get(case.id)
204
+ plans.append(
205
+ _Plan(
206
+ case=case,
207
+ prediction=prediction,
208
+ gold_index=add(case, case.gold_sql),
209
+ pred_index=add(case, "" if prediction is None else prediction.sql),
210
+ tied_index=None if tied_sql is None else add(case, tied_sql),
211
+ )
212
+ )
213
+
214
+ results = execute_all(queries, timeout=timeout, max_rows=max_rows, workers=workers)
215
+ return tuple(_record(plan, results) for plan in plans)
216
+
217
+
218
+ def _record(plan: _Plan, results: list[QueryResult]) -> CaseRecord:
219
+ case, prediction = plan.case, plan.prediction
220
+ gold = results[plan.gold_index]
221
+ pred = results[plan.pred_index]
222
+ tied = None if plan.tied_index is None else results[plan.tied_index]
223
+ state: PredictionState
224
+ if prediction is None:
225
+ state = "missing"
226
+ elif prediction.sql == "":
227
+ state = "empty"
228
+ else:
229
+ state = "given"
230
+ pred_sql = "" if prediction is None else prediction.sql
231
+ verdict = judge_case(gold, pred, pred_is_empty=state != "given", tied=tied)
232
+ return CaseRecord(
233
+ id=case.id,
234
+ db_id=case.db_id,
235
+ difficulty=case.difficulty,
236
+ question=case.question,
237
+ evidence=case.evidence,
238
+ gold_sql=case.gold_sql,
239
+ pred_sql=pred_sql,
240
+ prediction_state=state,
241
+ verdict=verdict,
242
+ gold=sample_of(gold),
243
+ pred=sample_of(pred),
244
+ pred_meta={} if prediction is None else prediction.meta,
245
+ diff=_diff(verdict, gold, pred, case.gold_sql, pred_sql),
246
+ )
247
+
248
+
249
+ def _diff(
250
+ verdict: Verdict,
251
+ gold: QueryResult,
252
+ pred: QueryResult,
253
+ gold_sql: str,
254
+ pred_sql: str,
255
+ ) -> CaseDiff | None:
256
+ """Facts for cases the model got wrong (ADR 0031).
257
+
258
+ None for correct cases, and for a gold query that failed: that is the
259
+ dataset's problem, not the prediction's.
260
+ """
261
+ if isinstance(verdict, Correct):
262
+ return None
263
+ if isinstance(verdict, ExecError) and verdict.side == "gold":
264
+ return None
265
+ return CaseDiff(
266
+ output=compare_outputs(gold, pred),
267
+ structure=structural_facts(gold_sql, pred_sql),
268
+ )
@@ -0,0 +1,5 @@
1
+ """Read-only SQL execution with deadlines.
2
+
3
+ Not part of the public API yet; the public surface is decided once ``eval``
4
+ works end to end.
5
+ """
@@ -0,0 +1,162 @@
1
+ """Run SQL read-only against SQLite databases, with a per-query deadline.
2
+
3
+ The deadline is enforced by SQLite's progress handler (ADR 0020); defaults
4
+ follow ADR 0021.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import os
10
+ import sqlite3
11
+ import time
12
+ from collections.abc import Sequence
13
+ from concurrent.futures import ThreadPoolExecutor
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Literal
17
+
18
+ from text2sql_eval.errors import InputError
19
+
20
+ DEFAULT_TIMEOUT_SECONDS = 30.0
21
+ """Same value as BIRD's official script, but applied to each query separately."""
22
+
23
+ # SQLite virtual machine instructions between deadline checks. Small enough to
24
+ # stop within milliseconds, large enough that the check costs nothing visible.
25
+ _PROGRESS_INTERVAL = 10_000
26
+ _FETCH_BATCH = 1_000
27
+
28
+ SqlValue = int | float | str | bytes | None
29
+ """The Python types sqlite3 returns for SQLite values (no type conversion)."""
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class Rows:
34
+ """A query that finished. ``columns`` are the result column names."""
35
+
36
+ columns: tuple[str, ...]
37
+ rows: tuple[tuple[SqlValue, ...], ...]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class QueryFailure:
42
+ """A query that did not produce rows.
43
+
44
+ ``kind`` is ``"error"`` (SQLite rejected or failed the query),
45
+ ``"timeout"`` (the deadline passed) or ``"row_limit"`` (more rows than
46
+ allowed).
47
+ """
48
+
49
+ kind: Literal["error", "timeout", "row_limit"]
50
+ message: str
51
+
52
+
53
+ QueryResult = Rows | QueryFailure
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class Query:
58
+ """One SQL statement to run against one database file."""
59
+
60
+ db_path: Path
61
+ sql: str
62
+
63
+
64
+ def default_workers() -> int:
65
+ """Number of worker threads used when the caller does not choose."""
66
+ return min(8, os.cpu_count() or 1)
67
+
68
+
69
+ def execute_query(
70
+ db_path: Path,
71
+ sql: str,
72
+ *,
73
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
74
+ max_rows: int | None = None,
75
+ ) -> QueryResult:
76
+ """Run one statement on a read-only connection and collect its rows.
77
+
78
+ Failures of the SQL itself (syntax errors, unknown tables, writes, more
79
+ than one statement, the deadline, the row limit) are returned as
80
+ ``QueryFailure`` rather than raised, because they are results to judge.
81
+
82
+ Raises:
83
+ InputError: if ``db_path`` is not an existing file.
84
+
85
+ Example:
86
+ >>> result = execute_query(Path("superhero.sqlite"), "SELECT COUNT(*) FROM superhero")
87
+ >>> result
88
+ Rows(columns=('COUNT(*)',), rows=((750,),))
89
+ """
90
+ _require_database(db_path)
91
+ deadline = time.monotonic() + timeout
92
+ timed_out = False
93
+
94
+ def stop_after_deadline() -> int:
95
+ nonlocal timed_out
96
+ if time.monotonic() > deadline:
97
+ timed_out = True
98
+ return 1 # a non-zero return makes SQLite abort the statement
99
+ return 0
100
+
101
+ # as_uri() percent-encodes the path, so names with spaces or '#' work.
102
+ connection = sqlite3.connect(db_path.resolve().as_uri() + "?mode=ro", uri=True)
103
+ try:
104
+ connection.set_progress_handler(stop_after_deadline, _PROGRESS_INTERVAL)
105
+ cursor = connection.execute(sql)
106
+ columns = tuple(str(column[0]) for column in cursor.description or ())
107
+ rows: list[tuple[SqlValue, ...]] = []
108
+ while batch := cursor.fetchmany(_FETCH_BATCH):
109
+ rows.extend(batch)
110
+ if max_rows is not None and len(rows) > max_rows:
111
+ return QueryFailure("row_limit", f"more than {max_rows} rows")
112
+ return Rows(columns=columns, rows=tuple(rows))
113
+ except sqlite3.Error as exc:
114
+ if timed_out:
115
+ return QueryFailure("timeout", f"did not finish within {timeout:g} s")
116
+ return QueryFailure("error", str(exc))
117
+ finally:
118
+ connection.close()
119
+
120
+
121
+ def execute_all(
122
+ queries: Sequence[Query],
123
+ *,
124
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
125
+ max_rows: int | None = None,
126
+ workers: int | None = None,
127
+ ) -> list[QueryResult]:
128
+ """Run many queries in parallel threads; results come back in input order.
129
+
130
+ Each query opens its own connection. sqlite3 releases the GIL while a
131
+ statement runs, so threads execute queries truly in parallel.
132
+
133
+ Raises:
134
+ InputError: if any database file is missing (checked before running).
135
+ ValueError: if ``workers`` is less than 1.
136
+
137
+ Example:
138
+ >>> db = Path("superhero.sqlite")
139
+ >>> execute_all([Query(db, "SELECT 1"), Query(db, "SELEC")], workers=2)
140
+ [Rows(columns=('1',), rows=((1,),)), QueryFailure(kind='error', message=...)]
141
+ """
142
+ if workers is not None and workers < 1:
143
+ raise ValueError(f"workers must be at least 1, got {workers}")
144
+ for db_path in sorted({query.db_path for query in queries}):
145
+ _require_database(db_path)
146
+
147
+ def run(query: Query) -> QueryResult:
148
+ return execute_query(
149
+ query.db_path, query.sql, timeout=timeout, max_rows=max_rows
150
+ )
151
+
152
+ with ThreadPoolExecutor(max_workers=workers or default_workers()) as pool:
153
+ return list(pool.map(run, queries))
154
+
155
+
156
+ def _require_database(db_path: Path) -> None:
157
+ if not db_path.is_file():
158
+ raise InputError(
159
+ db_path,
160
+ "database file not found",
161
+ hint="databases are expected at <db-dir>/<db_id>/<db_id>.sqlite",
162
+ )
@@ -0,0 +1,24 @@
1
+ """What differs between a wrong prediction and the gold answer (ADR 0031, 0033).
2
+
3
+ ``output`` compares executed results; ``structure`` compares parsed SQL.
4
+ Not part of the public API yet.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+
11
+ from text2sql_eval.facts.output import OutputDiff
12
+ from text2sql_eval.facts.structure import StructureReport
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class CaseDiff:
17
+ """Everything the tool can say about why one case is not correct.
18
+
19
+ ``output`` is the output difference, ``structure`` the SQL structure
20
+ difference.
21
+ """
22
+
23
+ output: OutputDiff
24
+ structure: StructureReport