text2sql-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- text2sql_eval/__init__.py +101 -0
- text2sql_eval/cli/__init__.py +1 -0
- text2sql_eval/cli/main.py +482 -0
- text2sql_eval/compare/__init__.py +1 -0
- text2sql_eval/compare/runs.py +205 -0
- text2sql_eval/csv_safety.py +32 -0
- text2sql_eval/datasets/__init__.py +5 -0
- text2sql_eval/datasets/_json.py +73 -0
- text2sql_eval/datasets/bird.py +187 -0
- text2sql_eval/datasets/predictions.py +187 -0
- text2sql_eval/errors.py +43 -0
- text2sql_eval/evaluate.py +268 -0
- text2sql_eval/execution/__init__.py +5 -0
- text2sql_eval/execution/executor.py +162 -0
- text2sql_eval/facts/__init__.py +24 -0
- text2sql_eval/facts/output.py +204 -0
- text2sql_eval/facts/structure.py +299 -0
- text2sql_eval/judge/__init__.py +5 -0
- text2sql_eval/judge/ex.py +179 -0
- text2sql_eval/py.typed +0 -0
- text2sql_eval/report/__init__.py +4 -0
- text2sql_eval/report/csv_report.py +81 -0
- text2sql_eval/report/highlight.py +238 -0
- text2sql_eval/report/html_report.py +595 -0
- text2sql_eval/report/templates/report.html.j2 +284 -0
- text2sql_eval/runfile.py +585 -0
- text2sql_eval-0.1.0.dist-info/METADATA +176 -0
- text2sql_eval-0.1.0.dist-info/RECORD +31 -0
- text2sql_eval-0.1.0.dist-info/WHEEL +4 -0
- text2sql_eval-0.1.0.dist-info/entry_points.txt +3 -0
- text2sql_eval-0.1.0.dist-info/licenses/LICENSE +25 -0
text2sql_eval/runfile.py
ADDED
|
@@ -0,0 +1,585 @@
|
|
|
1
|
+
"""The run file: one JSON document per evaluation run (ADR 0026).
|
|
2
|
+
|
|
3
|
+
``report`` and ``compare`` read it, and people open it directly, so the format
|
|
4
|
+
is documented in ``docs/reference/run-file.md`` and versioned by
|
|
5
|
+
``SCHEMA_VERSION``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
from collections import Counter
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Literal, TypeVar, get_args
|
|
17
|
+
|
|
18
|
+
from text2sql_eval.datasets._json import (
|
|
19
|
+
as_array,
|
|
20
|
+
as_object,
|
|
21
|
+
json_type_name,
|
|
22
|
+
parse_json,
|
|
23
|
+
read_utf8,
|
|
24
|
+
)
|
|
25
|
+
from text2sql_eval.errors import InputError
|
|
26
|
+
from text2sql_eval.execution.executor import QueryFailure, Rows, SqlValue
|
|
27
|
+
from text2sql_eval.facts import CaseDiff
|
|
28
|
+
from text2sql_eval.facts.output import OutputDiff, OutputDifference, Relation
|
|
29
|
+
from text2sql_eval.facts.structure import Fact, FactKind, StructureReport
|
|
30
|
+
from text2sql_eval.judge.ex import (
|
|
31
|
+
Correct,
|
|
32
|
+
ExecError,
|
|
33
|
+
Flag,
|
|
34
|
+
FlagKind,
|
|
35
|
+
Incorrect,
|
|
36
|
+
Verdict,
|
|
37
|
+
status_of,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
SCHEMA_VERSION = 3
|
|
41
|
+
SAMPLE_ROWS = 20
|
|
42
|
+
"""Rows kept per query result; ``row_count`` still says how many there were."""
|
|
43
|
+
|
|
44
|
+
# A bound TypeVar instead of 3.12's ``def f[T: str]`` (ADR 0009).
|
|
45
|
+
_Choice = TypeVar("_Choice", bound=str)
|
|
46
|
+
|
|
47
|
+
Side = Literal["gold", "pred"]
|
|
48
|
+
|
|
49
|
+
PredictionState = Literal["given", "empty", "missing"]
|
|
50
|
+
PredictionFormat = Literal["text", "jsonl"]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class FileRef:
|
|
55
|
+
"""An input file as it was when the run started."""
|
|
56
|
+
|
|
57
|
+
path: str
|
|
58
|
+
sha256: str
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class ResultSample:
|
|
63
|
+
"""The first ``SAMPLE_ROWS`` rows of a query result, and the full count."""
|
|
64
|
+
|
|
65
|
+
columns: tuple[str, ...]
|
|
66
|
+
rows: tuple[tuple[SqlValue, ...], ...]
|
|
67
|
+
row_count: int
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
QuerySummary = ResultSample | QueryFailure
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class CaseRecord:
|
|
75
|
+
"""Everything needed to show and compare one judged case."""
|
|
76
|
+
|
|
77
|
+
id: str
|
|
78
|
+
db_id: str
|
|
79
|
+
difficulty: str | None
|
|
80
|
+
question: str
|
|
81
|
+
evidence: str
|
|
82
|
+
gold_sql: str
|
|
83
|
+
pred_sql: str
|
|
84
|
+
prediction_state: PredictionState
|
|
85
|
+
verdict: Verdict
|
|
86
|
+
gold: QuerySummary
|
|
87
|
+
pred: QuerySummary
|
|
88
|
+
pred_meta: dict[str, object] = field(default_factory=dict[str, object])
|
|
89
|
+
diff: CaseDiff | None = None
|
|
90
|
+
"""Why the case is not correct; ``None`` for correct cases (ADR 0031)."""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass(frozen=True)
|
|
94
|
+
class RunSettings:
|
|
95
|
+
"""How the run executed queries, so a later run can tell if they differ."""
|
|
96
|
+
|
|
97
|
+
timeout_seconds: float
|
|
98
|
+
max_rows: int | None
|
|
99
|
+
allow_missing: bool
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True)
|
|
103
|
+
class Summary:
|
|
104
|
+
"""Number of cases per status; ``total`` is their sum."""
|
|
105
|
+
|
|
106
|
+
total: int
|
|
107
|
+
correct: int
|
|
108
|
+
incorrect: int
|
|
109
|
+
error: int
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def ex(self) -> float:
|
|
113
|
+
"""Execution accuracy: correct cases over all cases (0.0 for no cases)."""
|
|
114
|
+
return self.correct / self.total if self.total else 0.0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(frozen=True)
|
|
118
|
+
class Run:
|
|
119
|
+
"""One evaluation run: inputs, settings, user metadata and judged cases."""
|
|
120
|
+
|
|
121
|
+
tool_version: str
|
|
122
|
+
created_at: str
|
|
123
|
+
dataset: FileRef
|
|
124
|
+
predictions: FileRef
|
|
125
|
+
prediction_format: PredictionFormat
|
|
126
|
+
tied: FileRef | None
|
|
127
|
+
db_dir: str
|
|
128
|
+
settings: RunSettings
|
|
129
|
+
meta: dict[str, str]
|
|
130
|
+
cases: tuple[CaseRecord, ...]
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def summary(self) -> Summary:
|
|
134
|
+
counts = Counter(status_of(case.verdict) for case in self.cases)
|
|
135
|
+
return Summary(
|
|
136
|
+
total=len(self.cases),
|
|
137
|
+
correct=counts["correct"],
|
|
138
|
+
incorrect=counts["incorrect"],
|
|
139
|
+
error=counts["error"],
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def sha256_of(path: Path) -> str:
|
|
144
|
+
"""Hex SHA-256 of a file, so a later run can tell whether inputs changed."""
|
|
145
|
+
digest = hashlib.sha256()
|
|
146
|
+
with path.open("rb") as file:
|
|
147
|
+
while chunk := file.read(1 << 20):
|
|
148
|
+
digest.update(chunk)
|
|
149
|
+
return digest.hexdigest()
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def sample_of(result: Rows | QueryFailure) -> QuerySummary:
|
|
153
|
+
"""Keep the first ``SAMPLE_ROWS`` rows of a result for the run file."""
|
|
154
|
+
if isinstance(result, QueryFailure):
|
|
155
|
+
return result
|
|
156
|
+
return ResultSample(
|
|
157
|
+
columns=result.columns,
|
|
158
|
+
rows=result.rows[:SAMPLE_ROWS],
|
|
159
|
+
row_count=len(result.rows),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# --- writing ---------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def write_run(run: Run, path: Path) -> None:
|
|
167
|
+
"""Write ``run`` as UTF-8 JSON, creating parent directories as needed.
|
|
168
|
+
|
|
169
|
+
Example:
|
|
170
|
+
>>> write_run(run, Path("runs/v3.json"))
|
|
171
|
+
>>> read_run(Path("runs/v3.json")) == run
|
|
172
|
+
True
|
|
173
|
+
"""
|
|
174
|
+
summary = run.summary
|
|
175
|
+
document: dict[str, object] = {
|
|
176
|
+
"schema_version": SCHEMA_VERSION,
|
|
177
|
+
"tool": {"name": "text2sql-eval", "version": run.tool_version},
|
|
178
|
+
"created_at": run.created_at,
|
|
179
|
+
"inputs": {
|
|
180
|
+
"dataset": _file_ref(run.dataset),
|
|
181
|
+
"predictions": {
|
|
182
|
+
**_file_ref(run.predictions),
|
|
183
|
+
"format": run.prediction_format,
|
|
184
|
+
},
|
|
185
|
+
"tied": None if run.tied is None else _file_ref(run.tied),
|
|
186
|
+
"db_dir": run.db_dir,
|
|
187
|
+
},
|
|
188
|
+
"settings": {
|
|
189
|
+
"timeout_seconds": run.settings.timeout_seconds,
|
|
190
|
+
"max_rows": run.settings.max_rows,
|
|
191
|
+
"allow_missing": run.settings.allow_missing,
|
|
192
|
+
},
|
|
193
|
+
"meta": run.meta,
|
|
194
|
+
"summary": {
|
|
195
|
+
"total": summary.total,
|
|
196
|
+
"correct": summary.correct,
|
|
197
|
+
"incorrect": summary.incorrect,
|
|
198
|
+
"error": summary.error,
|
|
199
|
+
"ex": summary.ex,
|
|
200
|
+
},
|
|
201
|
+
"cases": [_case_to_json(case) for case in run.cases],
|
|
202
|
+
}
|
|
203
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
204
|
+
# allow_nan=False: NaN and Infinity are not JSON; _encode_value spells them out.
|
|
205
|
+
text = json.dumps(document, ensure_ascii=False, indent=2, allow_nan=False)
|
|
206
|
+
path.write_text(text + "\n", encoding="utf-8")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _file_ref(ref: FileRef) -> dict[str, object]:
|
|
210
|
+
return {"path": ref.path, "sha256": ref.sha256}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _case_to_json(case: CaseRecord) -> dict[str, object]:
|
|
214
|
+
verdict = case.verdict
|
|
215
|
+
error = (
|
|
216
|
+
{"side": verdict.side, "kind": verdict.kind, "message": verdict.message}
|
|
217
|
+
if isinstance(verdict, ExecError)
|
|
218
|
+
else None
|
|
219
|
+
)
|
|
220
|
+
return {
|
|
221
|
+
"id": case.id,
|
|
222
|
+
"db_id": case.db_id,
|
|
223
|
+
"difficulty": case.difficulty,
|
|
224
|
+
"question": case.question,
|
|
225
|
+
"evidence": case.evidence,
|
|
226
|
+
"gold_sql": case.gold_sql,
|
|
227
|
+
"pred_sql": case.pred_sql,
|
|
228
|
+
"prediction_state": case.prediction_state,
|
|
229
|
+
"pred_meta": case.pred_meta,
|
|
230
|
+
"status": status_of(verdict),
|
|
231
|
+
"error": error,
|
|
232
|
+
"flags": [{"kind": flag.kind, "detail": flag.detail} for flag in verdict.flags],
|
|
233
|
+
"gold": _summary_to_json(case.gold),
|
|
234
|
+
"pred": _summary_to_json(case.pred),
|
|
235
|
+
"diff": None if case.diff is None else _diff_to_json(case.diff),
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _diff_to_json(diff: CaseDiff) -> dict[str, object]:
|
|
240
|
+
output, structure = diff.output, diff.structure
|
|
241
|
+
return {
|
|
242
|
+
"output": {
|
|
243
|
+
"differences": list(output.differences),
|
|
244
|
+
"gold_columns": output.gold_columns,
|
|
245
|
+
"pred_columns": output.pred_columns,
|
|
246
|
+
"gold_distinct_rows": output.gold_distinct_rows,
|
|
247
|
+
"pred_distinct_rows": output.pred_distinct_rows,
|
|
248
|
+
"gold_contained": output.gold_contained,
|
|
249
|
+
"relation": output.relation,
|
|
250
|
+
},
|
|
251
|
+
"sql_structure": {
|
|
252
|
+
"differences": [
|
|
253
|
+
{"kind": fact.kind, "gold": fact.gold, "pred": fact.pred}
|
|
254
|
+
for fact in structure.facts
|
|
255
|
+
],
|
|
256
|
+
"parse_failed": structure.parse_failed,
|
|
257
|
+
},
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _summary_to_json(summary: QuerySummary) -> dict[str, object]:
|
|
262
|
+
if isinstance(summary, QueryFailure):
|
|
263
|
+
return {"failure": {"kind": summary.kind, "message": summary.message}}
|
|
264
|
+
return {
|
|
265
|
+
"columns": list(summary.columns),
|
|
266
|
+
"rows": [[_encode_value(value) for value in row] for row in summary.rows],
|
|
267
|
+
"row_count": summary.row_count,
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _encode_value(value: SqlValue) -> object:
|
|
272
|
+
"""Map a SQLite value to JSON. Bytes and non-finite floats get tagged objects."""
|
|
273
|
+
if isinstance(value, bytes):
|
|
274
|
+
return {"$bytes": value.hex()}
|
|
275
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
276
|
+
return {"$float": repr(value)} # 'inf', '-inf' or 'nan'
|
|
277
|
+
return value
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
# --- reading ---------------------------------------------------------------
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def read_run(path: Path) -> Run:
|
|
284
|
+
"""Read a run file written by ``write_run``.
|
|
285
|
+
|
|
286
|
+
Raises:
|
|
287
|
+
InputError: if the file is unreadable, has another ``schema_version``,
|
|
288
|
+
or is missing fields or has fields of the wrong type.
|
|
289
|
+
|
|
290
|
+
Example:
|
|
291
|
+
>>> run = read_run(Path("examples/results/gpt-4o-mini-evidence.run.json"))
|
|
292
|
+
>>> run.summary.correct, run.summary.total
|
|
293
|
+
(267, 432)
|
|
294
|
+
"""
|
|
295
|
+
reader = _Reader(path)
|
|
296
|
+
document = parse_json(read_utf8(path), path)
|
|
297
|
+
root = as_object(document)
|
|
298
|
+
if root is None or root.get("schema_version") is None:
|
|
299
|
+
found = (
|
|
300
|
+
"it has no schema_version"
|
|
301
|
+
if root is not None
|
|
302
|
+
else f"it is a JSON {json_type_name(document)}"
|
|
303
|
+
)
|
|
304
|
+
raise InputError(
|
|
305
|
+
path,
|
|
306
|
+
f"not a run file: {found}",
|
|
307
|
+
hint="pass the file that `text2sql-eval eval --out` wrote",
|
|
308
|
+
)
|
|
309
|
+
version = root.get("schema_version")
|
|
310
|
+
if version != SCHEMA_VERSION:
|
|
311
|
+
newer = isinstance(version, int) and version > SCHEMA_VERSION
|
|
312
|
+
raise InputError(
|
|
313
|
+
path,
|
|
314
|
+
f"schema_version is {version!r}, but this tool reads {SCHEMA_VERSION}",
|
|
315
|
+
hint="upgrade text2sql-eval to read it"
|
|
316
|
+
if newer
|
|
317
|
+
else "re-run eval with this version of text2sql-eval",
|
|
318
|
+
)
|
|
319
|
+
tool = reader.obj(root.get("tool"), "tool")
|
|
320
|
+
inputs = reader.obj(root.get("inputs"), "inputs")
|
|
321
|
+
predictions = reader.obj(inputs.get("predictions"), "inputs.predictions")
|
|
322
|
+
prediction_format = reader.choice(
|
|
323
|
+
predictions.get("format"),
|
|
324
|
+
"inputs.predictions.format",
|
|
325
|
+
get_args(PredictionFormat),
|
|
326
|
+
)
|
|
327
|
+
tied_value = inputs.get("tied")
|
|
328
|
+
settings = reader.obj(root.get("settings"), "settings")
|
|
329
|
+
meta = reader.obj(root.get("meta"), "meta")
|
|
330
|
+
cases = reader.array(root.get("cases"), "cases")
|
|
331
|
+
return Run(
|
|
332
|
+
tool_version=reader.str_(tool.get("version"), "tool.version"),
|
|
333
|
+
created_at=reader.str_(root.get("created_at"), "created_at"),
|
|
334
|
+
dataset=reader.file_ref(inputs.get("dataset"), "inputs.dataset"),
|
|
335
|
+
predictions=reader.file_ref(predictions, "inputs.predictions"),
|
|
336
|
+
prediction_format=prediction_format,
|
|
337
|
+
tied=None if tied_value is None else reader.file_ref(tied_value, "inputs.tied"),
|
|
338
|
+
db_dir=reader.str_(inputs.get("db_dir"), "inputs.db_dir"),
|
|
339
|
+
settings=RunSettings(
|
|
340
|
+
timeout_seconds=reader.number(
|
|
341
|
+
settings.get("timeout_seconds"), "settings.timeout_seconds"
|
|
342
|
+
),
|
|
343
|
+
max_rows=reader.optional_int(settings.get("max_rows"), "settings.max_rows"),
|
|
344
|
+
allow_missing=reader.bool_(
|
|
345
|
+
settings.get("allow_missing"), "settings.allow_missing"
|
|
346
|
+
),
|
|
347
|
+
),
|
|
348
|
+
meta={key: reader.str_(value, f"meta.{key}") for key, value in meta.items()},
|
|
349
|
+
cases=tuple(
|
|
350
|
+
reader.case(value, f"cases[{index}]") for index, value in enumerate(cases)
|
|
351
|
+
),
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class _Reader:
|
|
356
|
+
"""Typed field access that reports the JSON path of anything malformed."""
|
|
357
|
+
|
|
358
|
+
def __init__(self, path: Path) -> None:
|
|
359
|
+
self.path = path
|
|
360
|
+
|
|
361
|
+
def fail(self, where: str, expected: str) -> InputError:
|
|
362
|
+
return InputError(
|
|
363
|
+
self.path,
|
|
364
|
+
f"{where} must be {expected}",
|
|
365
|
+
hint="the file may be damaged or not written by text2sql-eval",
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
def obj(self, value: object, where: str) -> dict[str, object]:
|
|
369
|
+
result = as_object(value)
|
|
370
|
+
if result is None:
|
|
371
|
+
raise self.fail(where, "an object")
|
|
372
|
+
return result
|
|
373
|
+
|
|
374
|
+
def array(self, value: object, where: str) -> list[object]:
|
|
375
|
+
result = as_array(value)
|
|
376
|
+
if result is None:
|
|
377
|
+
raise self.fail(where, "an array")
|
|
378
|
+
return result
|
|
379
|
+
|
|
380
|
+
def str_(self, value: object, where: str) -> str:
|
|
381
|
+
if not isinstance(value, str):
|
|
382
|
+
raise self.fail(where, "a string")
|
|
383
|
+
return value
|
|
384
|
+
|
|
385
|
+
def optional_str(self, value: object, where: str) -> str | None:
|
|
386
|
+
return None if value is None else self.str_(value, where)
|
|
387
|
+
|
|
388
|
+
def int_(self, value: object, where: str) -> int:
|
|
389
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
390
|
+
raise self.fail(where, "an integer")
|
|
391
|
+
return value
|
|
392
|
+
|
|
393
|
+
def optional_int(self, value: object, where: str) -> int | None:
|
|
394
|
+
return None if value is None else self.int_(value, where)
|
|
395
|
+
|
|
396
|
+
def number(self, value: object, where: str) -> float:
|
|
397
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
398
|
+
raise self.fail(where, "a number")
|
|
399
|
+
return float(value)
|
|
400
|
+
|
|
401
|
+
def bool_(self, value: object, where: str) -> bool:
|
|
402
|
+
if not isinstance(value, bool):
|
|
403
|
+
raise self.fail(where, "true or false")
|
|
404
|
+
return value
|
|
405
|
+
|
|
406
|
+
def choice(
|
|
407
|
+
self, value: object, where: str, options: tuple[_Choice, ...]
|
|
408
|
+
) -> _Choice:
|
|
409
|
+
for option in options:
|
|
410
|
+
if value == option:
|
|
411
|
+
return option
|
|
412
|
+
raise self.fail(where, "one of " + ", ".join(options))
|
|
413
|
+
|
|
414
|
+
def file_ref(self, value: object, where: str) -> FileRef:
|
|
415
|
+
obj = self.obj(value, where)
|
|
416
|
+
return FileRef(
|
|
417
|
+
path=self.str_(obj.get("path"), f"{where}.path"),
|
|
418
|
+
sha256=self.str_(obj.get("sha256"), f"{where}.sha256"),
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
def case(self, value: object, where: str) -> CaseRecord:
|
|
422
|
+
obj = self.obj(value, where)
|
|
423
|
+
flags = tuple(
|
|
424
|
+
self.flag(item, f"{where}.flags[{index}]")
|
|
425
|
+
for index, item in enumerate(self.array(obj.get("flags"), f"{where}.flags"))
|
|
426
|
+
)
|
|
427
|
+
return CaseRecord(
|
|
428
|
+
id=self.str_(obj.get("id"), f"{where}.id"),
|
|
429
|
+
db_id=self.str_(obj.get("db_id"), f"{where}.db_id"),
|
|
430
|
+
difficulty=self.optional_str(obj.get("difficulty"), f"{where}.difficulty"),
|
|
431
|
+
question=self.str_(obj.get("question"), f"{where}.question"),
|
|
432
|
+
evidence=self.str_(obj.get("evidence"), f"{where}.evidence"),
|
|
433
|
+
gold_sql=self.str_(obj.get("gold_sql"), f"{where}.gold_sql"),
|
|
434
|
+
pred_sql=self.str_(obj.get("pred_sql"), f"{where}.pred_sql"),
|
|
435
|
+
prediction_state=self.choice(
|
|
436
|
+
obj.get("prediction_state"),
|
|
437
|
+
f"{where}.prediction_state",
|
|
438
|
+
get_args(PredictionState),
|
|
439
|
+
),
|
|
440
|
+
verdict=self.verdict(obj, where, flags),
|
|
441
|
+
gold=self.summary(obj.get("gold"), f"{where}.gold"),
|
|
442
|
+
pred=self.summary(obj.get("pred"), f"{where}.pred"),
|
|
443
|
+
pred_meta=self.obj(obj.get("pred_meta"), f"{where}.pred_meta"),
|
|
444
|
+
diff=self.diff(obj.get("diff"), f"{where}.diff"),
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
def diff(self, value: object, where: str) -> CaseDiff | None:
|
|
448
|
+
if value is None:
|
|
449
|
+
return None
|
|
450
|
+
obj = self.obj(value, where)
|
|
451
|
+
out_where, sql_where = f"{where}.output", f"{where}.sql_structure"
|
|
452
|
+
output = self.obj(obj.get("output"), out_where)
|
|
453
|
+
structure = self.obj(obj.get("sql_structure"), sql_where)
|
|
454
|
+
relation: Relation | None = None
|
|
455
|
+
if (relation_value := output.get("relation")) is not None:
|
|
456
|
+
relation = self.choice(
|
|
457
|
+
relation_value, f"{out_where}.relation", get_args(Relation)
|
|
458
|
+
)
|
|
459
|
+
contained: bool | None = None
|
|
460
|
+
if (contained_value := output.get("gold_contained")) is not None:
|
|
461
|
+
contained = self.bool_(contained_value, f"{out_where}.gold_contained")
|
|
462
|
+
parse_failed: Side | None = None
|
|
463
|
+
if (parse_failed_value := structure.get("parse_failed")) is not None:
|
|
464
|
+
parse_failed = self.choice(
|
|
465
|
+
parse_failed_value, f"{sql_where}.parse_failed", get_args(Side)
|
|
466
|
+
)
|
|
467
|
+
differences = self.array(output.get("differences"), f"{out_where}.differences")
|
|
468
|
+
facts = self.array(structure.get("differences"), f"{sql_where}.differences")
|
|
469
|
+
return CaseDiff(
|
|
470
|
+
output=OutputDiff(
|
|
471
|
+
differences=tuple(
|
|
472
|
+
self.choice(
|
|
473
|
+
item,
|
|
474
|
+
f"{out_where}.differences[{index}]",
|
|
475
|
+
get_args(OutputDifference),
|
|
476
|
+
)
|
|
477
|
+
for index, item in enumerate(differences)
|
|
478
|
+
),
|
|
479
|
+
gold_columns=self.int_(
|
|
480
|
+
output.get("gold_columns"), f"{out_where}.gold_columns"
|
|
481
|
+
),
|
|
482
|
+
pred_columns=self.int_(
|
|
483
|
+
output.get("pred_columns"), f"{out_where}.pred_columns"
|
|
484
|
+
),
|
|
485
|
+
gold_distinct_rows=self.int_(
|
|
486
|
+
output.get("gold_distinct_rows"), f"{out_where}.gold_distinct_rows"
|
|
487
|
+
),
|
|
488
|
+
pred_distinct_rows=self.int_(
|
|
489
|
+
output.get("pred_distinct_rows"), f"{out_where}.pred_distinct_rows"
|
|
490
|
+
),
|
|
491
|
+
gold_contained=contained,
|
|
492
|
+
relation=relation,
|
|
493
|
+
),
|
|
494
|
+
structure=StructureReport(
|
|
495
|
+
facts=tuple(
|
|
496
|
+
self.fact(item, f"{sql_where}.differences[{index}]")
|
|
497
|
+
for index, item in enumerate(facts)
|
|
498
|
+
),
|
|
499
|
+
parse_failed=parse_failed,
|
|
500
|
+
),
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
def fact(self, value: object, where: str) -> Fact:
|
|
504
|
+
obj = self.obj(value, where)
|
|
505
|
+
kind: FactKind = self.choice(
|
|
506
|
+
obj.get("kind"), f"{where}.kind", get_args(FactKind)
|
|
507
|
+
)
|
|
508
|
+
return Fact(
|
|
509
|
+
kind=kind,
|
|
510
|
+
gold=self.str_(obj.get("gold"), f"{where}.gold"),
|
|
511
|
+
pred=self.str_(obj.get("pred"), f"{where}.pred"),
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
def flag(self, value: object, where: str) -> Flag:
|
|
515
|
+
obj = self.obj(value, where)
|
|
516
|
+
kind: FlagKind = self.choice(
|
|
517
|
+
obj.get("kind"), f"{where}.kind", get_args(FlagKind)
|
|
518
|
+
)
|
|
519
|
+
return Flag(kind=kind, detail=self.str_(obj.get("detail"), f"{where}.detail"))
|
|
520
|
+
|
|
521
|
+
def verdict(
|
|
522
|
+
self, obj: dict[str, object], where: str, flags: tuple[Flag, ...]
|
|
523
|
+
) -> Verdict:
|
|
524
|
+
status = self.choice(
|
|
525
|
+
obj.get("status"), f"{where}.status", ("correct", "incorrect", "error")
|
|
526
|
+
)
|
|
527
|
+
if status == "correct":
|
|
528
|
+
return Correct(flags=flags)
|
|
529
|
+
if status == "incorrect":
|
|
530
|
+
return Incorrect(flags=flags)
|
|
531
|
+
error = self.obj(obj.get("error"), f"{where}.error")
|
|
532
|
+
return ExecError(
|
|
533
|
+
side=self.choice(
|
|
534
|
+
error.get("side"), f"{where}.error.side", ("pred", "gold")
|
|
535
|
+
),
|
|
536
|
+
kind=self.choice(
|
|
537
|
+
error.get("kind"),
|
|
538
|
+
f"{where}.error.kind",
|
|
539
|
+
("error", "timeout", "row_limit"),
|
|
540
|
+
),
|
|
541
|
+
message=self.str_(error.get("message"), f"{where}.error.message"),
|
|
542
|
+
flags=flags,
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
def summary(self, value: object, where: str) -> QuerySummary:
|
|
546
|
+
obj = self.obj(value, where)
|
|
547
|
+
if "failure" in obj:
|
|
548
|
+
failure = self.obj(obj["failure"], f"{where}.failure")
|
|
549
|
+
return QueryFailure(
|
|
550
|
+
kind=self.choice(
|
|
551
|
+
failure.get("kind"),
|
|
552
|
+
f"{where}.failure.kind",
|
|
553
|
+
("error", "timeout", "row_limit"),
|
|
554
|
+
),
|
|
555
|
+
message=self.str_(failure.get("message"), f"{where}.failure.message"),
|
|
556
|
+
)
|
|
557
|
+
rows = self.array(obj.get("rows"), f"{where}.rows")
|
|
558
|
+
return ResultSample(
|
|
559
|
+
columns=tuple(
|
|
560
|
+
self.str_(column, f"{where}.columns")
|
|
561
|
+
for column in self.array(obj.get("columns"), f"{where}.columns")
|
|
562
|
+
),
|
|
563
|
+
rows=tuple(
|
|
564
|
+
tuple(
|
|
565
|
+
self.value(item, f"{where}.rows[{index}]")
|
|
566
|
+
for item in self.array(row, f"{where}.rows[{index}]")
|
|
567
|
+
)
|
|
568
|
+
for index, row in enumerate(rows)
|
|
569
|
+
),
|
|
570
|
+
row_count=self.int_(obj.get("row_count"), f"{where}.row_count"),
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
def value(self, value: object, where: str) -> SqlValue:
|
|
574
|
+
if (
|
|
575
|
+
value is None
|
|
576
|
+
or isinstance(value, str | int | float)
|
|
577
|
+
and not isinstance(value, bool)
|
|
578
|
+
):
|
|
579
|
+
return value
|
|
580
|
+
tagged = self.obj(value, where)
|
|
581
|
+
if "$bytes" in tagged:
|
|
582
|
+
return bytes.fromhex(self.str_(tagged["$bytes"], f"{where}.$bytes"))
|
|
583
|
+
if "$float" in tagged:
|
|
584
|
+
return float(self.str_(tagged["$float"], f"{where}.$float"))
|
|
585
|
+
raise self.fail(where, "a SQL value")
|