text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,585 @@
1
+ """The run file: one JSON document per evaluation run (ADR 0026).
2
+
3
+ ``report`` and ``compare`` read it, and people open it directly, so the format
4
+ is documented in ``docs/reference/run-file.md`` and versioned by
5
+ ``SCHEMA_VERSION``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ import math
13
+ from collections import Counter
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from typing import Literal, TypeVar, get_args
17
+
18
+ from text2sql_eval.datasets._json import (
19
+ as_array,
20
+ as_object,
21
+ json_type_name,
22
+ parse_json,
23
+ read_utf8,
24
+ )
25
+ from text2sql_eval.errors import InputError
26
+ from text2sql_eval.execution.executor import QueryFailure, Rows, SqlValue
27
+ from text2sql_eval.facts import CaseDiff
28
+ from text2sql_eval.facts.output import OutputDiff, OutputDifference, Relation
29
+ from text2sql_eval.facts.structure import Fact, FactKind, StructureReport
30
+ from text2sql_eval.judge.ex import (
31
+ Correct,
32
+ ExecError,
33
+ Flag,
34
+ FlagKind,
35
+ Incorrect,
36
+ Verdict,
37
+ status_of,
38
+ )
39
+
40
+ SCHEMA_VERSION = 3
41
+ SAMPLE_ROWS = 20
42
+ """Rows kept per query result; ``row_count`` still says how many there were."""
43
+
44
+ # A bound TypeVar instead of 3.12's ``def f[T: str]`` (ADR 0009).
45
+ _Choice = TypeVar("_Choice", bound=str)
46
+
47
+ Side = Literal["gold", "pred"]
48
+
49
+ PredictionState = Literal["given", "empty", "missing"]
50
+ PredictionFormat = Literal["text", "jsonl"]
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class FileRef:
55
+ """An input file as it was when the run started."""
56
+
57
+ path: str
58
+ sha256: str
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class ResultSample:
63
+ """The first ``SAMPLE_ROWS`` rows of a query result, and the full count."""
64
+
65
+ columns: tuple[str, ...]
66
+ rows: tuple[tuple[SqlValue, ...], ...]
67
+ row_count: int
68
+
69
+
70
+ QuerySummary = ResultSample | QueryFailure
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class CaseRecord:
75
+ """Everything needed to show and compare one judged case."""
76
+
77
+ id: str
78
+ db_id: str
79
+ difficulty: str | None
80
+ question: str
81
+ evidence: str
82
+ gold_sql: str
83
+ pred_sql: str
84
+ prediction_state: PredictionState
85
+ verdict: Verdict
86
+ gold: QuerySummary
87
+ pred: QuerySummary
88
+ pred_meta: dict[str, object] = field(default_factory=dict[str, object])
89
+ diff: CaseDiff | None = None
90
+ """Why the case is not correct; ``None`` for correct cases (ADR 0031)."""
91
+
92
+
93
+ @dataclass(frozen=True)
94
+ class RunSettings:
95
+ """How the run executed queries, so a later run can tell if they differ."""
96
+
97
+ timeout_seconds: float
98
+ max_rows: int | None
99
+ allow_missing: bool
100
+
101
+
102
+ @dataclass(frozen=True)
103
+ class Summary:
104
+ """Number of cases per status; ``total`` is their sum."""
105
+
106
+ total: int
107
+ correct: int
108
+ incorrect: int
109
+ error: int
110
+
111
+ @property
112
+ def ex(self) -> float:
113
+ """Execution accuracy: correct cases over all cases (0.0 for no cases)."""
114
+ return self.correct / self.total if self.total else 0.0
115
+
116
+
117
+ @dataclass(frozen=True)
118
+ class Run:
119
+ """One evaluation run: inputs, settings, user metadata and judged cases."""
120
+
121
+ tool_version: str
122
+ created_at: str
123
+ dataset: FileRef
124
+ predictions: FileRef
125
+ prediction_format: PredictionFormat
126
+ tied: FileRef | None
127
+ db_dir: str
128
+ settings: RunSettings
129
+ meta: dict[str, str]
130
+ cases: tuple[CaseRecord, ...]
131
+
132
+ @property
133
+ def summary(self) -> Summary:
134
+ counts = Counter(status_of(case.verdict) for case in self.cases)
135
+ return Summary(
136
+ total=len(self.cases),
137
+ correct=counts["correct"],
138
+ incorrect=counts["incorrect"],
139
+ error=counts["error"],
140
+ )
141
+
142
+
143
+ def sha256_of(path: Path) -> str:
144
+ """Hex SHA-256 of a file, so a later run can tell whether inputs changed."""
145
+ digest = hashlib.sha256()
146
+ with path.open("rb") as file:
147
+ while chunk := file.read(1 << 20):
148
+ digest.update(chunk)
149
+ return digest.hexdigest()
150
+
151
+
152
+ def sample_of(result: Rows | QueryFailure) -> QuerySummary:
153
+ """Keep the first ``SAMPLE_ROWS`` rows of a result for the run file."""
154
+ if isinstance(result, QueryFailure):
155
+ return result
156
+ return ResultSample(
157
+ columns=result.columns,
158
+ rows=result.rows[:SAMPLE_ROWS],
159
+ row_count=len(result.rows),
160
+ )
161
+
162
+
163
+ # --- writing ---------------------------------------------------------------
164
+
165
+
166
+ def write_run(run: Run, path: Path) -> None:
167
+ """Write ``run`` as UTF-8 JSON, creating parent directories as needed.
168
+
169
+ Example:
170
+ >>> write_run(run, Path("runs/v3.json"))
171
+ >>> read_run(Path("runs/v3.json")) == run
172
+ True
173
+ """
174
+ summary = run.summary
175
+ document: dict[str, object] = {
176
+ "schema_version": SCHEMA_VERSION,
177
+ "tool": {"name": "text2sql-eval", "version": run.tool_version},
178
+ "created_at": run.created_at,
179
+ "inputs": {
180
+ "dataset": _file_ref(run.dataset),
181
+ "predictions": {
182
+ **_file_ref(run.predictions),
183
+ "format": run.prediction_format,
184
+ },
185
+ "tied": None if run.tied is None else _file_ref(run.tied),
186
+ "db_dir": run.db_dir,
187
+ },
188
+ "settings": {
189
+ "timeout_seconds": run.settings.timeout_seconds,
190
+ "max_rows": run.settings.max_rows,
191
+ "allow_missing": run.settings.allow_missing,
192
+ },
193
+ "meta": run.meta,
194
+ "summary": {
195
+ "total": summary.total,
196
+ "correct": summary.correct,
197
+ "incorrect": summary.incorrect,
198
+ "error": summary.error,
199
+ "ex": summary.ex,
200
+ },
201
+ "cases": [_case_to_json(case) for case in run.cases],
202
+ }
203
+ path.parent.mkdir(parents=True, exist_ok=True)
204
+ # allow_nan=False: NaN and Infinity are not JSON; _encode_value spells them out.
205
+ text = json.dumps(document, ensure_ascii=False, indent=2, allow_nan=False)
206
+ path.write_text(text + "\n", encoding="utf-8")
207
+
208
+
209
+ def _file_ref(ref: FileRef) -> dict[str, object]:
210
+ return {"path": ref.path, "sha256": ref.sha256}
211
+
212
+
213
+ def _case_to_json(case: CaseRecord) -> dict[str, object]:
214
+ verdict = case.verdict
215
+ error = (
216
+ {"side": verdict.side, "kind": verdict.kind, "message": verdict.message}
217
+ if isinstance(verdict, ExecError)
218
+ else None
219
+ )
220
+ return {
221
+ "id": case.id,
222
+ "db_id": case.db_id,
223
+ "difficulty": case.difficulty,
224
+ "question": case.question,
225
+ "evidence": case.evidence,
226
+ "gold_sql": case.gold_sql,
227
+ "pred_sql": case.pred_sql,
228
+ "prediction_state": case.prediction_state,
229
+ "pred_meta": case.pred_meta,
230
+ "status": status_of(verdict),
231
+ "error": error,
232
+ "flags": [{"kind": flag.kind, "detail": flag.detail} for flag in verdict.flags],
233
+ "gold": _summary_to_json(case.gold),
234
+ "pred": _summary_to_json(case.pred),
235
+ "diff": None if case.diff is None else _diff_to_json(case.diff),
236
+ }
237
+
238
+
239
+ def _diff_to_json(diff: CaseDiff) -> dict[str, object]:
240
+ output, structure = diff.output, diff.structure
241
+ return {
242
+ "output": {
243
+ "differences": list(output.differences),
244
+ "gold_columns": output.gold_columns,
245
+ "pred_columns": output.pred_columns,
246
+ "gold_distinct_rows": output.gold_distinct_rows,
247
+ "pred_distinct_rows": output.pred_distinct_rows,
248
+ "gold_contained": output.gold_contained,
249
+ "relation": output.relation,
250
+ },
251
+ "sql_structure": {
252
+ "differences": [
253
+ {"kind": fact.kind, "gold": fact.gold, "pred": fact.pred}
254
+ for fact in structure.facts
255
+ ],
256
+ "parse_failed": structure.parse_failed,
257
+ },
258
+ }
259
+
260
+
261
+ def _summary_to_json(summary: QuerySummary) -> dict[str, object]:
262
+ if isinstance(summary, QueryFailure):
263
+ return {"failure": {"kind": summary.kind, "message": summary.message}}
264
+ return {
265
+ "columns": list(summary.columns),
266
+ "rows": [[_encode_value(value) for value in row] for row in summary.rows],
267
+ "row_count": summary.row_count,
268
+ }
269
+
270
+
271
+ def _encode_value(value: SqlValue) -> object:
272
+ """Map a SQLite value to JSON. Bytes and non-finite floats get tagged objects."""
273
+ if isinstance(value, bytes):
274
+ return {"$bytes": value.hex()}
275
+ if isinstance(value, float) and not math.isfinite(value):
276
+ return {"$float": repr(value)} # 'inf', '-inf' or 'nan'
277
+ return value
278
+
279
+
280
+ # --- reading ---------------------------------------------------------------
281
+
282
+
283
+ def read_run(path: Path) -> Run:
284
+ """Read a run file written by ``write_run``.
285
+
286
+ Raises:
287
+ InputError: if the file is unreadable, has another ``schema_version``,
288
+ or is missing fields or has fields of the wrong type.
289
+
290
+ Example:
291
+ >>> run = read_run(Path("examples/results/gpt-4o-mini-evidence.run.json"))
292
+ >>> run.summary.correct, run.summary.total
293
+ (267, 432)
294
+ """
295
+ reader = _Reader(path)
296
+ document = parse_json(read_utf8(path), path)
297
+ root = as_object(document)
298
+ if root is None or root.get("schema_version") is None:
299
+ found = (
300
+ "it has no schema_version"
301
+ if root is not None
302
+ else f"it is a JSON {json_type_name(document)}"
303
+ )
304
+ raise InputError(
305
+ path,
306
+ f"not a run file: {found}",
307
+ hint="pass the file that `text2sql-eval eval --out` wrote",
308
+ )
309
+ version = root.get("schema_version")
310
+ if version != SCHEMA_VERSION:
311
+ newer = isinstance(version, int) and version > SCHEMA_VERSION
312
+ raise InputError(
313
+ path,
314
+ f"schema_version is {version!r}, but this tool reads {SCHEMA_VERSION}",
315
+ hint="upgrade text2sql-eval to read it"
316
+ if newer
317
+ else "re-run eval with this version of text2sql-eval",
318
+ )
319
+ tool = reader.obj(root.get("tool"), "tool")
320
+ inputs = reader.obj(root.get("inputs"), "inputs")
321
+ predictions = reader.obj(inputs.get("predictions"), "inputs.predictions")
322
+ prediction_format = reader.choice(
323
+ predictions.get("format"),
324
+ "inputs.predictions.format",
325
+ get_args(PredictionFormat),
326
+ )
327
+ tied_value = inputs.get("tied")
328
+ settings = reader.obj(root.get("settings"), "settings")
329
+ meta = reader.obj(root.get("meta"), "meta")
330
+ cases = reader.array(root.get("cases"), "cases")
331
+ return Run(
332
+ tool_version=reader.str_(tool.get("version"), "tool.version"),
333
+ created_at=reader.str_(root.get("created_at"), "created_at"),
334
+ dataset=reader.file_ref(inputs.get("dataset"), "inputs.dataset"),
335
+ predictions=reader.file_ref(predictions, "inputs.predictions"),
336
+ prediction_format=prediction_format,
337
+ tied=None if tied_value is None else reader.file_ref(tied_value, "inputs.tied"),
338
+ db_dir=reader.str_(inputs.get("db_dir"), "inputs.db_dir"),
339
+ settings=RunSettings(
340
+ timeout_seconds=reader.number(
341
+ settings.get("timeout_seconds"), "settings.timeout_seconds"
342
+ ),
343
+ max_rows=reader.optional_int(settings.get("max_rows"), "settings.max_rows"),
344
+ allow_missing=reader.bool_(
345
+ settings.get("allow_missing"), "settings.allow_missing"
346
+ ),
347
+ ),
348
+ meta={key: reader.str_(value, f"meta.{key}") for key, value in meta.items()},
349
+ cases=tuple(
350
+ reader.case(value, f"cases[{index}]") for index, value in enumerate(cases)
351
+ ),
352
+ )
353
+
354
+
355
+ class _Reader:
356
+ """Typed field access that reports the JSON path of anything malformed."""
357
+
358
+ def __init__(self, path: Path) -> None:
359
+ self.path = path
360
+
361
+ def fail(self, where: str, expected: str) -> InputError:
362
+ return InputError(
363
+ self.path,
364
+ f"{where} must be {expected}",
365
+ hint="the file may be damaged or not written by text2sql-eval",
366
+ )
367
+
368
+ def obj(self, value: object, where: str) -> dict[str, object]:
369
+ result = as_object(value)
370
+ if result is None:
371
+ raise self.fail(where, "an object")
372
+ return result
373
+
374
+ def array(self, value: object, where: str) -> list[object]:
375
+ result = as_array(value)
376
+ if result is None:
377
+ raise self.fail(where, "an array")
378
+ return result
379
+
380
+ def str_(self, value: object, where: str) -> str:
381
+ if not isinstance(value, str):
382
+ raise self.fail(where, "a string")
383
+ return value
384
+
385
+ def optional_str(self, value: object, where: str) -> str | None:
386
+ return None if value is None else self.str_(value, where)
387
+
388
+ def int_(self, value: object, where: str) -> int:
389
+ if isinstance(value, bool) or not isinstance(value, int):
390
+ raise self.fail(where, "an integer")
391
+ return value
392
+
393
+ def optional_int(self, value: object, where: str) -> int | None:
394
+ return None if value is None else self.int_(value, where)
395
+
396
+ def number(self, value: object, where: str) -> float:
397
+ if isinstance(value, bool) or not isinstance(value, int | float):
398
+ raise self.fail(where, "a number")
399
+ return float(value)
400
+
401
+ def bool_(self, value: object, where: str) -> bool:
402
+ if not isinstance(value, bool):
403
+ raise self.fail(where, "true or false")
404
+ return value
405
+
406
+ def choice(
407
+ self, value: object, where: str, options: tuple[_Choice, ...]
408
+ ) -> _Choice:
409
+ for option in options:
410
+ if value == option:
411
+ return option
412
+ raise self.fail(where, "one of " + ", ".join(options))
413
+
414
+ def file_ref(self, value: object, where: str) -> FileRef:
415
+ obj = self.obj(value, where)
416
+ return FileRef(
417
+ path=self.str_(obj.get("path"), f"{where}.path"),
418
+ sha256=self.str_(obj.get("sha256"), f"{where}.sha256"),
419
+ )
420
+
421
+ def case(self, value: object, where: str) -> CaseRecord:
422
+ obj = self.obj(value, where)
423
+ flags = tuple(
424
+ self.flag(item, f"{where}.flags[{index}]")
425
+ for index, item in enumerate(self.array(obj.get("flags"), f"{where}.flags"))
426
+ )
427
+ return CaseRecord(
428
+ id=self.str_(obj.get("id"), f"{where}.id"),
429
+ db_id=self.str_(obj.get("db_id"), f"{where}.db_id"),
430
+ difficulty=self.optional_str(obj.get("difficulty"), f"{where}.difficulty"),
431
+ question=self.str_(obj.get("question"), f"{where}.question"),
432
+ evidence=self.str_(obj.get("evidence"), f"{where}.evidence"),
433
+ gold_sql=self.str_(obj.get("gold_sql"), f"{where}.gold_sql"),
434
+ pred_sql=self.str_(obj.get("pred_sql"), f"{where}.pred_sql"),
435
+ prediction_state=self.choice(
436
+ obj.get("prediction_state"),
437
+ f"{where}.prediction_state",
438
+ get_args(PredictionState),
439
+ ),
440
+ verdict=self.verdict(obj, where, flags),
441
+ gold=self.summary(obj.get("gold"), f"{where}.gold"),
442
+ pred=self.summary(obj.get("pred"), f"{where}.pred"),
443
+ pred_meta=self.obj(obj.get("pred_meta"), f"{where}.pred_meta"),
444
+ diff=self.diff(obj.get("diff"), f"{where}.diff"),
445
+ )
446
+
447
+ def diff(self, value: object, where: str) -> CaseDiff | None:
448
+ if value is None:
449
+ return None
450
+ obj = self.obj(value, where)
451
+ out_where, sql_where = f"{where}.output", f"{where}.sql_structure"
452
+ output = self.obj(obj.get("output"), out_where)
453
+ structure = self.obj(obj.get("sql_structure"), sql_where)
454
+ relation: Relation | None = None
455
+ if (relation_value := output.get("relation")) is not None:
456
+ relation = self.choice(
457
+ relation_value, f"{out_where}.relation", get_args(Relation)
458
+ )
459
+ contained: bool | None = None
460
+ if (contained_value := output.get("gold_contained")) is not None:
461
+ contained = self.bool_(contained_value, f"{out_where}.gold_contained")
462
+ parse_failed: Side | None = None
463
+ if (parse_failed_value := structure.get("parse_failed")) is not None:
464
+ parse_failed = self.choice(
465
+ parse_failed_value, f"{sql_where}.parse_failed", get_args(Side)
466
+ )
467
+ differences = self.array(output.get("differences"), f"{out_where}.differences")
468
+ facts = self.array(structure.get("differences"), f"{sql_where}.differences")
469
+ return CaseDiff(
470
+ output=OutputDiff(
471
+ differences=tuple(
472
+ self.choice(
473
+ item,
474
+ f"{out_where}.differences[{index}]",
475
+ get_args(OutputDifference),
476
+ )
477
+ for index, item in enumerate(differences)
478
+ ),
479
+ gold_columns=self.int_(
480
+ output.get("gold_columns"), f"{out_where}.gold_columns"
481
+ ),
482
+ pred_columns=self.int_(
483
+ output.get("pred_columns"), f"{out_where}.pred_columns"
484
+ ),
485
+ gold_distinct_rows=self.int_(
486
+ output.get("gold_distinct_rows"), f"{out_where}.gold_distinct_rows"
487
+ ),
488
+ pred_distinct_rows=self.int_(
489
+ output.get("pred_distinct_rows"), f"{out_where}.pred_distinct_rows"
490
+ ),
491
+ gold_contained=contained,
492
+ relation=relation,
493
+ ),
494
+ structure=StructureReport(
495
+ facts=tuple(
496
+ self.fact(item, f"{sql_where}.differences[{index}]")
497
+ for index, item in enumerate(facts)
498
+ ),
499
+ parse_failed=parse_failed,
500
+ ),
501
+ )
502
+
503
+ def fact(self, value: object, where: str) -> Fact:
504
+ obj = self.obj(value, where)
505
+ kind: FactKind = self.choice(
506
+ obj.get("kind"), f"{where}.kind", get_args(FactKind)
507
+ )
508
+ return Fact(
509
+ kind=kind,
510
+ gold=self.str_(obj.get("gold"), f"{where}.gold"),
511
+ pred=self.str_(obj.get("pred"), f"{where}.pred"),
512
+ )
513
+
514
+ def flag(self, value: object, where: str) -> Flag:
515
+ obj = self.obj(value, where)
516
+ kind: FlagKind = self.choice(
517
+ obj.get("kind"), f"{where}.kind", get_args(FlagKind)
518
+ )
519
+ return Flag(kind=kind, detail=self.str_(obj.get("detail"), f"{where}.detail"))
520
+
521
+ def verdict(
522
+ self, obj: dict[str, object], where: str, flags: tuple[Flag, ...]
523
+ ) -> Verdict:
524
+ status = self.choice(
525
+ obj.get("status"), f"{where}.status", ("correct", "incorrect", "error")
526
+ )
527
+ if status == "correct":
528
+ return Correct(flags=flags)
529
+ if status == "incorrect":
530
+ return Incorrect(flags=flags)
531
+ error = self.obj(obj.get("error"), f"{where}.error")
532
+ return ExecError(
533
+ side=self.choice(
534
+ error.get("side"), f"{where}.error.side", ("pred", "gold")
535
+ ),
536
+ kind=self.choice(
537
+ error.get("kind"),
538
+ f"{where}.error.kind",
539
+ ("error", "timeout", "row_limit"),
540
+ ),
541
+ message=self.str_(error.get("message"), f"{where}.error.message"),
542
+ flags=flags,
543
+ )
544
+
545
+ def summary(self, value: object, where: str) -> QuerySummary:
546
+ obj = self.obj(value, where)
547
+ if "failure" in obj:
548
+ failure = self.obj(obj["failure"], f"{where}.failure")
549
+ return QueryFailure(
550
+ kind=self.choice(
551
+ failure.get("kind"),
552
+ f"{where}.failure.kind",
553
+ ("error", "timeout", "row_limit"),
554
+ ),
555
+ message=self.str_(failure.get("message"), f"{where}.failure.message"),
556
+ )
557
+ rows = self.array(obj.get("rows"), f"{where}.rows")
558
+ return ResultSample(
559
+ columns=tuple(
560
+ self.str_(column, f"{where}.columns")
561
+ for column in self.array(obj.get("columns"), f"{where}.columns")
562
+ ),
563
+ rows=tuple(
564
+ tuple(
565
+ self.value(item, f"{where}.rows[{index}]")
566
+ for item in self.array(row, f"{where}.rows[{index}]")
567
+ )
568
+ for index, row in enumerate(rows)
569
+ ),
570
+ row_count=self.int_(obj.get("row_count"), f"{where}.row_count"),
571
+ )
572
+
573
+ def value(self, value: object, where: str) -> SqlValue:
574
+ if (
575
+ value is None
576
+ or isinstance(value, str | int | float)
577
+ and not isinstance(value, bool)
578
+ ):
579
+ return value
580
+ tagged = self.obj(value, where)
581
+ if "$bytes" in tagged:
582
+ return bytes.fromhex(self.str_(tagged["$bytes"], f"{where}.$bytes"))
583
+ if "$float" in tagged:
584
+ return float(self.str_(tagged["$float"], f"{where}.$float"))
585
+ raise self.fail(where, "a SQL value")