text2sql-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,101 @@
1
+ """Deterministic, evidence-backed error analysis for text-to-SQL experiments.
2
+
3
+ The public API is exactly what ``__all__`` lists (ADR 0043). Anything else,
4
+ including the submodules, may change without notice.
5
+
6
+ Example:
7
+ The same steps as ``text2sql-eval eval`` and ``text2sql-eval report``:
8
+
9
+ >>> from pathlib import Path
10
+ >>> import text2sql_eval as t2s
11
+ >>> run = t2s.evaluate_files(
12
+ ... Path("dev.json"), Path("predictions.jsonl"), db_dir=Path("dev_databases")
13
+ ... )
14
+ >>> t2s.write_run(run, Path("runs/v3.json"))
15
+ >>> t2s.write_html_report(run, Path("runs/v3.html"), run_name="v3")
16
+ """
17
+
18
+ from importlib.metadata import version
19
+
20
+ from text2sql_eval.compare.runs import (
21
+ CaseChange,
22
+ Change,
23
+ Comparison,
24
+ GroupEx,
25
+ compare_runs,
26
+ write_comparison_csv,
27
+ )
28
+ from text2sql_eval.errors import InputError
29
+ from text2sql_eval.evaluate import evaluate_files
30
+ from text2sql_eval.execution.executor import QueryFailure, SqlValue
31
+ from text2sql_eval.facts import CaseDiff
32
+ from text2sql_eval.facts.output import OutputDiff, OutputDifference, Relation
33
+ from text2sql_eval.facts.structure import Fact, FactKind, StructureReport
34
+ from text2sql_eval.judge.ex import (
35
+ Correct,
36
+ ExecError,
37
+ Flag,
38
+ FlagKind,
39
+ Incorrect,
40
+ Verdict,
41
+ status_of,
42
+ )
43
+ from text2sql_eval.report.csv_report import write_report_csv
44
+ from text2sql_eval.report.html_report import write_html_report
45
+ from text2sql_eval.runfile import (
46
+ CaseRecord,
47
+ FileRef,
48
+ PredictionFormat,
49
+ PredictionState,
50
+ QuerySummary,
51
+ ResultSample,
52
+ Run,
53
+ RunSettings,
54
+ Summary,
55
+ read_run,
56
+ write_run,
57
+ )
58
+
59
+ __version__ = version("text2sql-eval")
60
+
61
+ # Sorted as Ruff requires; docs/reference/api.md groups them by purpose.
62
+ __all__ = [
63
+ "CaseChange",
64
+ "CaseDiff",
65
+ "CaseRecord",
66
+ "Change",
67
+ "Comparison",
68
+ "Correct",
69
+ "ExecError",
70
+ "Fact",
71
+ "FactKind",
72
+ "FileRef",
73
+ "Flag",
74
+ "FlagKind",
75
+ "GroupEx",
76
+ "Incorrect",
77
+ "InputError",
78
+ "OutputDiff",
79
+ "OutputDifference",
80
+ "PredictionFormat",
81
+ "PredictionState",
82
+ "QueryFailure",
83
+ "QuerySummary",
84
+ "Relation",
85
+ "ResultSample",
86
+ "Run",
87
+ "RunSettings",
88
+ "SqlValue",
89
+ "StructureReport",
90
+ "Summary",
91
+ "Verdict",
92
+ "__version__",
93
+ "compare_runs",
94
+ "evaluate_files",
95
+ "read_run",
96
+ "status_of",
97
+ "write_comparison_csv",
98
+ "write_html_report",
99
+ "write_report_csv",
100
+ "write_run",
101
+ ]
@@ -0,0 +1 @@
1
+ """Command-line interface. The entry point is ``text2sql_eval.cli.main:main``."""
@@ -0,0 +1,482 @@
1
+ """The ``text2sql-eval`` command (ADR 0025).
2
+
3
+ Exit codes: 0 success, 1 a check failed (``compare --max-drop``),
4
+ 2 usage or input error, 3 environment error, 130 interrupted.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import os
12
+ import sys
13
+ from collections import Counter
14
+ from collections.abc import Sequence
15
+ from pathlib import Path
16
+ from typing import Literal, TextIO
17
+
18
+ from text2sql_eval import __version__
19
+ from text2sql_eval.compare.runs import (
20
+ Comparison,
21
+ compare_runs,
22
+ write_comparison_csv,
23
+ )
24
+ from text2sql_eval.errors import InputError
25
+ from text2sql_eval.evaluate import evaluate_files
26
+ from text2sql_eval.execution.executor import DEFAULT_TIMEOUT_SECONDS
27
+ from text2sql_eval.report import csv_report, html_report
28
+ from text2sql_eval.runfile import PredictionFormat, Run, read_run, write_run
29
+
30
+ EXIT_OK = 0
31
+ EXIT_FAILED_CHECK = 1
32
+ EXIT_USAGE = 2
33
+ EXIT_ENVIRONMENT = 3
34
+ EXIT_INTERRUPTED = 130
35
+
36
+ _EVAL_EXAMPLES = """\
37
+ examples:
38
+ # With the example data in this repository:
39
+ text2sql-eval eval --dataset examples/bird-mini/dev.json \\
40
+ --db-dir examples/bird-mini/dev_databases \\
41
+ --pred examples/results/gpt-4o-mini-evidence.jsonl \\
42
+ --tied examples/bird-mini/dev_tied_append.json --out runs/evidence.json
43
+
44
+ # Your own predictions, one SQL per line in dataset order:
45
+ text2sql-eval eval --dataset dev.json --db-dir dev_databases \\
46
+ --pred predictions.txt --out runs/baseline.json
47
+
48
+ # JSONL predictions, recording facts about the run:
49
+ text2sql-eval eval --dataset dev.json --db-dir dev_databases \\
50
+ --pred predictions.jsonl --meta model=my-model --meta prompt=v3 \\
51
+ --out runs/v3.json
52
+ """
53
+
54
+
55
+ _REPORT_EXAMPLES = """\
56
+ examples:
57
+ # A committed run in this repository:
58
+ text2sql-eval report examples/results/gpt-4o-mini-evidence.run.json \\
59
+ --html runs/evidence.html --csv runs/evidence.csv
60
+
61
+ text2sql-eval report runs/v3.json --html runs/v3.html
62
+ """
63
+
64
+
65
+ _COMPARE_EXAMPLES = """\
66
+ examples:
67
+ # The two committed runs in this repository:
68
+ text2sql-eval compare examples/results/gpt-4o-mini-no-evidence.run.json \\
69
+ examples/results/gpt-4o-mini-evidence.run.json
70
+
71
+ # Fail (exit code 1) if EX drops by more than 1 percentage point:
72
+ text2sql-eval compare runs/main.json runs/branch.json --max-drop 1.0 \\
73
+ --csv runs/flips.csv
74
+ """
75
+
76
+
77
+ def main(argv: Sequence[str] | None = None) -> int:
78
+ """Run the command line. Returns the process exit code."""
79
+ parser = _build_parser()
80
+ args = parser.parse_args(argv) # exits with code 2 on usage errors
81
+ if args.command == "report" and args.html is None and args.csv is None:
82
+ parser.error("report: nothing to write; pass --html PATH, --csv PATH or both")
83
+ try:
84
+ if args.command == "eval":
85
+ return _run_eval(args, sys.stdout)
86
+ if args.command == "report":
87
+ return _run_report(args, sys.stdout)
88
+ if args.command == "compare":
89
+ return _run_compare(args, sys.stdout)
90
+ parser.error(f"unknown command {args.command!r}")
91
+ except InputError as error:
92
+ print(f"error: {error}", file=sys.stderr)
93
+ return EXIT_USAGE
94
+ except BrokenPipeError:
95
+ # The reader went away (e.g. `| head`); that is not our error.
96
+ _silence_stdout()
97
+ return EXIT_OK
98
+ except OSError as error:
99
+ print(f"error: {error}", file=sys.stderr)
100
+ return EXIT_ENVIRONMENT
101
+ except KeyboardInterrupt:
102
+ print("interrupted", file=sys.stderr)
103
+ return EXIT_INTERRUPTED
104
+
105
+
106
+ def _build_parser() -> argparse.ArgumentParser:
107
+ parser = argparse.ArgumentParser(
108
+ prog="text2sql-eval",
109
+ description="Judge text-to-SQL predictions and explain what went wrong.",
110
+ )
111
+ parser.add_argument(
112
+ "--version", action="version", version=f"%(prog)s {__version__}"
113
+ )
114
+ commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
115
+
116
+ run = commands.add_parser(
117
+ "eval",
118
+ help="execute and judge predictions, and write a run file",
119
+ description="Execute gold and predicted SQL, judge EX like BIRD's "
120
+ "official script, and write the results to a JSON run file.",
121
+ epilog=_EVAL_EXAMPLES,
122
+ formatter_class=argparse.RawDescriptionHelpFormatter,
123
+ )
124
+ run.add_argument("--dataset", type=Path, required=True, help="BIRD question file")
125
+ run.add_argument(
126
+ "--db-dir",
127
+ type=Path,
128
+ required=True,
129
+ help="folder holding <db_id>/<db_id>.sqlite",
130
+ )
131
+ run.add_argument(
132
+ "--pred",
133
+ type=Path,
134
+ required=True,
135
+ help="predictions: .txt (one SQL per line) or .jsonl",
136
+ )
137
+ run.add_argument(
138
+ "--pred-format",
139
+ choices=("text", "jsonl"),
140
+ help="override the format implied by the --pred file extension",
141
+ )
142
+ run.add_argument(
143
+ "--allow-missing",
144
+ action="store_true",
145
+ help="JSONL only: judge cases without a line as empty predictions",
146
+ )
147
+ run.add_argument(
148
+ "--tied",
149
+ type=Path,
150
+ help="BIRD tied-rows alternative gold file, used for flags only",
151
+ )
152
+ run.add_argument("--out", type=Path, required=True, help="run file to write")
153
+ run.add_argument(
154
+ "--timeout",
155
+ type=_positive_float,
156
+ default=DEFAULT_TIMEOUT_SECONDS,
157
+ help="seconds allowed per query (default: %(default)g)",
158
+ )
159
+ run.add_argument(
160
+ "--max-rows",
161
+ type=_positive_int,
162
+ help="fail queries returning more rows than this (default: no limit)",
163
+ )
164
+ run.add_argument(
165
+ "--workers", type=_positive_int, help="parallel queries (default: min(8, CPUs))"
166
+ )
167
+ run.add_argument(
168
+ "--meta",
169
+ action="append",
170
+ default=[],
171
+ type=_key_value,
172
+ metavar="KEY=VALUE",
173
+ help="record a fact about this run, such as model=... (repeatable)",
174
+ )
175
+ run.add_argument(
176
+ "--json", action="store_true", help="print the summary as JSON on stdout"
177
+ )
178
+
179
+ report = commands.add_parser(
180
+ "report",
181
+ help="write an HTML and/or CSV report of a run file",
182
+ description="Write a run file as a single-file HTML report that opens "
183
+ "offline, and/or as a CSV file for spreadsheet analysis.",
184
+ epilog=_REPORT_EXAMPLES,
185
+ formatter_class=argparse.RawDescriptionHelpFormatter,
186
+ )
187
+ report.add_argument("run", type=Path, help="run file written by eval")
188
+ report.add_argument("--html", type=Path, help="write the HTML report to this file")
189
+ report.add_argument(
190
+ "--csv",
191
+ type=Path,
192
+ help="write one row per case to this CSV file (UTF-8 with BOM for Excel)",
193
+ )
194
+
195
+ compare = commands.add_parser(
196
+ "compare",
197
+ help="compare two run files case by case",
198
+ description="Compare two run files over the cases they share: which "
199
+ "cases both got right, which only one got right, and how the kinds of "
200
+ "differences changed.",
201
+ epilog=_COMPARE_EXAMPLES,
202
+ formatter_class=argparse.RawDescriptionHelpFormatter,
203
+ )
204
+ compare.add_argument("base", type=Path, help="run file to compare from")
205
+ compare.add_argument("head", type=Path, help="run file to compare to")
206
+ compare.add_argument(
207
+ "--max-drop",
208
+ type=_non_negative_float,
209
+ metavar="PP",
210
+ help="exit with code 1 if EX drops by more than PP percentage points "
211
+ "(default: never fail)",
212
+ )
213
+ compare.add_argument(
214
+ "--csv", type=Path, help="write one row per common case to this CSV file"
215
+ )
216
+ compare.add_argument(
217
+ "--json", action="store_true", help="print the summary as JSON on stdout"
218
+ )
219
+ return parser
220
+
221
+
222
+ def _run_eval(args: argparse.Namespace, out: TextIO) -> int:
223
+ out_path: Path = args.out
224
+ pred_format: PredictionFormat | None = args.pred_format
225
+ run = evaluate_files(
226
+ args.dataset,
227
+ args.pred,
228
+ db_dir=args.db_dir,
229
+ prediction_format=pred_format,
230
+ allow_missing=args.allow_missing,
231
+ tied=args.tied,
232
+ timeout=args.timeout,
233
+ max_rows=args.max_rows,
234
+ workers=args.workers,
235
+ meta=dict(args.meta),
236
+ )
237
+ write_run(run, out_path)
238
+ _print_summary(run, out_path, out, as_json=args.json)
239
+ return EXIT_OK
240
+
241
+
242
+ def _run_report(args: argparse.Namespace, out: TextIO) -> int:
243
+ run_path: Path = args.run
244
+ html_path: Path | None = args.html
245
+ csv_path: Path | None = args.csv
246
+ run = read_run(run_path)
247
+ if html_path is not None:
248
+ html_report.write_html_report(run, html_path, run_name=run_path.name)
249
+ if csv_path is not None:
250
+ csv_report.write_report_csv(run, csv_path)
251
+ summary = run.summary
252
+ print(f"Report of {run_path}: {summary.total} cases, EX {summary.ex:.1%}", file=out)
253
+ if html_path is not None:
254
+ print(f"HTML: {html_path}", file=out)
255
+ if csv_path is not None:
256
+ print(f"CSV: {csv_path}", file=out)
257
+ return EXIT_OK
258
+
259
+
260
+ def _run_compare(args: argparse.Namespace, out: TextIO) -> int:
261
+ base_path: Path = args.base
262
+ head_path: Path = args.head
263
+ comparison = compare_runs(read_run(base_path), read_run(head_path))
264
+ if not comparison.cases:
265
+ raise InputError(
266
+ head_path,
267
+ f"shares no case ids with {base_path}",
268
+ hint="compare runs made on the same dataset",
269
+ )
270
+ csv_path: Path | None = args.csv
271
+ if csv_path is not None:
272
+ write_comparison_csv(comparison, csv_path)
273
+
274
+ max_drop: float | None = args.max_drop
275
+ failed = max_drop is not None and comparison.delta_pp < -max_drop
276
+ try:
277
+ if args.json:
278
+ _print_comparison_json(comparison, out, failed=failed)
279
+ else:
280
+ _print_comparison(comparison, base_path, head_path, out)
281
+ if csv_path is not None:
282
+ print(f"CSV: {csv_path}", file=out)
283
+ except BrokenPipeError:
284
+ # Output cut short (e.g. `| head`) must not hide a failed --max-drop.
285
+ _silence_stdout()
286
+ if failed:
287
+ print(
288
+ f"FAILED: EX dropped by {-comparison.delta_pp:.1f} pp, "
289
+ f"more than --max-drop {max_drop:g}",
290
+ file=sys.stderr,
291
+ )
292
+ return EXIT_FAILED_CHECK
293
+ return EXIT_OK
294
+
295
+
296
+ def _print_comparison(
297
+ comparison: Comparison, base_path: Path, head_path: Path, out: TextIO
298
+ ) -> None:
299
+ if comparison.dataset_mismatch:
300
+ print(
301
+ "WARNING: the runs used different dataset files (SHA-256 differs); "
302
+ "comparing only the cases both have",
303
+ file=out,
304
+ )
305
+ for label, ids in (
306
+ ("base", comparison.only_in_base),
307
+ ("head", comparison.only_in_head),
308
+ ):
309
+ if ids:
310
+ print(f"WARNING: {len(ids)} cases only in {label}, left out", file=out)
311
+ counts = comparison.counts()
312
+ print(
313
+ f"Comparing {base_path} -> {head_path} ({len(comparison.cases)} common cases)",
314
+ file=out,
315
+ )
316
+ print(
317
+ f"EX: {comparison.base_ex:.1%} -> {comparison.head_ex:.1%} "
318
+ f"({comparison.delta_pp:+.1f} pp)",
319
+ file=out,
320
+ )
321
+ print(
322
+ f"Both correct {counts['both_correct']} | Fixed {counts['fixed']} | "
323
+ f"Regressed {counts['regressed']} | Both wrong {counts['both_wrong']}",
324
+ file=out,
325
+ )
326
+ groupings: tuple[tuple[Literal["difficulty", "db_id"], str], ...] = (
327
+ ("difficulty", "By difficulty"),
328
+ ("db_id", "By database"),
329
+ )
330
+ for key, title in groupings:
331
+ listing = ", ".join(
332
+ f"{g.name} {g.base_ex:.1%} -> {g.head_ex:.1%}" for g in comparison.by(key)
333
+ )
334
+ print(f"{title}: {listing}", file=out)
335
+ for title, (base, head) in (
336
+ ("Output difference", comparison.output_differences()),
337
+ ("SQL structure difference", comparison.structure_differences()),
338
+ ):
339
+ kinds = sorted(set(base) | set(head), key=lambda k: (-base[k], k))
340
+ if kinds:
341
+ listing = ", ".join(f"{k} {base[k]} -> {head[k]}" for k in kinds)
342
+ print(f"{title}: {listing}", file=out)
343
+
344
+
345
+ def _print_comparison_json(
346
+ comparison: Comparison, out: TextIO, *, failed: bool
347
+ ) -> None:
348
+ counts = comparison.counts()
349
+ output_base, output_head = comparison.output_differences()
350
+ structure_base, structure_head = comparison.structure_differences()
351
+ document = {
352
+ "common_cases": len(comparison.cases),
353
+ "dataset_mismatch": comparison.dataset_mismatch,
354
+ "only_in_base": list(comparison.only_in_base),
355
+ "only_in_head": list(comparison.only_in_head),
356
+ "base_ex": comparison.base_ex,
357
+ "head_ex": comparison.head_ex,
358
+ "delta_pp": comparison.delta_pp,
359
+ "both_correct": counts["both_correct"],
360
+ "fixed": counts["fixed"],
361
+ "regressed": counts["regressed"],
362
+ "both_wrong": counts["both_wrong"],
363
+ "fixed_ids": [c.base.id for c in comparison.cases if c.change == "fixed"],
364
+ "regressed_ids": [
365
+ c.base.id for c in comparison.cases if c.change == "regressed"
366
+ ],
367
+ "output_differences": {
368
+ "base": dict(sorted(output_base.items())),
369
+ "head": dict(sorted(output_head.items())),
370
+ },
371
+ "sql_structure_differences": {
372
+ "base": dict(sorted(structure_base.items())),
373
+ "head": dict(sorted(structure_head.items())),
374
+ },
375
+ "failed_max_drop": failed,
376
+ }
377
+ print(json.dumps(document), file=out)
378
+
379
+
380
+ def _silence_stdout() -> None:
381
+ """Point stdout at devnull so the interpreter's final flush does not raise."""
382
+ os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
383
+
384
+
385
+ def _print_summary(run: Run, out_path: Path, out: TextIO, *, as_json: bool) -> None:
386
+ summary = run.summary
387
+ flags = Counter(flag.kind for case in run.cases for flag in case.verdict.flags)
388
+ states = Counter(case.prediction_state for case in run.cases)
389
+ outputs = Counter(
390
+ difference
391
+ for case in run.cases
392
+ if case.diff is not None
393
+ for difference in case.diff.output.differences
394
+ )
395
+ facts = Counter(
396
+ fact.kind
397
+ for case in run.cases
398
+ if case.diff is not None
399
+ for fact in case.diff.structure.facts
400
+ )
401
+ if as_json:
402
+ document = {
403
+ "run_file": str(out_path),
404
+ "total": summary.total,
405
+ "correct": summary.correct,
406
+ "incorrect": summary.incorrect,
407
+ "error": summary.error,
408
+ "ex": summary.ex,
409
+ "missing_predictions": states["missing"],
410
+ "empty_predictions": states["empty"],
411
+ "flags": dict(sorted(flags.items())),
412
+ "output_differences": dict(sorted(outputs.items())),
413
+ "sql_structure_differences": dict(sorted(facts.items())),
414
+ }
415
+ print(json.dumps(document), file=out)
416
+ return
417
+ print(
418
+ f"Evaluated {summary.total} cases: EX {summary.ex:.1%} "
419
+ f"({summary.correct} correct, {summary.incorrect} incorrect, "
420
+ f"{summary.error} {'error' if summary.error == 1 else 'errors'})",
421
+ file=out,
422
+ )
423
+ if states["missing"] or states["empty"]:
424
+ # Both are judged as empty SQL (ADR 0023); say so, or the incorrect
425
+ # count looks like model mistakes.
426
+ print(
427
+ f"Predictions: {states['missing']} missing, {states['empty']} empty "
428
+ "(judged as empty SQL)",
429
+ file=out,
430
+ )
431
+ # Counts are per kind; one wrong case can have several of each.
432
+ if outputs:
433
+ listing = ", ".join(f"{count} {kind}" for kind, count in outputs.most_common())
434
+ print(f"Output difference: {listing}", file=out)
435
+ if facts:
436
+ listing = ", ".join(f"{count} {kind}" for kind, count in facts.most_common())
437
+ print(f"SQL structure difference: {listing}", file=out)
438
+ if flags:
439
+ listing = ", ".join(f"{count} {kind}" for kind, count in sorted(flags.items()))
440
+ print(f"Flags: {listing}", file=out)
441
+ print(f"Run file: {out_path}", file=out)
442
+
443
+
444
+ def _positive_float(text: str) -> float:
445
+ try:
446
+ value = float(text)
447
+ except ValueError:
448
+ raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
449
+ if not value > 0:
450
+ raise argparse.ArgumentTypeError(f"must be greater than 0, got {text}")
451
+ return value
452
+
453
+
454
+ def _non_negative_float(text: str) -> float:
455
+ try:
456
+ value = float(text)
457
+ except ValueError:
458
+ raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
459
+ if not value >= 0:
460
+ raise argparse.ArgumentTypeError(f"must be 0 or more, got {text}")
461
+ return value
462
+
463
+
464
+ def _positive_int(text: str) -> int:
465
+ try:
466
+ value = int(text)
467
+ except ValueError:
468
+ raise argparse.ArgumentTypeError(f"not an integer: {text!r}") from None
469
+ if value < 1:
470
+ raise argparse.ArgumentTypeError(f"must be at least 1, got {text}")
471
+ return value
472
+
473
+
474
+ def _key_value(text: str) -> tuple[str, str]:
475
+ key, sep, value = text.partition("=")
476
+ if not sep or not key:
477
+ raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got {text!r}")
478
+ return key, value
479
+
480
+
481
+ if __name__ == "__main__":
482
+ sys.exit(main())
@@ -0,0 +1 @@
1
+ """Case-by-case comparison of two runs. Not part of the public API yet."""