sqlverdict 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlverdict/__init__.py +5 -0
- sqlverdict/_version.py +1 -0
- sqlverdict/check.py +127 -0
- sqlverdict/cli.py +159 -0
- sqlverdict/compare.py +136 -0
- sqlverdict/hints.py +34 -0
- sqlverdict/loader.py +175 -0
- sqlverdict/model.py +55 -0
- sqlverdict/normalize.py +73 -0
- sqlverdict/output/__init__.py +0 -0
- sqlverdict/output/html_out.py +101 -0
- sqlverdict/output/json_out.py +5 -0
- sqlverdict/output/table.py +92 -0
- sqlverdict/score.py +125 -0
- sqlverdict/templates/AGENTS.md +41 -0
- sqlverdict/templates/golden.yaml +62 -0
- sqlverdict/templates/predictions.yaml +23 -0
- sqlverdict-0.1.0.dist-info/METADATA +298 -0
- sqlverdict-0.1.0.dist-info/RECORD +22 -0
- sqlverdict-0.1.0.dist-info/WHEEL +4 -0
- sqlverdict-0.1.0.dist-info/entry_points.txt +2 -0
- sqlverdict-0.1.0.dist-info/licenses/LICENSE +21 -0
sqlverdict/__init__.py
ADDED
sqlverdict/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
sqlverdict/check.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from datetime import date
|
|
4
|
+
|
|
5
|
+
from sqlverdict.hints import _SELECT_STAR
|
|
6
|
+
|
|
7
|
+
LEVELS = ("L1", "L2", "L3", "L4")
|
|
8
|
+
TRAPS = ("dup_join", "unique_id", "biz_term", "empty_ok", "none")
|
|
9
|
+
STALE_DAYS = 30
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Issue:
|
|
14
|
+
severity: str # "error" | "warn"
|
|
15
|
+
id: str | None
|
|
16
|
+
message: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _item_issues(it):
|
|
20
|
+
out = []
|
|
21
|
+
|
|
22
|
+
def err(m):
|
|
23
|
+
out.append(Issue("error", it.id, m))
|
|
24
|
+
|
|
25
|
+
def warn(m):
|
|
26
|
+
out.append(Issue("warn", it.id, m))
|
|
27
|
+
|
|
28
|
+
if it.problem:
|
|
29
|
+
err(f"형식 오류로 채점할 수 없습니다: {it.problem}")
|
|
30
|
+
if not it.question:
|
|
31
|
+
err("question이 없습니다")
|
|
32
|
+
if it.level not in LEVELS:
|
|
33
|
+
err(f"level은 {'/'.join(LEVELS)} 중 하나여야 합니다")
|
|
34
|
+
if it.trap not in TRAPS:
|
|
35
|
+
err(f"trap은 {'/'.join(TRAPS)} 중 하나여야 합니다")
|
|
36
|
+
elif it.trap != "none" and not it.why:
|
|
37
|
+
err("trap이 있으면 why가 필요합니다 (사람이 이것만 읽고 검증)")
|
|
38
|
+
if not it.verified:
|
|
39
|
+
warn("verified: false — 사람이 확인하기 전까지 채점에서 제외됩니다")
|
|
40
|
+
if it.expected and not it.expected.rows and not it.empty_ok:
|
|
41
|
+
warn("정답이 빈 결과입니다. 의도한 것이면 empty_ok: true")
|
|
42
|
+
return out
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_VOLATILE = re.compile(
|
|
46
|
+
r"\b(now|curdate|curtime|sysdate|systimestamp|getdate|random|rand|uuid|newid|"
|
|
47
|
+
r"current_date|current_time|current_timestamp|localtimestamp)\b", re.IGNORECASE)
|
|
48
|
+
|
|
49
|
+
_OPEN_PERIOD = re.compile(
|
|
50
|
+
r"최근|오늘|어제|올해|금년|작년|지금|현재|"
|
|
51
|
+
r"이번\s*(주|달|월|분기|해)|지난\s*(주|달|월|분기|해)|"
|
|
52
|
+
r"\b(recent(ly)?|today|yesterday|current(ly)?|"
|
|
53
|
+
r"(this|last)\s+(week|month|quarter|year))\b", re.IGNORECASE)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _text_issues(it):
|
|
57
|
+
out = []
|
|
58
|
+
if it.gold_sql:
|
|
59
|
+
if _SELECT_STAR.search(it.gold_sql):
|
|
60
|
+
out.append(Issue("warn", it.id, "gold_sql에 SELECT *: 필요한 컬럼만 고르세요"))
|
|
61
|
+
m = _VOLATILE.search(it.gold_sql)
|
|
62
|
+
if m:
|
|
63
|
+
out.append(Issue("warn", it.id,
|
|
64
|
+
f"gold_sql에 시간·난수 함수 {m.group(1)}: 뽑을 때마다 정답이 바뀝니다"))
|
|
65
|
+
if it.question:
|
|
66
|
+
m = _OPEN_PERIOD.search(it.question)
|
|
67
|
+
if m:
|
|
68
|
+
out.append(Issue("warn", it.id,
|
|
69
|
+
f"질문에 열린 기간 표현 '{m.group(0)}': '2018년'처럼 닫힌 과거 기간을 권장합니다 (정답이 금방 낡음)"))
|
|
70
|
+
return out
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _set_issues(golden, today):
|
|
74
|
+
out = []
|
|
75
|
+
seen = set()
|
|
76
|
+
for it in golden.items:
|
|
77
|
+
if it.id in seen:
|
|
78
|
+
out.append(Issue("error", it.id, "id가 중복됩니다"))
|
|
79
|
+
seen.add(it.id)
|
|
80
|
+
if not golden.items:
|
|
81
|
+
return out + [Issue("warn", None, "문항이 없습니다")]
|
|
82
|
+
|
|
83
|
+
if {it.trap for it in golden.items} <= {"none", None}:
|
|
84
|
+
out.append(Issue("warn", None, "함정 문항이 없습니다: 쉬운 문항만으로는 silent_wrong을 잡지 못합니다"))
|
|
85
|
+
levels = {it.level for it in golden.items if it.level in LEVELS}
|
|
86
|
+
if len(levels) == 1:
|
|
87
|
+
out.append(Issue("warn", None, f"난이도가 {levels.pop()} 하나뿐입니다: 점수가 부풀려질 수 있습니다"))
|
|
88
|
+
|
|
89
|
+
if golden.snapshot is None:
|
|
90
|
+
out.append(Issue("warn", None, "snapshot이 없습니다: 정답이 낡았는지 판단할 기준이 없습니다"))
|
|
91
|
+
else:
|
|
92
|
+
try:
|
|
93
|
+
age = (today - date.fromisoformat(golden.snapshot)).days
|
|
94
|
+
except ValueError:
|
|
95
|
+
out.append(Issue("warn", None, "snapshot은 YYYY-MM-DD 형식이어야 합니다"))
|
|
96
|
+
else:
|
|
97
|
+
if age > STALE_DAYS:
|
|
98
|
+
out.append(Issue("warn", None, f"snapshot이 {age}일 지났습니다: 정답 다시 뽑기를 권장합니다"))
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _pred_issues(golden, preds):
|
|
103
|
+
out = []
|
|
104
|
+
gold_ids = {it.id for it in golden.items}
|
|
105
|
+
seen = set()
|
|
106
|
+
for p in preds.items:
|
|
107
|
+
if p.id in seen:
|
|
108
|
+
out.append(Issue("error", p.id, "예측 id가 중복됩니다"))
|
|
109
|
+
seen.add(p.id)
|
|
110
|
+
if p.id not in gold_ids:
|
|
111
|
+
out.append(Issue("error", p.id, "골든셋에 없는 id입니다"))
|
|
112
|
+
if p.problem:
|
|
113
|
+
out.append(Issue("error", p.id, f"예측 형식 오류: {p.problem}"))
|
|
114
|
+
for it in golden.items:
|
|
115
|
+
if it.id not in seen:
|
|
116
|
+
out.append(Issue("warn", it.id, "예측이 없습니다 (missing_prediction)"))
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def check(golden, preds=None, today=None):
|
|
121
|
+
today = today or date.today()
|
|
122
|
+
out = _set_issues(golden, today)
|
|
123
|
+
for it in golden.items:
|
|
124
|
+
out += _item_issues(it) + _text_issues(it)
|
|
125
|
+
if preds is not None:
|
|
126
|
+
out += _pred_issues(golden, preds)
|
|
127
|
+
return out
|
sqlverdict/cli.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
from dataclasses import asdict
|
|
6
|
+
from importlib.resources import files
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from sqlverdict._version import __version__
|
|
10
|
+
from sqlverdict.check import check
|
|
11
|
+
from sqlverdict.compare import DEFAULT_TOLERANCE
|
|
12
|
+
from sqlverdict.loader import InputError, load_golden, load_predictions
|
|
13
|
+
from sqlverdict.output.html_out import render_html
|
|
14
|
+
from sqlverdict.output.json_out import to_json
|
|
15
|
+
from sqlverdict.output.table import render_table, visible
|
|
16
|
+
from sqlverdict.score import STATUSES, VERDICTS, score
|
|
17
|
+
|
|
18
|
+
FAIL_CHOICES = tuple(x for x in VERDICTS + STATUSES if x != "correct")
|
|
19
|
+
TEMPLATES = ("golden.yaml", "predictions.yaml", "AGENTS.md")
|
|
20
|
+
RENDER = {"table": render_table, "json": to_json, "html": render_html}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _non_negative(kind):
|
|
24
|
+
def conv(s):
|
|
25
|
+
v = kind(s)
|
|
26
|
+
if v < 0:
|
|
27
|
+
raise argparse.ArgumentTypeError(f"0 이상이어야 합니다: {s}")
|
|
28
|
+
return v
|
|
29
|
+
return conv
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _parser():
|
|
33
|
+
p = argparse.ArgumentParser(prog="sqlverdict", description="Text-to-SQL 결과 채점기")
|
|
34
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
35
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
36
|
+
|
|
37
|
+
i = sub.add_parser("init", help="예시 golden.yaml·predictions.yaml·AGENTS.md 생성")
|
|
38
|
+
i.add_argument("dir", nargs="?", default="evals", help="만들 폴더 (기본 evals)")
|
|
39
|
+
i.add_argument("--force", action="store_true", help="이미 있는 파일 덮어쓰기")
|
|
40
|
+
|
|
41
|
+
c = sub.add_parser("check", help="골든셋 검사 (+ 예측 id 짝 확인)")
|
|
42
|
+
c.add_argument("golden")
|
|
43
|
+
c.add_argument("predictions", nargs="?")
|
|
44
|
+
c.add_argument("--format", choices=("table", "json"), default="table")
|
|
45
|
+
|
|
46
|
+
s = sub.add_parser("score", help="채점")
|
|
47
|
+
s.add_argument("golden")
|
|
48
|
+
s.add_argument("predictions")
|
|
49
|
+
s.add_argument("--format", choices=("table", "json", "html"), default="table")
|
|
50
|
+
s.add_argument("-o", "--output", help="저장할 파일 (없으면 화면 출력)")
|
|
51
|
+
s.add_argument("--tolerance", type=_non_negative(float), default=DEFAULT_TOLERANCE)
|
|
52
|
+
s.add_argument("--baseline", help="이전 실행의 JSON 결과")
|
|
53
|
+
s.add_argument("--fail-on", action="append", choices=FAIL_CHOICES, default=[])
|
|
54
|
+
s.add_argument("--head", type=_non_negative(int), default=0)
|
|
55
|
+
s.add_argument("--include-expected", action="store_true")
|
|
56
|
+
s.add_argument("--include-unverified", action="store_true")
|
|
57
|
+
return p
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _write(text, path=None):
|
|
61
|
+
if not text.endswith("\n"):
|
|
62
|
+
text += "\n"
|
|
63
|
+
if path:
|
|
64
|
+
with open(path, "w", encoding="utf-8", newline="\n", errors="backslashreplace") as f:
|
|
65
|
+
f.write(text)
|
|
66
|
+
print(f"저장: {path}", file=sys.stderr)
|
|
67
|
+
return
|
|
68
|
+
if not sys.stdout.isatty() and hasattr(sys.stdout, "reconfigure"):
|
|
69
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
|
|
70
|
+
sys.stdout.write(text)
|
|
71
|
+
sys.stdout.flush()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _cmd_check(args):
|
|
75
|
+
golden = load_golden(args.golden)
|
|
76
|
+
preds = load_predictions(args.predictions) if args.predictions else None
|
|
77
|
+
issues = sorted(check(golden, preds), key=lambda i: i.severity != "error")
|
|
78
|
+
errors = sum(i.severity == "error" for i in issues)
|
|
79
|
+
if args.format == "json":
|
|
80
|
+
_write(to_json({
|
|
81
|
+
"schema_version": "1",
|
|
82
|
+
"run": {"golden": args.golden, "predictions": args.predictions,
|
|
83
|
+
"tool_version": __version__},
|
|
84
|
+
"summary": {"error": errors, "warn": len(issues) - errors},
|
|
85
|
+
"issues": [asdict(i) for i in issues],
|
|
86
|
+
}))
|
|
87
|
+
else:
|
|
88
|
+
for i in issues:
|
|
89
|
+
print(visible(f"[{i.severity}] {i.id or '(전체)'}: {i.message}"))
|
|
90
|
+
print(f"error {errors}개 · warn {len(issues) - errors}개" if issues else "문제 없음")
|
|
91
|
+
return 1 if errors else 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _load_baseline(path):
|
|
95
|
+
try:
|
|
96
|
+
with open(path, encoding="utf-8") as f:
|
|
97
|
+
data = json.load(f)
|
|
98
|
+
except FileNotFoundError:
|
|
99
|
+
raise InputError(f"파일이 없습니다: {path}") from None
|
|
100
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
101
|
+
raise InputError(f"JSON 형식 오류 ({path}): {e}") from None
|
|
102
|
+
items = data.get("items") if isinstance(data, dict) else None
|
|
103
|
+
if not isinstance(items, list) or not all(isinstance(r, dict) and "id" in r and "label" in r
|
|
104
|
+
for r in items):
|
|
105
|
+
raise InputError(f"{path}: sqlverdict score --format json 결과가 아닙니다")
|
|
106
|
+
return data
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _cmd_score(args):
|
|
110
|
+
golden = load_golden(args.golden)
|
|
111
|
+
preds = load_predictions(args.predictions)
|
|
112
|
+
baseline = _load_baseline(args.baseline) if args.baseline else None
|
|
113
|
+
report = score(golden, preds, tolerance=args.tolerance,
|
|
114
|
+
include_unverified=args.include_unverified, head=args.head,
|
|
115
|
+
include_expected=args.include_expected, baseline=baseline,
|
|
116
|
+
golden_path=args.golden, preds_path=args.predictions)
|
|
117
|
+
_write(RENDER[args.format](report), args.output)
|
|
118
|
+
hit = {k: report["summary"][k] for k in args.fail_on if report["summary"][k]}
|
|
119
|
+
if hit:
|
|
120
|
+
print("--fail-on 걸림: " + ", ".join(f"{k} {n}개" for k, n in hit.items()), file=sys.stderr)
|
|
121
|
+
return 1
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _cmd_init(args):
|
|
126
|
+
dest = Path(args.dir)
|
|
127
|
+
exists = [n for n in TEMPLATES if (dest / n).exists()]
|
|
128
|
+
if exists and not args.force:
|
|
129
|
+
print(f"이미 있음: {', '.join(exists)} (덮어쓰려면 --force)", file=sys.stderr)
|
|
130
|
+
return 2
|
|
131
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
132
|
+
src = files("sqlverdict") / "templates"
|
|
133
|
+
for n in TEMPLATES:
|
|
134
|
+
(dest / n).write_text((src / n).read_text(encoding="utf-8"), encoding="utf-8", newline="\n")
|
|
135
|
+
print(f"생성: {dest / n}")
|
|
136
|
+
print(f"다음: sqlverdict check {dest / 'golden.yaml'} → 예시 데이터를 실제 DB 정답으로 교체")
|
|
137
|
+
return 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
COMMANDS = {"init": _cmd_init, "check": _cmd_check, "score": _cmd_score}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def main(argv=None):
|
|
144
|
+
for stream in (sys.stdout, sys.stderr):
|
|
145
|
+
if hasattr(stream, "reconfigure"):
|
|
146
|
+
stream.reconfigure(errors="backslashreplace")
|
|
147
|
+
args = _parser().parse_args(argv)
|
|
148
|
+
try:
|
|
149
|
+
return COMMANDS[args.cmd](args)
|
|
150
|
+
except BrokenPipeError:
|
|
151
|
+
os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
|
|
152
|
+
return 141
|
|
153
|
+
except (InputError, OSError) as e:
|
|
154
|
+
print(f"sqlverdict: {visible(str(e))}", file=sys.stderr)
|
|
155
|
+
return 2
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
if __name__ == "__main__":
|
|
159
|
+
sys.exit(main())
|
sqlverdict/compare.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
|
|
3
|
+
from sqlverdict.hints import build_hints
|
|
4
|
+
from sqlverdict.model import Result, Verdict
|
|
5
|
+
from sqlverdict.normalize import normalize_rows
|
|
6
|
+
|
|
7
|
+
# 계산 오차만 흡수 (numpy 기본 1e-5는 큰 금액에서 너무 너그러움 → code_design.md 결정 D1)
|
|
8
|
+
STRICT_RTOL, STRICT_ATOL = 1e-9, 1e-9
|
|
9
|
+
# 허용 오차는 정답 대비 상대 비율 (0.001 = 0.1%) → 결정 D2
|
|
10
|
+
DEFAULT_TOLERANCE = 0.001
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _is_num(v):
|
|
14
|
+
return isinstance(v, (int, float)) and not isinstance(v, bool)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def cell_match(exp, got, tolerance):
|
|
18
|
+
if exp is None or got is None:
|
|
19
|
+
return "strict" if exp is None and got is None else None
|
|
20
|
+
if _is_num(exp) and _is_num(got):
|
|
21
|
+
diff = abs(exp - got)
|
|
22
|
+
noise = STRICT_ATOL + STRICT_RTOL * abs(exp)
|
|
23
|
+
if diff <= noise:
|
|
24
|
+
return "strict"
|
|
25
|
+
if diff <= tolerance * abs(exp) + noise:
|
|
26
|
+
return "approx"
|
|
27
|
+
return None
|
|
28
|
+
return "strict" if exp == got else None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def match_columns(exp_cols, got_cols, want):
|
|
32
|
+
lower = [c.lower() for c in got_cols]
|
|
33
|
+
by_name = {c: lower.index(c.lower()) for c in want if c.lower() in lower}
|
|
34
|
+
used = set(by_name.values())
|
|
35
|
+
mapping = dict(by_name)
|
|
36
|
+
for c in want:
|
|
37
|
+
if c in mapping:
|
|
38
|
+
continue
|
|
39
|
+
i = exp_cols.index(c)
|
|
40
|
+
mapping[c] = i if i < len(got_cols) and i not in used else None
|
|
41
|
+
matched_by = "name" if len(by_name) == len(want) else "position"
|
|
42
|
+
return mapping, matched_by
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def sort_key(row):
|
|
46
|
+
return tuple(
|
|
47
|
+
(0, 0) if v is None else (1, v) if _is_num(v) else (2, str(v))
|
|
48
|
+
for v in row
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _project(rows, idx):
|
|
53
|
+
return [tuple(r[i] for i in idx) for r in rows]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _ratio(e, g, k):
|
|
57
|
+
ratios = set()
|
|
58
|
+
for er, gr in zip(e, g):
|
|
59
|
+
a, b = er[k], gr[k]
|
|
60
|
+
if not (_is_num(a) and _is_num(b)) or a == 0:
|
|
61
|
+
return None
|
|
62
|
+
ratios.add(round(b / a, 6))
|
|
63
|
+
return ratios.pop() if len(ratios) == 1 else None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def to_result(cursor):
|
|
67
|
+
if cursor.description is None:
|
|
68
|
+
raise ValueError("결과가 없는 커서입니다: SELECT를 실행한 커서를 넘기세요")
|
|
69
|
+
columns = [d[0] for d in cursor.description]
|
|
70
|
+
rows = [tuple(r.values()) if isinstance(r, Mapping) else tuple(r) for r in cursor.fetchall()]
|
|
71
|
+
return Result(columns, rows)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _as_result(r):
|
|
75
|
+
if isinstance(r, Result):
|
|
76
|
+
return r
|
|
77
|
+
return Result(list(r["columns"]), [tuple(row) for row in r["rows"]])
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=None,
|
|
81
|
+
ordered=False, compare_columns=None, allow_extra_columns=False,
|
|
82
|
+
tolerance=DEFAULT_TOLERANCE, pred_sql=None):
|
|
83
|
+
v = _compare(expected, got, dialect=dialect, expected_dialect=expected_dialect,
|
|
84
|
+
got_dialect=got_dialect, ordered=ordered, compare_columns=compare_columns,
|
|
85
|
+
allow_extra_columns=allow_extra_columns, tolerance=tolerance)
|
|
86
|
+
v.hints = build_hints(v.label, v.detail, pred_sql)
|
|
87
|
+
return v
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=None,
|
|
91
|
+
ordered=False, compare_columns=None, allow_extra_columns=False, tolerance=DEFAULT_TOLERANCE):
|
|
92
|
+
exp_d = expected_dialect or dialect
|
|
93
|
+
got_d = got_dialect or dialect or exp_d
|
|
94
|
+
if exp_d is None:
|
|
95
|
+
raise ValueError("dialect 또는 expected_dialect가 필요합니다")
|
|
96
|
+
exp, got = _as_result(expected), _as_result(got)
|
|
97
|
+
want = list(compare_columns or exp.columns)
|
|
98
|
+
mapping, matched_by = match_columns(exp.columns, got.columns, want)
|
|
99
|
+
detail = {
|
|
100
|
+
"expected_rows": len(exp.rows), "got_rows": len(got.rows), "matched_by": matched_by,
|
|
101
|
+
"column_map": {c: got.columns[i] for c, i in mapping.items()
|
|
102
|
+
if i is not None and got.columns[i] != c},
|
|
103
|
+
}
|
|
104
|
+
missing = [c for c, i in mapping.items() if i is None]
|
|
105
|
+
if missing:
|
|
106
|
+
detail["missing_columns"] = missing
|
|
107
|
+
return Verdict("silent_wrong", detail)
|
|
108
|
+
e = _project(normalize_rows(exp.rows, exp_d), [exp.columns.index(c) for c in want])
|
|
109
|
+
g = _project(normalize_rows(got.rows, got_d), [mapping[c] for c in want])
|
|
110
|
+
if len(e) != len(g):
|
|
111
|
+
return Verdict("silent_wrong", detail)
|
|
112
|
+
if not ordered:
|
|
113
|
+
e, g = sorted(e, key=sort_key), sorted(g, key=sort_key)
|
|
114
|
+
|
|
115
|
+
approx, differing = False, []
|
|
116
|
+
for er, gr in zip(e, g):
|
|
117
|
+
for c, a, b in zip(want, er, gr):
|
|
118
|
+
m = cell_match(a, b, tolerance)
|
|
119
|
+
if m is None and c not in differing:
|
|
120
|
+
differing.append(c)
|
|
121
|
+
approx = approx or m == "approx"
|
|
122
|
+
|
|
123
|
+
if differing:
|
|
124
|
+
detail["differing_columns"] = differing
|
|
125
|
+
for c in differing:
|
|
126
|
+
r = _ratio(e, g, want.index(c))
|
|
127
|
+
if r is not None and r != 1:
|
|
128
|
+
detail["ratio"] = r
|
|
129
|
+
break
|
|
130
|
+
return Verdict("silent_wrong", detail)
|
|
131
|
+
detail["approx"] = approx
|
|
132
|
+
if len(got.columns) > len(exp.columns) and not allow_extra_columns:
|
|
133
|
+
detail["extra_columns"] = [col for i, col in enumerate(got.columns)
|
|
134
|
+
if i not in mapping.values()]
|
|
135
|
+
return Verdict("extra_columns", detail)
|
|
136
|
+
return Verdict("correct_approx" if approx else "correct", detail)
|
sqlverdict/hints.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
_SELECT_STAR = re.compile(r"select\s+(distinct\s+)?\*", re.IGNORECASE)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _ratio_hint(r):
|
|
7
|
+
head = f"값이 정확히 {r:g}배: "
|
|
8
|
+
if r in (100, 0.01):
|
|
9
|
+
return head + "비율 단위 차이(% vs 0~1) 의심"
|
|
10
|
+
if r > 1:
|
|
11
|
+
return head + "중복 조인·필터 누락 의심"
|
|
12
|
+
return head + "조건 과다·INNER JOIN으로 행 누락 의심"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def build_hints(label, detail, pred_sql=None):
|
|
16
|
+
hints = []
|
|
17
|
+
er, gr = detail.get("expected_rows"), detail.get("got_rows")
|
|
18
|
+
if label == "silent_wrong" and er and gr and gr > er and gr % er == 0:
|
|
19
|
+
hints.append(f"행 수가 정확히 {gr // er}배: 중복 조인 의심")
|
|
20
|
+
if label == "silent_wrong" and er and gr == 0:
|
|
21
|
+
hints.append("결과가 비어 있음: WHERE 조건이나 조인 키 불일치 의심")
|
|
22
|
+
if "ratio" in detail:
|
|
23
|
+
hints.append(_ratio_hint(detail["ratio"]))
|
|
24
|
+
if detail.get("missing_columns"):
|
|
25
|
+
hints.append("필요한 컬럼이 없음: " + ", ".join(detail["missing_columns"]))
|
|
26
|
+
if detail.get("matched_by") == "position":
|
|
27
|
+
hints.append("이름이 달라 위치로 비교함: 컬럼 이름을 맞추거나 compare_columns 지정 권장")
|
|
28
|
+
if label == "correct_approx":
|
|
29
|
+
hints.append("허용 오차 안에서만 같음: 반올림·자료형(FLOAT/DECIMAL) 차이 의심")
|
|
30
|
+
if label == "extra_columns":
|
|
31
|
+
hints.append("필요 없는 컬럼 포함: " + ", ".join(detail.get("extra_columns", [])))
|
|
32
|
+
if pred_sql and _SELECT_STAR.search(pred_sql):
|
|
33
|
+
hints.append("SELECT * 사용: 필요한 컬럼만 고르세요")
|
|
34
|
+
return hints
|
sqlverdict/loader.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from dataclasses import fields
|
|
3
|
+
from datetime import date
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
from sqlverdict.model import Golden, GoldenItem, PredItem, Predictions, Result
|
|
8
|
+
from sqlverdict.normalize import DIALECTS
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _AliasError(yaml.YAMLError):
|
|
12
|
+
def __init__(self, mark):
|
|
13
|
+
super().__init__(f"YAML 별칭(*이름)은 쓸 수 없습니다 ({mark.line + 1}번째 줄)")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _Loader(yaml.SafeLoader):
|
|
17
|
+
def compose_node(self, parent, index):
|
|
18
|
+
if self.check_event(yaml.AliasEvent):
|
|
19
|
+
raise _AliasError(self.peek_event().start_mark)
|
|
20
|
+
return super().compose_node(parent, index)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# YAML 1.1이 값을 몰래 바꾸는 규칙 제거: yes/no→bool, 01234→8진수, 12:30:00→60진수, 1_000, 0x1F
|
|
24
|
+
# 남기는 것: true/false, 10진수 정수, 소수점·지수 표기 실수, .inf/.nan
|
|
25
|
+
_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$")
|
|
26
|
+
_INT = re.compile(r"^[-+]?(0|[1-9][0-9]*)$")
|
|
27
|
+
_FLOAT = re.compile(r"^[-+]?([0-9]+(\.[0-9]*)?[eE][-+]?[0-9]+|[0-9]+\.[0-9]*|\.[0-9]+)$"
|
|
28
|
+
r"|^[-+]?\.(inf|Inf|INF)$|^\.(nan|NaN|NAN)$")
|
|
29
|
+
_REPLACED = {f"tag:yaml.org,2002:{t}" for t in ("bool", "int", "float")}
|
|
30
|
+
|
|
31
|
+
_Loader.yaml_implicit_resolvers = {
|
|
32
|
+
ch: [r for r in rs if r[0] not in _REPLACED]
|
|
33
|
+
for ch, rs in yaml.SafeLoader.yaml_implicit_resolvers.items()
|
|
34
|
+
}
|
|
35
|
+
_Loader.add_implicit_resolver("tag:yaml.org,2002:bool", _BOOL, list("tTfF"))
|
|
36
|
+
_Loader.add_implicit_resolver("tag:yaml.org,2002:int", _INT, list("-+0123456789"))
|
|
37
|
+
_Loader.add_implicit_resolver("tag:yaml.org,2002:float", _FLOAT, list("-+0123456789."))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class InputError(Exception):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read(path):
|
|
45
|
+
try:
|
|
46
|
+
with open(path, encoding="utf-8") as f:
|
|
47
|
+
data = yaml.load(f, Loader=_Loader) # noqa: S506 — _Loader는 SafeLoader 하위 클래스
|
|
48
|
+
except FileNotFoundError:
|
|
49
|
+
raise InputError(f"파일이 없습니다: {path}") from None
|
|
50
|
+
except yaml.YAMLError as e:
|
|
51
|
+
raise InputError(f"YAML 형식 오류 ({path}): {e}") from None
|
|
52
|
+
except UnicodeDecodeError:
|
|
53
|
+
raise InputError(f"{path}: UTF-8로 저장된 파일이 아닙니다 (메모장이면 '다른 이름으로 저장' → 인코딩 UTF-8)") from None
|
|
54
|
+
if not isinstance(data, dict) or not isinstance(data.get("items"), list):
|
|
55
|
+
raise InputError(f"{path}: 최상위에 items 목록이 필요합니다")
|
|
56
|
+
return data
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _dialect(value, path, required):
|
|
60
|
+
if value is None and not required:
|
|
61
|
+
return None
|
|
62
|
+
if value not in DIALECTS:
|
|
63
|
+
raise InputError(f"{path}: dialect는 {', '.join(DIALECTS)} 중 하나여야 합니다 (받은 값: {value!r})")
|
|
64
|
+
return value
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _result(raw, unique_columns):
|
|
68
|
+
if not isinstance(raw, dict) or not isinstance(raw.get("columns"), list) \
|
|
69
|
+
or not isinstance(raw.get("rows"), list):
|
|
70
|
+
return None, "columns와 rows 목록이 필요합니다"
|
|
71
|
+
cols = [str(c) for c in raw["columns"]]
|
|
72
|
+
if unique_columns and len({c.lower() for c in cols}) != len(cols):
|
|
73
|
+
return None, "컬럼 이름이 중복됩니다"
|
|
74
|
+
for i, row in enumerate(raw["rows"], 1):
|
|
75
|
+
if not isinstance(row, list) or len(row) != len(cols):
|
|
76
|
+
return None, f"{i}번째 행의 값 개수가 컬럼 수({len(cols)})와 다릅니다"
|
|
77
|
+
return Result(cols, [tuple(r) for r in raw["rows"]]), None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_GOLDEN_KEYS = {f.name for f in fields(GoldenItem)} - {"problem"}
|
|
81
|
+
_BOOL_KEYS = ("verified", "ordered", "allow_extra_columns", "empty_ok")
|
|
82
|
+
_TEXT_KEYS = ("question", "level", "trap", "why", "gold_sql")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _golden_item(raw, n):
|
|
86
|
+
if not isinstance(raw, dict):
|
|
87
|
+
return GoldenItem(id=f"#{n}", question=None, expected=None,
|
|
88
|
+
problem="문항이 '키: 값' 형식이 아닙니다")
|
|
89
|
+
item_id = f"#{n}" if raw.get("id") is None else str(raw["id"])
|
|
90
|
+
expected, p = _result(raw.get("expected"), unique_columns=True)
|
|
91
|
+
problems = [p] if p else []
|
|
92
|
+
|
|
93
|
+
unknown = sorted(set(raw) - _GOLDEN_KEYS, key=str)
|
|
94
|
+
if unknown:
|
|
95
|
+
problems.append(f"알 수 없는 필드: {', '.join(map(str, unknown))}")
|
|
96
|
+
for k in _BOOL_KEYS:
|
|
97
|
+
if k in raw and not isinstance(raw[k], bool):
|
|
98
|
+
problems.append(f"{k}는 true 또는 false여야 합니다")
|
|
99
|
+
tol = raw.get("tolerance")
|
|
100
|
+
if tol is not None and (isinstance(tol, bool) or not isinstance(tol, (int, float)) or tol < 0):
|
|
101
|
+
problems.append("tolerance는 0 이상의 숫자여야 합니다")
|
|
102
|
+
cc = raw.get("compare_columns")
|
|
103
|
+
if cc is not None:
|
|
104
|
+
if not isinstance(cc, list) or not cc:
|
|
105
|
+
problems.append("compare_columns는 비어 있지 않은 목록이어야 합니다")
|
|
106
|
+
elif expected and not {str(c) for c in cc} <= set(expected.columns):
|
|
107
|
+
problems.append("compare_columns에 정답 표에 없는 컬럼이 있습니다")
|
|
108
|
+
|
|
109
|
+
kw = {k: raw[k] for k in _GOLDEN_KEYS - {"id", "expected"} if k in raw}
|
|
110
|
+
kw.setdefault("question", None)
|
|
111
|
+
for k in _TEXT_KEYS:
|
|
112
|
+
if kw.get(k) is not None:
|
|
113
|
+
kw[k] = str(kw[k])
|
|
114
|
+
if isinstance(cc, list):
|
|
115
|
+
kw["compare_columns"] = [str(c) for c in cc]
|
|
116
|
+
return GoldenItem(id=item_id, expected=expected,
|
|
117
|
+
problem="; ".join(problems) or None, **kw)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_PRED_KEYS = {"id", "result", "error", "pred_sql"}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _pred_item(raw, n):
|
|
124
|
+
if not isinstance(raw, dict):
|
|
125
|
+
return PredItem(id=f"#{n}", problem="문항이 '키: 값' 형식이 아닙니다")
|
|
126
|
+
item_id = f"#{n}" if raw.get("id") is None else str(raw["id"])
|
|
127
|
+
problems = []
|
|
128
|
+
unknown = sorted(set(raw) - _PRED_KEYS, key=str)
|
|
129
|
+
if unknown:
|
|
130
|
+
problems.append(f"알 수 없는 필드: {', '.join(map(str, unknown))}")
|
|
131
|
+
has_result, has_error = raw.get("result") is not None, raw.get("error") is not None
|
|
132
|
+
if has_result == has_error:
|
|
133
|
+
problems.append("result와 error 중 정확히 하나가 필요합니다")
|
|
134
|
+
result = None
|
|
135
|
+
if has_result:
|
|
136
|
+
result, p = _result(raw["result"], unique_columns=False)
|
|
137
|
+
if p:
|
|
138
|
+
problems.append(p)
|
|
139
|
+
return PredItem(id=item_id, result=result,
|
|
140
|
+
error=str(raw["error"]) if has_error else None,
|
|
141
|
+
pred_sql=None if raw.get("pred_sql") is None else str(raw["pred_sql"]),
|
|
142
|
+
problem="; ".join(problems) or None)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
_GOLDEN_TOP = {"dialect", "snapshot", "items"}
|
|
146
|
+
_PRED_TOP = {"run_name", "dialect", "items"}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _check_top(data, allowed, path):
|
|
150
|
+
unknown = sorted(set(data) - allowed, key=str)
|
|
151
|
+
if unknown:
|
|
152
|
+
raise InputError(f"{path}: 알 수 없는 최상위 필드: {', '.join(map(str, unknown))}")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _snapshot(v):
|
|
156
|
+
if v is None:
|
|
157
|
+
return None
|
|
158
|
+
return v.isoformat() if isinstance(v, date) else str(v)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def load_golden(path):
|
|
162
|
+
data = _read(path)
|
|
163
|
+
_check_top(data, _GOLDEN_TOP, path)
|
|
164
|
+
return Golden(dialect=_dialect(data.get("dialect"), path, required=True),
|
|
165
|
+
snapshot=_snapshot(data.get("snapshot")),
|
|
166
|
+
items=[_golden_item(r, n) for n, r in enumerate(data["items"], 1)])
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def load_predictions(path):
|
|
170
|
+
data = _read(path)
|
|
171
|
+
_check_top(data, _PRED_TOP, path)
|
|
172
|
+
run_name = data.get("run_name")
|
|
173
|
+
return Predictions(run_name=None if run_name is None else str(run_name),
|
|
174
|
+
dialect=_dialect(data.get("dialect"), path, required=False),
|
|
175
|
+
items=[_pred_item(r, n) for n, r in enumerate(data["items"], 1)])
|