sqlverdict 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sqlverdict/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ from sqlverdict._version import __version__
2
+ from sqlverdict.compare import compare, to_result
3
+ from sqlverdict.model import Result, Verdict
4
+
5
+ __all__ = ["compare", "to_result", "Result", "Verdict", "__version__"]
sqlverdict/_version.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
sqlverdict/check.py ADDED
@@ -0,0 +1,127 @@
1
+ import re
2
+ from dataclasses import dataclass
3
+ from datetime import date
4
+
5
+ from sqlverdict.hints import _SELECT_STAR
6
+
7
+ LEVELS = ("L1", "L2", "L3", "L4")
8
+ TRAPS = ("dup_join", "unique_id", "biz_term", "empty_ok", "none")
9
+ STALE_DAYS = 30
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class Issue:
14
+ severity: str # "error" | "warn"
15
+ id: str | None
16
+ message: str
17
+
18
+
19
+ def _item_issues(it):
20
+ out = []
21
+
22
+ def err(m):
23
+ out.append(Issue("error", it.id, m))
24
+
25
+ def warn(m):
26
+ out.append(Issue("warn", it.id, m))
27
+
28
+ if it.problem:
29
+ err(f"형식 오류로 채점할 수 없습니다: {it.problem}")
30
+ if not it.question:
31
+ err("question이 없습니다")
32
+ if it.level not in LEVELS:
33
+ err(f"level은 {'/'.join(LEVELS)} 중 하나여야 합니다")
34
+ if it.trap not in TRAPS:
35
+ err(f"trap은 {'/'.join(TRAPS)} 중 하나여야 합니다")
36
+ elif it.trap != "none" and not it.why:
37
+ err("trap이 있으면 why가 필요합니다 (사람이 이것만 읽고 검증)")
38
+ if not it.verified:
39
+ warn("verified: false — 사람이 확인하기 전까지 채점에서 제외됩니다")
40
+ if it.expected and not it.expected.rows and not it.empty_ok:
41
+ warn("정답이 빈 결과입니다. 의도한 것이면 empty_ok: true")
42
+ return out
43
+
44
+
45
+ _VOLATILE = re.compile(
46
+ r"\b(now|curdate|curtime|sysdate|systimestamp|getdate|random|rand|uuid|newid|"
47
+ r"current_date|current_time|current_timestamp|localtimestamp)\b", re.IGNORECASE)
48
+
49
+ _OPEN_PERIOD = re.compile(
50
+ r"최근|오늘|어제|올해|금년|작년|지금|현재|"
51
+ r"이번\s*(주|달|월|분기|해)|지난\s*(주|달|월|분기|해)|"
52
+ r"\b(recent(ly)?|today|yesterday|current(ly)?|"
53
+ r"(this|last)\s+(week|month|quarter|year))\b", re.IGNORECASE)
54
+
55
+
56
+ def _text_issues(it):
57
+ out = []
58
+ if it.gold_sql:
59
+ if _SELECT_STAR.search(it.gold_sql):
60
+ out.append(Issue("warn", it.id, "gold_sql에 SELECT *: 필요한 컬럼만 고르세요"))
61
+ m = _VOLATILE.search(it.gold_sql)
62
+ if m:
63
+ out.append(Issue("warn", it.id,
64
+ f"gold_sql에 시간·난수 함수 {m.group(1)}: 뽑을 때마다 정답이 바뀝니다"))
65
+ if it.question:
66
+ m = _OPEN_PERIOD.search(it.question)
67
+ if m:
68
+ out.append(Issue("warn", it.id,
69
+ f"질문에 열린 기간 표현 '{m.group(0)}': '2018년'처럼 닫힌 과거 기간을 권장합니다 (정답이 금방 낡음)"))
70
+ return out
71
+
72
+
73
+ def _set_issues(golden, today):
74
+ out = []
75
+ seen = set()
76
+ for it in golden.items:
77
+ if it.id in seen:
78
+ out.append(Issue("error", it.id, "id가 중복됩니다"))
79
+ seen.add(it.id)
80
+ if not golden.items:
81
+ return out + [Issue("warn", None, "문항이 없습니다")]
82
+
83
+ if {it.trap for it in golden.items} <= {"none", None}:
84
+ out.append(Issue("warn", None, "함정 문항이 없습니다: 쉬운 문항만으로는 silent_wrong을 잡지 못합니다"))
85
+ levels = {it.level for it in golden.items if it.level in LEVELS}
86
+ if len(levels) == 1:
87
+ out.append(Issue("warn", None, f"난이도가 {levels.pop()} 하나뿐입니다: 점수가 부풀려질 수 있습니다"))
88
+
89
+ if golden.snapshot is None:
90
+ out.append(Issue("warn", None, "snapshot이 없습니다: 정답이 낡았는지 판단할 기준이 없습니다"))
91
+ else:
92
+ try:
93
+ age = (today - date.fromisoformat(golden.snapshot)).days
94
+ except ValueError:
95
+ out.append(Issue("warn", None, "snapshot은 YYYY-MM-DD 형식이어야 합니다"))
96
+ else:
97
+ if age > STALE_DAYS:
98
+ out.append(Issue("warn", None, f"snapshot이 {age}일 지났습니다: 정답 다시 뽑기를 권장합니다"))
99
+ return out
100
+
101
+
102
+ def _pred_issues(golden, preds):
103
+ out = []
104
+ gold_ids = {it.id for it in golden.items}
105
+ seen = set()
106
+ for p in preds.items:
107
+ if p.id in seen:
108
+ out.append(Issue("error", p.id, "예측 id가 중복됩니다"))
109
+ seen.add(p.id)
110
+ if p.id not in gold_ids:
111
+ out.append(Issue("error", p.id, "골든셋에 없는 id입니다"))
112
+ if p.problem:
113
+ out.append(Issue("error", p.id, f"예측 형식 오류: {p.problem}"))
114
+ for it in golden.items:
115
+ if it.id not in seen:
116
+ out.append(Issue("warn", it.id, "예측이 없습니다 (missing_prediction)"))
117
+ return out
118
+
119
+
120
+ def check(golden, preds=None, today=None):
121
+ today = today or date.today()
122
+ out = _set_issues(golden, today)
123
+ for it in golden.items:
124
+ out += _item_issues(it) + _text_issues(it)
125
+ if preds is not None:
126
+ out += _pred_issues(golden, preds)
127
+ return out
sqlverdict/cli.py ADDED
@@ -0,0 +1,159 @@
1
+ import argparse
2
+ import json
3
+ import os
4
+ import sys
5
+ from dataclasses import asdict
6
+ from importlib.resources import files
7
+ from pathlib import Path
8
+
9
+ from sqlverdict._version import __version__
10
+ from sqlverdict.check import check
11
+ from sqlverdict.compare import DEFAULT_TOLERANCE
12
+ from sqlverdict.loader import InputError, load_golden, load_predictions
13
+ from sqlverdict.output.html_out import render_html
14
+ from sqlverdict.output.json_out import to_json
15
+ from sqlverdict.output.table import render_table, visible
16
+ from sqlverdict.score import STATUSES, VERDICTS, score
17
+
18
+ FAIL_CHOICES = tuple(x for x in VERDICTS + STATUSES if x != "correct")
19
+ TEMPLATES = ("golden.yaml", "predictions.yaml", "AGENTS.md")
20
+ RENDER = {"table": render_table, "json": to_json, "html": render_html}
21
+
22
+
23
+ def _non_negative(kind):
24
+ def conv(s):
25
+ v = kind(s)
26
+ if v < 0:
27
+ raise argparse.ArgumentTypeError(f"0 이상이어야 합니다: {s}")
28
+ return v
29
+ return conv
30
+
31
+
32
+ def _parser():
33
+ p = argparse.ArgumentParser(prog="sqlverdict", description="Text-to-SQL 결과 채점기")
34
+ p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
35
+ sub = p.add_subparsers(dest="cmd", required=True)
36
+
37
+ i = sub.add_parser("init", help="예시 golden.yaml·predictions.yaml·AGENTS.md 생성")
38
+ i.add_argument("dir", nargs="?", default="evals", help="만들 폴더 (기본 evals)")
39
+ i.add_argument("--force", action="store_true", help="이미 있는 파일 덮어쓰기")
40
+
41
+ c = sub.add_parser("check", help="골든셋 검사 (+ 예측 id 짝 확인)")
42
+ c.add_argument("golden")
43
+ c.add_argument("predictions", nargs="?")
44
+ c.add_argument("--format", choices=("table", "json"), default="table")
45
+
46
+ s = sub.add_parser("score", help="채점")
47
+ s.add_argument("golden")
48
+ s.add_argument("predictions")
49
+ s.add_argument("--format", choices=("table", "json", "html"), default="table")
50
+ s.add_argument("-o", "--output", help="저장할 파일 (없으면 화면 출력)")
51
+ s.add_argument("--tolerance", type=_non_negative(float), default=DEFAULT_TOLERANCE)
52
+ s.add_argument("--baseline", help="이전 실행의 JSON 결과")
53
+ s.add_argument("--fail-on", action="append", choices=FAIL_CHOICES, default=[])
54
+ s.add_argument("--head", type=_non_negative(int), default=0)
55
+ s.add_argument("--include-expected", action="store_true")
56
+ s.add_argument("--include-unverified", action="store_true")
57
+ return p
58
+
59
+
60
+ def _write(text, path=None):
61
+ if not text.endswith("\n"):
62
+ text += "\n"
63
+ if path:
64
+ with open(path, "w", encoding="utf-8", newline="\n", errors="backslashreplace") as f:
65
+ f.write(text)
66
+ print(f"저장: {path}", file=sys.stderr)
67
+ return
68
+ if not sys.stdout.isatty() and hasattr(sys.stdout, "reconfigure"):
69
+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
70
+ sys.stdout.write(text)
71
+ sys.stdout.flush()
72
+
73
+
74
+ def _cmd_check(args):
75
+ golden = load_golden(args.golden)
76
+ preds = load_predictions(args.predictions) if args.predictions else None
77
+ issues = sorted(check(golden, preds), key=lambda i: i.severity != "error")
78
+ errors = sum(i.severity == "error" for i in issues)
79
+ if args.format == "json":
80
+ _write(to_json({
81
+ "schema_version": "1",
82
+ "run": {"golden": args.golden, "predictions": args.predictions,
83
+ "tool_version": __version__},
84
+ "summary": {"error": errors, "warn": len(issues) - errors},
85
+ "issues": [asdict(i) for i in issues],
86
+ }))
87
+ else:
88
+ for i in issues:
89
+ print(visible(f"[{i.severity}] {i.id or '(전체)'}: {i.message}"))
90
+ print(f"error {errors}개 · warn {len(issues) - errors}개" if issues else "문제 없음")
91
+ return 1 if errors else 0
92
+
93
+
94
+ def _load_baseline(path):
95
+ try:
96
+ with open(path, encoding="utf-8") as f:
97
+ data = json.load(f)
98
+ except FileNotFoundError:
99
+ raise InputError(f"파일이 없습니다: {path}") from None
100
+ except (json.JSONDecodeError, UnicodeDecodeError) as e:
101
+ raise InputError(f"JSON 형식 오류 ({path}): {e}") from None
102
+ items = data.get("items") if isinstance(data, dict) else None
103
+ if not isinstance(items, list) or not all(isinstance(r, dict) and "id" in r and "label" in r
104
+ for r in items):
105
+ raise InputError(f"{path}: sqlverdict score --format json 결과가 아닙니다")
106
+ return data
107
+
108
+
109
+ def _cmd_score(args):
110
+ golden = load_golden(args.golden)
111
+ preds = load_predictions(args.predictions)
112
+ baseline = _load_baseline(args.baseline) if args.baseline else None
113
+ report = score(golden, preds, tolerance=args.tolerance,
114
+ include_unverified=args.include_unverified, head=args.head,
115
+ include_expected=args.include_expected, baseline=baseline,
116
+ golden_path=args.golden, preds_path=args.predictions)
117
+ _write(RENDER[args.format](report), args.output)
118
+ hit = {k: report["summary"][k] for k in args.fail_on if report["summary"][k]}
119
+ if hit:
120
+ print("--fail-on 걸림: " + ", ".join(f"{k} {n}개" for k, n in hit.items()), file=sys.stderr)
121
+ return 1
122
+ return 0
123
+
124
+
125
+ def _cmd_init(args):
126
+ dest = Path(args.dir)
127
+ exists = [n for n in TEMPLATES if (dest / n).exists()]
128
+ if exists and not args.force:
129
+ print(f"이미 있음: {', '.join(exists)} (덮어쓰려면 --force)", file=sys.stderr)
130
+ return 2
131
+ dest.mkdir(parents=True, exist_ok=True)
132
+ src = files("sqlverdict") / "templates"
133
+ for n in TEMPLATES:
134
+ (dest / n).write_text((src / n).read_text(encoding="utf-8"), encoding="utf-8", newline="\n")
135
+ print(f"생성: {dest / n}")
136
+ print(f"다음: sqlverdict check {dest / 'golden.yaml'} → 예시 데이터를 실제 DB 정답으로 교체")
137
+ return 0
138
+
139
+
140
+ COMMANDS = {"init": _cmd_init, "check": _cmd_check, "score": _cmd_score}
141
+
142
+
143
+ def main(argv=None):
144
+ for stream in (sys.stdout, sys.stderr):
145
+ if hasattr(stream, "reconfigure"):
146
+ stream.reconfigure(errors="backslashreplace")
147
+ args = _parser().parse_args(argv)
148
+ try:
149
+ return COMMANDS[args.cmd](args)
150
+ except BrokenPipeError:
151
+ os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
152
+ return 141
153
+ except (InputError, OSError) as e:
154
+ print(f"sqlverdict: {visible(str(e))}", file=sys.stderr)
155
+ return 2
156
+
157
+
158
+ if __name__ == "__main__":
159
+ sys.exit(main())
sqlverdict/compare.py ADDED
@@ -0,0 +1,136 @@
1
+ from collections.abc import Mapping
2
+
3
+ from sqlverdict.hints import build_hints
4
+ from sqlverdict.model import Result, Verdict
5
+ from sqlverdict.normalize import normalize_rows
6
+
7
+ # 계산 오차만 흡수 (numpy 기본 1e-5는 큰 금액에서 너무 너그러움 → code_design.md 결정 D1)
8
+ STRICT_RTOL, STRICT_ATOL = 1e-9, 1e-9
9
+ # 허용 오차는 정답 대비 상대 비율 (0.001 = 0.1%) → 결정 D2
10
+ DEFAULT_TOLERANCE = 0.001
11
+
12
+
13
+ def _is_num(v):
14
+ return isinstance(v, (int, float)) and not isinstance(v, bool)
15
+
16
+
17
+ def cell_match(exp, got, tolerance):
18
+ if exp is None or got is None:
19
+ return "strict" if exp is None and got is None else None
20
+ if _is_num(exp) and _is_num(got):
21
+ diff = abs(exp - got)
22
+ noise = STRICT_ATOL + STRICT_RTOL * abs(exp)
23
+ if diff <= noise:
24
+ return "strict"
25
+ if diff <= tolerance * abs(exp) + noise:
26
+ return "approx"
27
+ return None
28
+ return "strict" if exp == got else None
29
+
30
+
31
+ def match_columns(exp_cols, got_cols, want):
32
+ lower = [c.lower() for c in got_cols]
33
+ by_name = {c: lower.index(c.lower()) for c in want if c.lower() in lower}
34
+ used = set(by_name.values())
35
+ mapping = dict(by_name)
36
+ for c in want:
37
+ if c in mapping:
38
+ continue
39
+ i = exp_cols.index(c)
40
+ mapping[c] = i if i < len(got_cols) and i not in used else None
41
+ matched_by = "name" if len(by_name) == len(want) else "position"
42
+ return mapping, matched_by
43
+
44
+
45
+ def sort_key(row):
46
+ return tuple(
47
+ (0, 0) if v is None else (1, v) if _is_num(v) else (2, str(v))
48
+ for v in row
49
+ )
50
+
51
+
52
+ def _project(rows, idx):
53
+ return [tuple(r[i] for i in idx) for r in rows]
54
+
55
+
56
+ def _ratio(e, g, k):
57
+ ratios = set()
58
+ for er, gr in zip(e, g):
59
+ a, b = er[k], gr[k]
60
+ if not (_is_num(a) and _is_num(b)) or a == 0:
61
+ return None
62
+ ratios.add(round(b / a, 6))
63
+ return ratios.pop() if len(ratios) == 1 else None
64
+
65
+
66
+ def to_result(cursor):
67
+ if cursor.description is None:
68
+ raise ValueError("결과가 없는 커서입니다: SELECT를 실행한 커서를 넘기세요")
69
+ columns = [d[0] for d in cursor.description]
70
+ rows = [tuple(r.values()) if isinstance(r, Mapping) else tuple(r) for r in cursor.fetchall()]
71
+ return Result(columns, rows)
72
+
73
+
74
+ def _as_result(r):
75
+ if isinstance(r, Result):
76
+ return r
77
+ return Result(list(r["columns"]), [tuple(row) for row in r["rows"]])
78
+
79
+
80
+ def compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=None,
81
+ ordered=False, compare_columns=None, allow_extra_columns=False,
82
+ tolerance=DEFAULT_TOLERANCE, pred_sql=None):
83
+ v = _compare(expected, got, dialect=dialect, expected_dialect=expected_dialect,
84
+ got_dialect=got_dialect, ordered=ordered, compare_columns=compare_columns,
85
+ allow_extra_columns=allow_extra_columns, tolerance=tolerance)
86
+ v.hints = build_hints(v.label, v.detail, pred_sql)
87
+ return v
88
+
89
+
90
+ def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=None,
91
+ ordered=False, compare_columns=None, allow_extra_columns=False, tolerance=DEFAULT_TOLERANCE):
92
+ exp_d = expected_dialect or dialect
93
+ got_d = got_dialect or dialect or exp_d
94
+ if exp_d is None:
95
+ raise ValueError("dialect 또는 expected_dialect가 필요합니다")
96
+ exp, got = _as_result(expected), _as_result(got)
97
+ want = list(compare_columns or exp.columns)
98
+ mapping, matched_by = match_columns(exp.columns, got.columns, want)
99
+ detail = {
100
+ "expected_rows": len(exp.rows), "got_rows": len(got.rows), "matched_by": matched_by,
101
+ "column_map": {c: got.columns[i] for c, i in mapping.items()
102
+ if i is not None and got.columns[i] != c},
103
+ }
104
+ missing = [c for c, i in mapping.items() if i is None]
105
+ if missing:
106
+ detail["missing_columns"] = missing
107
+ return Verdict("silent_wrong", detail)
108
+ e = _project(normalize_rows(exp.rows, exp_d), [exp.columns.index(c) for c in want])
109
+ g = _project(normalize_rows(got.rows, got_d), [mapping[c] for c in want])
110
+ if len(e) != len(g):
111
+ return Verdict("silent_wrong", detail)
112
+ if not ordered:
113
+ e, g = sorted(e, key=sort_key), sorted(g, key=sort_key)
114
+
115
+ approx, differing = False, []
116
+ for er, gr in zip(e, g):
117
+ for c, a, b in zip(want, er, gr):
118
+ m = cell_match(a, b, tolerance)
119
+ if m is None and c not in differing:
120
+ differing.append(c)
121
+ approx = approx or m == "approx"
122
+
123
+ if differing:
124
+ detail["differing_columns"] = differing
125
+ for c in differing:
126
+ r = _ratio(e, g, want.index(c))
127
+ if r is not None and r != 1:
128
+ detail["ratio"] = r
129
+ break
130
+ return Verdict("silent_wrong", detail)
131
+ detail["approx"] = approx
132
+ if len(got.columns) > len(exp.columns) and not allow_extra_columns:
133
+ detail["extra_columns"] = [col for i, col in enumerate(got.columns)
134
+ if i not in mapping.values()]
135
+ return Verdict("extra_columns", detail)
136
+ return Verdict("correct_approx" if approx else "correct", detail)
sqlverdict/hints.py ADDED
@@ -0,0 +1,34 @@
1
+ import re
2
+
3
+ _SELECT_STAR = re.compile(r"select\s+(distinct\s+)?\*", re.IGNORECASE)
4
+
5
+
6
+ def _ratio_hint(r):
7
+ head = f"값이 정확히 {r:g}배: "
8
+ if r in (100, 0.01):
9
+ return head + "비율 단위 차이(% vs 0~1) 의심"
10
+ if r > 1:
11
+ return head + "중복 조인·필터 누락 의심"
12
+ return head + "조건 과다·INNER JOIN으로 행 누락 의심"
13
+
14
+
15
+ def build_hints(label, detail, pred_sql=None):
16
+ hints = []
17
+ er, gr = detail.get("expected_rows"), detail.get("got_rows")
18
+ if label == "silent_wrong" and er and gr and gr > er and gr % er == 0:
19
+ hints.append(f"행 수가 정확히 {gr // er}배: 중복 조인 의심")
20
+ if label == "silent_wrong" and er and gr == 0:
21
+ hints.append("결과가 비어 있음: WHERE 조건이나 조인 키 불일치 의심")
22
+ if "ratio" in detail:
23
+ hints.append(_ratio_hint(detail["ratio"]))
24
+ if detail.get("missing_columns"):
25
+ hints.append("필요한 컬럼이 없음: " + ", ".join(detail["missing_columns"]))
26
+ if detail.get("matched_by") == "position":
27
+ hints.append("이름이 달라 위치로 비교함: 컬럼 이름을 맞추거나 compare_columns 지정 권장")
28
+ if label == "correct_approx":
29
+ hints.append("허용 오차 안에서만 같음: 반올림·자료형(FLOAT/DECIMAL) 차이 의심")
30
+ if label == "extra_columns":
31
+ hints.append("필요 없는 컬럼 포함: " + ", ".join(detail.get("extra_columns", [])))
32
+ if pred_sql and _SELECT_STAR.search(pred_sql):
33
+ hints.append("SELECT * 사용: 필요한 컬럼만 고르세요")
34
+ return hints
sqlverdict/loader.py ADDED
@@ -0,0 +1,175 @@
1
+ import re
2
+ from dataclasses import fields
3
+ from datetime import date
4
+
5
+ import yaml
6
+
7
+ from sqlverdict.model import Golden, GoldenItem, PredItem, Predictions, Result
8
+ from sqlverdict.normalize import DIALECTS
9
+
10
+
11
+ class _AliasError(yaml.YAMLError):
12
+ def __init__(self, mark):
13
+ super().__init__(f"YAML 별칭(*이름)은 쓸 수 없습니다 ({mark.line + 1}번째 줄)")
14
+
15
+
16
+ class _Loader(yaml.SafeLoader):
17
+ def compose_node(self, parent, index):
18
+ if self.check_event(yaml.AliasEvent):
19
+ raise _AliasError(self.peek_event().start_mark)
20
+ return super().compose_node(parent, index)
21
+
22
+
23
+ # YAML 1.1이 값을 몰래 바꾸는 규칙 제거: yes/no→bool, 01234→8진수, 12:30:00→60진수, 1_000, 0x1F
24
+ # 남기는 것: true/false, 10진수 정수, 소수점·지수 표기 실수, .inf/.nan
25
+ _BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$")
26
+ _INT = re.compile(r"^[-+]?(0|[1-9][0-9]*)$")
27
+ _FLOAT = re.compile(r"^[-+]?([0-9]+(\.[0-9]*)?[eE][-+]?[0-9]+|[0-9]+\.[0-9]*|\.[0-9]+)$"
28
+ r"|^[-+]?\.(inf|Inf|INF)$|^\.(nan|NaN|NAN)$")
29
+ _REPLACED = {f"tag:yaml.org,2002:{t}" for t in ("bool", "int", "float")}
30
+
31
+ _Loader.yaml_implicit_resolvers = {
32
+ ch: [r for r in rs if r[0] not in _REPLACED]
33
+ for ch, rs in yaml.SafeLoader.yaml_implicit_resolvers.items()
34
+ }
35
+ _Loader.add_implicit_resolver("tag:yaml.org,2002:bool", _BOOL, list("tTfF"))
36
+ _Loader.add_implicit_resolver("tag:yaml.org,2002:int", _INT, list("-+0123456789"))
37
+ _Loader.add_implicit_resolver("tag:yaml.org,2002:float", _FLOAT, list("-+0123456789."))
38
+
39
+
40
+ class InputError(Exception):
41
+ pass
42
+
43
+
44
+ def _read(path):
45
+ try:
46
+ with open(path, encoding="utf-8") as f:
47
+ data = yaml.load(f, Loader=_Loader) # noqa: S506 — _Loader는 SafeLoader 하위 클래스
48
+ except FileNotFoundError:
49
+ raise InputError(f"파일이 없습니다: {path}") from None
50
+ except yaml.YAMLError as e:
51
+ raise InputError(f"YAML 형식 오류 ({path}): {e}") from None
52
+ except UnicodeDecodeError:
53
+ raise InputError(f"{path}: UTF-8로 저장된 파일이 아닙니다 (메모장이면 '다른 이름으로 저장' → 인코딩 UTF-8)") from None
54
+ if not isinstance(data, dict) or not isinstance(data.get("items"), list):
55
+ raise InputError(f"{path}: 최상위에 items 목록이 필요합니다")
56
+ return data
57
+
58
+
59
+ def _dialect(value, path, required):
60
+ if value is None and not required:
61
+ return None
62
+ if value not in DIALECTS:
63
+ raise InputError(f"{path}: dialect는 {', '.join(DIALECTS)} 중 하나여야 합니다 (받은 값: {value!r})")
64
+ return value
65
+
66
+
67
+ def _result(raw, unique_columns):
68
+ if not isinstance(raw, dict) or not isinstance(raw.get("columns"), list) \
69
+ or not isinstance(raw.get("rows"), list):
70
+ return None, "columns와 rows 목록이 필요합니다"
71
+ cols = [str(c) for c in raw["columns"]]
72
+ if unique_columns and len({c.lower() for c in cols}) != len(cols):
73
+ return None, "컬럼 이름이 중복됩니다"
74
+ for i, row in enumerate(raw["rows"], 1):
75
+ if not isinstance(row, list) or len(row) != len(cols):
76
+ return None, f"{i}번째 행의 값 개수가 컬럼 수({len(cols)})와 다릅니다"
77
+ return Result(cols, [tuple(r) for r in raw["rows"]]), None
78
+
79
+
80
+ _GOLDEN_KEYS = {f.name for f in fields(GoldenItem)} - {"problem"}
81
+ _BOOL_KEYS = ("verified", "ordered", "allow_extra_columns", "empty_ok")
82
+ _TEXT_KEYS = ("question", "level", "trap", "why", "gold_sql")
83
+
84
+
85
+ def _golden_item(raw, n):
86
+ if not isinstance(raw, dict):
87
+ return GoldenItem(id=f"#{n}", question=None, expected=None,
88
+ problem="문항이 '키: 값' 형식이 아닙니다")
89
+ item_id = f"#{n}" if raw.get("id") is None else str(raw["id"])
90
+ expected, p = _result(raw.get("expected"), unique_columns=True)
91
+ problems = [p] if p else []
92
+
93
+ unknown = sorted(set(raw) - _GOLDEN_KEYS, key=str)
94
+ if unknown:
95
+ problems.append(f"알 수 없는 필드: {', '.join(map(str, unknown))}")
96
+ for k in _BOOL_KEYS:
97
+ if k in raw and not isinstance(raw[k], bool):
98
+ problems.append(f"{k}는 true 또는 false여야 합니다")
99
+ tol = raw.get("tolerance")
100
+ if tol is not None and (isinstance(tol, bool) or not isinstance(tol, (int, float)) or tol < 0):
101
+ problems.append("tolerance는 0 이상의 숫자여야 합니다")
102
+ cc = raw.get("compare_columns")
103
+ if cc is not None:
104
+ if not isinstance(cc, list) or not cc:
105
+ problems.append("compare_columns는 비어 있지 않은 목록이어야 합니다")
106
+ elif expected and not {str(c) for c in cc} <= set(expected.columns):
107
+ problems.append("compare_columns에 정답 표에 없는 컬럼이 있습니다")
108
+
109
+ kw = {k: raw[k] for k in _GOLDEN_KEYS - {"id", "expected"} if k in raw}
110
+ kw.setdefault("question", None)
111
+ for k in _TEXT_KEYS:
112
+ if kw.get(k) is not None:
113
+ kw[k] = str(kw[k])
114
+ if isinstance(cc, list):
115
+ kw["compare_columns"] = [str(c) for c in cc]
116
+ return GoldenItem(id=item_id, expected=expected,
117
+ problem="; ".join(problems) or None, **kw)
118
+
119
+
120
+ _PRED_KEYS = {"id", "result", "error", "pred_sql"}
121
+
122
+
123
+ def _pred_item(raw, n):
124
+ if not isinstance(raw, dict):
125
+ return PredItem(id=f"#{n}", problem="문항이 '키: 값' 형식이 아닙니다")
126
+ item_id = f"#{n}" if raw.get("id") is None else str(raw["id"])
127
+ problems = []
128
+ unknown = sorted(set(raw) - _PRED_KEYS, key=str)
129
+ if unknown:
130
+ problems.append(f"알 수 없는 필드: {', '.join(map(str, unknown))}")
131
+ has_result, has_error = raw.get("result") is not None, raw.get("error") is not None
132
+ if has_result == has_error:
133
+ problems.append("result와 error 중 정확히 하나가 필요합니다")
134
+ result = None
135
+ if has_result:
136
+ result, p = _result(raw["result"], unique_columns=False)
137
+ if p:
138
+ problems.append(p)
139
+ return PredItem(id=item_id, result=result,
140
+ error=str(raw["error"]) if has_error else None,
141
+ pred_sql=None if raw.get("pred_sql") is None else str(raw["pred_sql"]),
142
+ problem="; ".join(problems) or None)
143
+
144
+
145
+ _GOLDEN_TOP = {"dialect", "snapshot", "items"}
146
+ _PRED_TOP = {"run_name", "dialect", "items"}
147
+
148
+
149
+ def _check_top(data, allowed, path):
150
+ unknown = sorted(set(data) - allowed, key=str)
151
+ if unknown:
152
+ raise InputError(f"{path}: 알 수 없는 최상위 필드: {', '.join(map(str, unknown))}")
153
+
154
+
155
+ def _snapshot(v):
156
+ if v is None:
157
+ return None
158
+ return v.isoformat() if isinstance(v, date) else str(v)
159
+
160
+
161
+ def load_golden(path):
162
+ data = _read(path)
163
+ _check_top(data, _GOLDEN_TOP, path)
164
+ return Golden(dialect=_dialect(data.get("dialect"), path, required=True),
165
+ snapshot=_snapshot(data.get("snapshot")),
166
+ items=[_golden_item(r, n) for n, r in enumerate(data["items"], 1)])
167
+
168
+
169
+ def load_predictions(path):
170
+ data = _read(path)
171
+ _check_top(data, _PRED_TOP, path)
172
+ run_name = data.get("run_name")
173
+ return Predictions(run_name=None if run_name is None else str(run_name),
174
+ dialect=_dialect(data.get("dialect"), path, required=False),
175
+ items=[_pred_item(r, n) for n, r in enumerate(data["items"], 1)])