sqlverdict 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sqlverdict/model.py ADDED
@@ -0,0 +1,55 @@
1
+ from dataclasses import dataclass, field
2
+
3
+
4
+ @dataclass(frozen=True)
5
+ class Result:
6
+ columns: list[str]
7
+ rows: list[tuple]
8
+
9
+
10
+ @dataclass
11
+ class Verdict:
12
+ label: str
13
+ detail: dict = field(default_factory=dict)
14
+ hints: list[str] = field(default_factory=list)
15
+
16
+
17
+ @dataclass
18
+ class GoldenItem:
19
+ id: str
20
+ question: str | None
21
+ expected: Result | None
22
+ level: str | None = None
23
+ trap: str | None = None
24
+ why: str | None = None
25
+ verified: bool = False
26
+ ordered: bool = False
27
+ compare_columns: list[str] | None = None
28
+ allow_extra_columns: bool = False
29
+ tolerance: float | None = None
30
+ empty_ok: bool = False
31
+ gold_sql: str | None = None
32
+ problem: str | None = None
33
+
34
+
35
+ @dataclass
36
+ class Golden:
37
+ dialect: str
38
+ snapshot: str | None
39
+ items: list[GoldenItem]
40
+
41
+
42
+ @dataclass
43
+ class PredItem:
44
+ id: str
45
+ result: Result | None = None
46
+ error: str | None = None
47
+ pred_sql: str | None = None
48
+ problem: str | None = None
49
+
50
+
51
+ @dataclass
52
+ class Predictions:
53
+ run_name: str | None
54
+ dialect: str | None
55
+ items: list[PredItem]
@@ -0,0 +1,73 @@
1
+ import json
2
+ import math
3
+ from datetime import date, datetime, time, timedelta, timezone
4
+ from decimal import Decimal
5
+ from uuid import UUID
6
+
7
+ DIALECTS = ("sqlite", "mysql", "postgresql", "oracle")
8
+
9
+
10
+ def _number(v):
11
+ if isinstance(v, bool):
12
+ return int(v)
13
+ if isinstance(v, Decimal):
14
+ if v.is_nan():
15
+ return "NaN"
16
+ if v.is_finite() and v == v.to_integral_value():
17
+ return int(v)
18
+ return float(v)
19
+ if isinstance(v, float):
20
+ if math.isnan(v):
21
+ return "NaN"
22
+ if v.is_integer():
23
+ return int(v)
24
+ return v
25
+
26
+
27
+ def _other(v):
28
+ if isinstance(v, UUID):
29
+ return str(v)
30
+ if isinstance(v, (dict, list)):
31
+ return json.dumps(v, sort_keys=True, ensure_ascii=False, default=str)
32
+ return v
33
+
34
+
35
+ def _clock(h, m, s, us):
36
+ frac = f".{us:06d}".rstrip("0") if us else ""
37
+ return f"{h:02d}:{m:02d}:{s:02d}{frac}"
38
+
39
+
40
+ def _day(v):
41
+ return f"{v.year:04d}-{v.month:02d}-{v.day:02d}"
42
+
43
+
44
+ def _temporal(v):
45
+ if isinstance(v, datetime):
46
+ if v.utcoffset() is not None:
47
+ v = v.astimezone(timezone.utc)
48
+ return f"{_day(v)} {_clock(v.hour, v.minute, v.second, v.microsecond)}+00:00"
49
+ if v.time() == time(0):
50
+ return _day(v)
51
+ return f"{_day(v)} {_clock(v.hour, v.minute, v.second, v.microsecond)}"
52
+ if isinstance(v, date):
53
+ return _day(v)
54
+ if isinstance(v, time):
55
+ return _clock(v.hour, v.minute, v.second, v.microsecond)
56
+ if isinstance(v, timedelta):
57
+ if timedelta(0) <= v < timedelta(days=1):
58
+ h, rest = divmod(v.seconds, 3600)
59
+ return _clock(h, *divmod(rest, 60), v.microseconds)
60
+ return str(v)
61
+ return v
62
+
63
+
64
+ def normalize_value(v, dialect):
65
+ if dialect not in DIALECTS:
66
+ raise ValueError(f"지원하지 않는 dialect: {dialect!r} (가능: {', '.join(DIALECTS)})")
67
+ if dialect == "oracle" and v == "":
68
+ return None
69
+ return _number(_other(_temporal(v)))
70
+
71
+
72
+ def normalize_rows(rows, dialect):
73
+ return [tuple(normalize_value(v, dialect) for v in row) for row in rows]
File without changes
@@ -0,0 +1,101 @@
1
+ from html import escape
2
+
3
+ from sqlverdict.output.table import _info, _pct
4
+ from sqlverdict.score import STATUSES, VERDICTS
5
+
6
+ _CSP = "default-src 'none'; style-src 'unsafe-inline'"
7
+
8
+ _CSS = """
9
+ :root{--bg:#fff;--fg:#1f2328;--muted:#656d76;--line:#d0d7de;--head:#f6f8fa;
10
+ --bad:#cf222e;--bad-bg:#ffebe9;--warn:#9a6700;--warn-bg:#fff8c5;--err:#bc4c00;--err-bg:#fff1e5}
11
+ @media (prefers-color-scheme:dark){:root{--bg:#0d1117;--fg:#e6edf3;--muted:#8d96a0;--line:#30363d;
12
+ --head:#161b22;--bad:#ff7b72;--bad-bg:#3c1618;--warn:#d29922;--warn-bg:#2e2a1a;--err:#f0883e;--err-bg:#3a2414}}
13
+ *{box-sizing:border-box}
14
+ body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 -apple-system,"Apple SD Gothic Neo","Noto Sans KR",sans-serif}
15
+ main{max-width:1100px;margin:0 auto;padding:24px 16px}
16
+ h1{font-size:20px;margin:0 0 4px}h2{font-size:16px;margin:28px 0 8px}
17
+ .meta{color:var(--muted)}
18
+ .warn{background:var(--warn-bg);color:var(--warn);border:1px solid var(--warn);border-radius:6px;padding:8px 12px;margin:12px 0}
19
+ .big{font-size:28px;font-weight:600}
20
+ .scroll{overflow-x:auto}
21
+ table{border-collapse:collapse;width:100%}
22
+ th,td{border-bottom:1px solid var(--line);padding:6px 8px;text-align:left;vertical-align:top}
23
+ th{background:var(--head);white-space:nowrap}
24
+ tr.silent_wrong td:nth-child(2),tr.unscorable td:nth-child(2){color:var(--bad);background:var(--bad-bg);font-weight:600}
25
+ tr.error td:nth-child(2),tr.missing_prediction td:nth-child(2){color:var(--err);background:var(--err-bg)}
26
+ tr.correct_approx td:nth-child(2),tr.extra_columns td:nth-child(2){color:var(--warn);background:var(--warn-bg)}
27
+ """
28
+
29
+
30
+ def esc(v):
31
+ return escape("" if v is None else str(v), quote=True)
32
+
33
+
34
+ def _page(title, body):
35
+ return (
36
+ '<!doctype html>\n<html lang="ko"><head><meta charset="utf-8">'
37
+ f'<meta http-equiv="Content-Security-Policy" content="{_CSP}">'
38
+ '<meta name="viewport" content="width=device-width, initial-scale=1">'
39
+ f"<title>{esc(title)}</title><style>{_CSS}</style></head>"
40
+ f"<body><main>{body}</main></body></html>"
41
+ )
42
+
43
+
44
+ def _table(headers, rows, row_class=None):
45
+ head = "".join(f"<th>{esc(h)}</th>" for h in headers)
46
+ body = ""
47
+ for i, r in enumerate(rows):
48
+ cls = f' class="{esc(row_class[i])}"' if row_class else ""
49
+ body += f"<tr{cls}>" + "".join(f"<td>{esc(c)}</td>" for c in r) + "</tr>"
50
+ return f'<div class="scroll"><table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>'
51
+
52
+
53
+ def _values(title, v):
54
+ return f"<h3>{esc(title)}</h3>" + _table(v["columns"], v["rows"])
55
+
56
+
57
+ def render_html(report):
58
+ run, s = report["run"], report["summary"]
59
+ dialect = run["dialect"]
60
+ if run.get("pred_dialect") and run["pred_dialect"] != dialect:
61
+ dialect += f" → {run['pred_dialect']}"
62
+ tol = "-" if run.get("tolerance") is None else f"{run['tolerance'] * 100:g}%"
63
+ acc = "-" if s["accuracy"] is None else f"{s['accuracy']:.1%}"
64
+
65
+ parts = [
66
+ "<h1>sqlverdict 보고서</h1>",
67
+ f'<p class="meta">{esc(run["run_name"] or "-")} · {esc(dialect)} · snapshot {esc(run["snapshot"] or "-")}'
68
+ f" · tolerance {esc(tol)} · v{esc(run['tool_version'])}</p>",
69
+ ]
70
+ parts += [f'<div class="warn">{esc(w)}</div>' for w in report["warnings"]]
71
+ parts.append(f'<p><span class="big">{esc(acc)}</span> correct {s["correct"]} / 채점 {s["scored"]}'
72
+ f' / 전체 {s["total"]}</p>')
73
+ parts.append(_table(list(VERDICTS) + list(STATUSES), [[s[k] for k in VERDICTS + STATUSES]]))
74
+ if report["regressions"]:
75
+ parts.append(f"<p>새로 틀림: {esc(', '.join(report['regressions']))}</p>")
76
+ if report["improvements"]:
77
+ parts.append(f"<p>새로 맞음: {esc(', '.join(report['improvements']))}</p>")
78
+
79
+ for title, key in (("함정별", "by_trap"), ("난이도별", "by_level")):
80
+ if report[key]:
81
+ rows = [[k, g["total"], g["correct"], _pct(g["correct"], g["total"])] for k, g in report[key].items()]
82
+ parts += [f"<h2>{title}</h2>", _table([title[:-1], "채점", "정답", "정답률"], rows)]
83
+
84
+ shown = [r for r in report["items"] if r["label"] not in ("correct", "skipped_unverified")]
85
+ if shown:
86
+ rows = [[r["id"], r["label"], r["level"], r["trap"], r["question"], _info(r)] for r in shown]
87
+ parts += ["<h2>고칠 문항</h2>",
88
+ _table(["id", "판정", "level", "trap", "질문", "안내"], rows, [r["label"] for r in shown])]
89
+
90
+ with_values = [r for r in report["items"] if "got_head" in r or "expected" in r]
91
+ if with_values:
92
+ parts.append("<h2>결과 값 (옵션으로 포함됨)</h2>")
93
+ for r in with_values:
94
+ inner = ""
95
+ if "expected" in r:
96
+ inner += _values("정답", r["expected"])
97
+ if "got_head" in r:
98
+ inner += _values("예측 (앞부분)", r["got_head"])
99
+ parts.append(f"<details><summary>{esc(r['id'])} · {esc(r['label'])}</summary>{inner}</details>")
100
+
101
+ return _page(f"sqlverdict · {run['run_name'] or 'report'}", "".join(parts))
@@ -0,0 +1,5 @@
1
+ import json
2
+
3
+
4
+ def to_json(report):
5
+ return json.dumps(report, ensure_ascii=False, indent=2)
@@ -0,0 +1,92 @@
1
+ import unicodedata
2
+
3
+ from sqlverdict.score import STATUSES, VERDICTS
4
+
5
+ QUESTION_W, INFO_W = 30, 70
6
+
7
+
8
+ def _width(s):
9
+ return sum(2 if unicodedata.east_asian_width(ch) in "WF" else 1 for ch in s)
10
+
11
+
12
+ def visible(s):
13
+ return "".join(ch if ch == "\n" or unicodedata.category(ch) not in ("Cc", "Cf", "Cs")
14
+ else ch.encode("unicode_escape").decode() for ch in s)
15
+
16
+
17
+ def _cut(s, limit):
18
+ s = visible(" ".join(str(s).split()))
19
+ out, w = "", 0
20
+ for ch in s:
21
+ cw = _width(ch)
22
+ if w + cw > limit - 1:
23
+ return out + "…"
24
+ out, w = out + ch, w + cw
25
+ return out
26
+
27
+
28
+ def _pad(s, width, right=False):
29
+ fill = " " * (width - _width(s))
30
+ return fill + s if right else s + fill
31
+
32
+
33
+ def _table(headers, rows, right=()):
34
+ headers = [visible(str(h)) for h in headers]
35
+ rows = [[visible(str(c)) for c in r] for r in rows]
36
+ widths = [max(_width(x) for x in col) for col in zip(headers, *rows)]
37
+
38
+ def line(cells):
39
+ return " ".join(_pad(c, w, i in right) for i, (c, w) in enumerate(zip(cells, widths))).rstrip()
40
+
41
+ return "\n".join([line(headers), line(["-" * w for w in widths])] + [line(r) for r in rows])
42
+
43
+
44
+ def _pct(a, b):
45
+ return f"{a / b:.1%}" if b else "-"
46
+
47
+
48
+ def _groups(title, groups):
49
+ rows = [[k, g["total"], g["correct"], _pct(g["correct"], g["total"])] for k, g in groups.items()]
50
+ return _table([title, "채점", "정답", "정답률"], rows, right={1, 2, 3})
51
+
52
+
53
+ def _info(r):
54
+ if r.get("reason"):
55
+ return r["reason"]
56
+ if r.get("error"):
57
+ return "실행 실패: " + r["error"]
58
+ return " / ".join(r["hints"])
59
+
60
+
61
+ def render_table(report):
62
+ run, s = report["run"], report["summary"]
63
+ dialect = run["dialect"]
64
+ if run.get("pred_dialect") and run["pred_dialect"] != dialect:
65
+ dialect += f" → {run['pred_dialect']}"
66
+ head = (f"sqlverdict {run['tool_version']} · {run['run_name'] or '-'} · {dialect}"
67
+ f" · snapshot {run['snapshot'] or '-'}")
68
+ blocks = [head]
69
+ if report["warnings"]:
70
+ blocks.append("\n".join(f"! {w}" for w in report["warnings"]))
71
+
72
+ acc = "-" if s["accuracy"] is None else f"{s['accuracy']:.1%}"
73
+ lines = [f"정확도 {acc} (correct {s['correct']} / 채점 {s['scored']} / 전체 {s['total']})",
74
+ "판정 " + " · ".join(f"{k} {s[k]}" for k in VERDICTS),
75
+ "제외 " + " · ".join(f"{k} {s[k]}" for k in STATUSES)]
76
+ if report["regressions"]:
77
+ lines.append("새로 틀림 " + ", ".join(report["regressions"]))
78
+ if report["improvements"]:
79
+ lines.append("새로 맞음 " + ", ".join(report["improvements"]))
80
+ blocks.append("\n".join(lines))
81
+
82
+ if report["by_trap"]:
83
+ blocks.append(_groups("함정", report["by_trap"]))
84
+ if report["by_level"]:
85
+ blocks.append(_groups("난이도", report["by_level"]))
86
+
87
+ shown = [r for r in report["items"] if r["label"] not in ("correct", "skipped_unverified")]
88
+ if shown:
89
+ rows = [[r["id"], r["label"], r["level"] or "-", r["trap"] or "-",
90
+ _cut(r["question"] or "-", QUESTION_W), _cut(_info(r), INFO_W)] for r in shown]
91
+ blocks.append(_table(["id", "판정", "level", "trap", "질문", "안내"], rows))
92
+ return visible("\n\n".join(blocks))
sqlverdict/score.py ADDED
@@ -0,0 +1,125 @@
1
+ from datetime import date
2
+
3
+ from sqlverdict._version import __version__
4
+ from sqlverdict.check import STALE_DAYS
5
+ from sqlverdict.compare import DEFAULT_TOLERANCE, compare
6
+ from sqlverdict.normalize import normalize_rows
7
+
8
+ VERDICTS = ("correct", "correct_approx", "extra_columns", "silent_wrong", "error")
9
+ STATUSES = ("skipped_unverified", "unscorable", "missing_prediction")
10
+ ERROR_MAX = 300
11
+ MASS_FAIL_MIN, MASS_FAIL_RATIO = 3, 0.2
12
+
13
+
14
+ def _score_item(it, pred, *, golden_dialect, pred_dialect, tolerance,
15
+ include_unverified=False, head=0, include_expected=False):
16
+ rec = {"id": it.id, "question": it.question, "level": it.level, "trap": it.trap,
17
+ "label": None, "diff": {}, "hints": []}
18
+ if it.problem:
19
+ rec.update(label="unscorable", reason=it.problem)
20
+ elif not it.verified and not include_unverified:
21
+ rec["label"] = "skipped_unverified"
22
+ elif pred is None:
23
+ rec["label"] = "missing_prediction"
24
+ elif pred.problem:
25
+ rec.update(label="unscorable", reason=pred.problem)
26
+ elif pred.error is not None:
27
+ rec.update(label="error", error=pred.error[:ERROR_MAX])
28
+ else:
29
+ v = compare(it.expected, pred.result,
30
+ expected_dialect=golden_dialect, got_dialect=pred_dialect or golden_dialect,
31
+ ordered=it.ordered, compare_columns=it.compare_columns,
32
+ allow_extra_columns=it.allow_extra_columns,
33
+ tolerance=tolerance if it.tolerance is None else it.tolerance,
34
+ pred_sql=pred.pred_sql)
35
+ rec.update(label=v.label, diff=v.detail, hints=v.hints)
36
+ if head:
37
+ rows = normalize_rows(pred.result.rows[:head], pred_dialect or golden_dialect)
38
+ rec["got_head"] = {"columns": pred.result.columns, "rows": [list(r) for r in rows]}
39
+ if include_expected and it.expected:
40
+ rows = normalize_rows(it.expected.rows, golden_dialect)
41
+ rec["expected"] = {"columns": it.expected.columns, "rows": [list(r) for r in rows]}
42
+ return rec
43
+
44
+
45
+ def _summary(records):
46
+ summary = {"total": len(records)}
47
+ for name in STATUSES + VERDICTS:
48
+ summary[name] = sum(r["label"] == name for r in records)
49
+ scored = sum(summary[v] for v in VERDICTS)
50
+ summary["scored"] = scored
51
+ summary["accuracy"] = round(summary["correct"] / scored, 3) if scored else None
52
+ return summary
53
+
54
+
55
+ def _group(records, key):
56
+ out = {}
57
+ for r in records:
58
+ if r["label"] not in VERDICTS:
59
+ continue
60
+ g = out.setdefault(r[key] or "unknown", {"total": 0, "correct": 0})
61
+ g["total"] += 1
62
+ g["correct"] += r["label"] == "correct"
63
+ return dict(sorted(out.items()))
64
+
65
+
66
+ def _baseline_diff(records, baseline):
67
+ before = {r["id"]: r["label"] for r in baseline.get("items", [])}
68
+ regressions, improvements = [], []
69
+ for r in records:
70
+ prev, now = before.get(r["id"]), r["label"]
71
+ if prev is None or now not in VERDICTS or prev not in VERDICTS:
72
+ continue
73
+ if prev == "correct" and now != "correct":
74
+ regressions.append(r["id"])
75
+ elif prev != "correct" and now == "correct":
76
+ improvements.append(r["id"])
77
+ return regressions, improvements
78
+
79
+
80
+ def _warnings(golden, preds, records, regressions, today):
81
+ out = []
82
+ if golden.snapshot:
83
+ try:
84
+ age = (today - date.fromisoformat(golden.snapshot)).days
85
+ except ValueError:
86
+ age = None
87
+ if age is not None and age > STALE_DAYS:
88
+ out.append(f"snapshot이 {age}일 지남: 정답 다시 뽑기 권장")
89
+ scored = sum(r["label"] in VERDICTS for r in records)
90
+ if len(regressions) >= max(MASS_FAIL_MIN, MASS_FAIL_RATIO * scored):
91
+ out.append(f"이전 실행 대비 {len(regressions)}문항이 한꺼번에 틀림: "
92
+ "SQL 문제가 아니라 DB 데이터가 바뀌어 정답이 낡았을 수 있음")
93
+ gold_ids = {it.id for it in golden.items}
94
+ unknown = [p.id for p in preds.items if p.id not in gold_ids]
95
+ if unknown:
96
+ out.append(f"골든셋에 없는 예측 id {len(unknown)}개는 채점하지 않음 (check로 확인)")
97
+ return out
98
+
99
+
100
+ def score(golden, preds, *, tolerance=DEFAULT_TOLERANCE, include_unverified=False, head=0,
101
+ include_expected=False, baseline=None, today=None,
102
+ golden_path=None, preds_path=None):
103
+ today = today or date.today()
104
+ by_id = {}
105
+ for p in preds.items:
106
+ by_id.setdefault(p.id, p)
107
+ records = [_score_item(it, by_id.get(it.id), golden_dialect=golden.dialect,
108
+ pred_dialect=preds.dialect, tolerance=tolerance,
109
+ include_unverified=include_unverified, head=head,
110
+ include_expected=include_expected)
111
+ for it in golden.items]
112
+ regressions, improvements = _baseline_diff(records, baseline) if baseline else ([], [])
113
+ return {
114
+ "schema_version": "1",
115
+ "run": {"golden": golden_path, "predictions": preds_path, "run_name": preds.run_name,
116
+ "dialect": golden.dialect, "pred_dialect": preds.dialect,
117
+ "snapshot": golden.snapshot, "tolerance": tolerance, "tool_version": __version__},
118
+ "summary": _summary(records),
119
+ "by_trap": _group(records, "trap"),
120
+ "by_level": _group(records, "level"),
121
+ "regressions": regressions,
122
+ "improvements": improvements,
123
+ "warnings": _warnings(golden, preds, records, regressions, today),
124
+ "items": records,
125
+ }
@@ -0,0 +1,41 @@
1
+ # 골든셋 작성 규칙 (sqlverdict)
2
+
3
+ 이 폴더의 golden.yaml은 Text-to-SQL 채점 기준이다. 정답이 틀리면 채점 전체가 틀린다.
4
+ AI 에이전트는 이 폴더에서 작업할 때 아래 규칙을 그대로 따른다.
5
+
6
+ ## 작업 순서 (이 폴더에서 실행)
7
+ 1. 문항은 닫힌 과거 기간으로 쓴다 ("최근 30일" X → "2024년" O)
8
+ 2. 채점 전에 gold_sql을 실행해 expected를 갱신하고 snapshot을 그날 날짜로 바꾼다
9
+ 3. `sqlverdict check golden.yaml predictions.yaml --format json` → `summary.error`가 0이 될 때까지 issues를 고친다
10
+ 4. `sqlverdict score golden.yaml predictions.yaml --format json`
11
+ 5. warnings가 있으면 2번부터 다시
12
+
13
+ ## 규칙
14
+ - AI가 추가하거나 고친 문항은 반드시 `verified: false`. true로 바꾸는 건 사람만 한다
15
+ - gold_sql을 고치면 verified를 false로 되돌린다
16
+ - 함정 문항을 섞는다: dup_join(중복 조인), unique_id(동명이인 등 식별자), biz_term(사내 용어 정의), empty_ok(빈 결과가 정답)
17
+ - trap이 none이 아니면 why에 "무엇을 시험하는지 + 정답 근거"를 쓴다. 사람은 why만 읽고 검증한다
18
+ - 난이도(L1~L4)를 고르게 섞는다
19
+ - 빈 결과가 정답이면 `empty_ok: true`
20
+ - id는 중복 금지. 예측 id는 골든셋 id와 같아야 한다
21
+ - gold_sql은 필요한 컬럼만 (SELECT * 금지), NOW()·RANDOM() 같은 함수 금지
22
+ - 상위 N 문항은 N번째와 N+1번째가 동점이 아니어야 한다
23
+ - 질문은 사람마다 다른 답이 나오지 않게 구체적으로 쓴다
24
+
25
+ ## 하지 말 것
26
+ - 예측 SQL을 만들 때 golden.yaml의 expected·gold_sql을 보지 않는다 (시험 문제 유출)
27
+ - 예측을 맞추려고 expected를 고치지 않는다
28
+ - `--include-expected`, `--head`로 정답·결과 값을 보고서에 넣어 공유하지 않는다 (정답 유출·실무 데이터 노출)
29
+
30
+ ## 필드
31
+ | 필드 | 필수 | 설명 |
32
+ |---|---|---|
33
+ | id, question, level, trap, expected | O | expected = {columns, rows} |
34
+ | why | trap≠none이면 O | 검증 근거 |
35
+ | verified | | 기본 false (채점 제외) |
36
+ | ordered | | 행 순서까지 비교 (ORDER BY 문항) |
37
+ | compare_columns | | 이 컬럼만 비교 |
38
+ | allow_extra_columns | | 예측의 추가 컬럼 허용 |
39
+ | tolerance | | 이 문항의 상대 오차 (기본 0.001 = 0.1%) |
40
+ | empty_ok | | 빈 결과가 정답 |
41
+ | gold_sql | | 정답 SQL (다시 뽑기·검사용) |
@@ -0,0 +1,62 @@
1
+ # sqlverdict 골든셋 예시 — 숫자는 가상 데이터입니다.
2
+ # 실제 DB에서 gold_sql을 실행해 expected를 다시 뽑고 snapshot을 그날 날짜로 바꾸기 전에는 채점에 쓰지 마세요.
3
+ # 작성 규칙: 같은 폴더의 AGENTS.md
4
+ dialect: sqlite # sqlite | mysql | postgresql | oracle
5
+ snapshot: 2025-01-02 # expected를 DB에서 뽑은 날짜 (30일 지나면 경고)
6
+ items:
7
+ - id: Q001
8
+ question: 2024년 주문 건수는?
9
+ level: L1
10
+ trap: none
11
+ verified: true
12
+ gold_sql: SELECT COUNT(*) AS n FROM orders WHERE order_date >= '2024-01-01' AND order_date < '2025-01-01'
13
+ expected: {columns: [n], rows: [[1240]]}
14
+
15
+ - id: Q002
16
+ question: 2024년에 주문한 고객은 몇 명?
17
+ level: L2
18
+ trap: dup_join
19
+ why: orders-order_items 조인 시 고객이 상품 수만큼 중복됨. COUNT(DISTINCT customer_id)여야 함
20
+ verified: true
21
+ gold_sql: SELECT COUNT(DISTINCT customer_id) AS n FROM orders WHERE order_date >= '2024-01-01' AND order_date < '2025-01-01'
22
+ expected: {columns: [n], rows: [[312]]}
23
+
24
+ - id: Q003
25
+ question: 2024년 매출 상위 3명 고객의 id, 이름, 매출은?
26
+ level: L3
27
+ trap: unique_id
28
+ why: 이름이 같은 고객(김민준)이 둘. 이름이 아니라 customer_id로 묶어야 함. 4위와 동점 없음 확인
29
+ verified: true
30
+ ordered: true
31
+ expected:
32
+ columns: [customer_id, name, revenue]
33
+ rows:
34
+ - [17, 김민준, 4820000]
35
+ - [203, 이서연, 4510000]
36
+ - [88, 김민준, 4390000]
37
+
38
+ - id: Q004
39
+ question: 2024년 활성 고객의 평균 주문 금액은?
40
+ level: L3
41
+ trap: biz_term
42
+ why: 사내 정의상 '활성 고객' = 해당 연도 주문 2회 이상. 전체 고객 평균이 아님
43
+ verified: true
44
+ expected: {columns: [avg_amount], rows: [[152340.3333]]}
45
+
46
+ - id: Q005
47
+ question: 2024년 환불 주문 중 금액이 1천만 원 이상인 주문 id는?
48
+ level: L4
49
+ trap: empty_ok
50
+ why: 2024년 환불 최대 금액은 320만 원. 결과가 없는 게 정답
51
+ verified: true
52
+ empty_ok: true
53
+ expected: {columns: [order_id], rows: []}
54
+
55
+ - id: Q006
56
+ question: 2024년 카테고리별 주문 건수는?
57
+ level: L2
58
+ trap: none
59
+ verified: false # AI가 추가한 문항: 사람이 확인하고 true로 바꾸기 전까지 채점 제외
60
+ expected:
61
+ columns: [category, n]
62
+ rows: [[가전, 410], [도서, 530], [의류, 300]]
@@ -0,0 +1,23 @@
1
+ # 예측 결과 예시 — Text-to-SQL이 만든 SQL을 실행한 결과를 문항 id별로 적습니다.
2
+ # 각 문항은 result(실행 결과) 또는 error(실행 실패 메시지) 중 정확히 하나.
3
+ # dialect를 생략하면 골든셋의 dialect를 따릅니다.
4
+ run_name: example
5
+ items:
6
+ - id: Q001 # → correct
7
+ result: {columns: [n], rows: [[1240]]}
8
+
9
+ - id: Q002 # → silent_wrong (3배: 중복 조인 의심)
10
+ pred_sql: SELECT COUNT(o.customer_id) AS n FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.order_date >= '2024-01-01' AND o.order_date < '2025-01-01'
11
+ result: {columns: [n], rows: [[936]]}
12
+
13
+ - id: Q003 # → error
14
+ error: "no such column: c.nm"
15
+
16
+ - id: Q004 # → correct_approx (반올림 차이)
17
+ result: {columns: [avg_amount], rows: [[152340.33]]}
18
+
19
+ - id: Q005 # → correct (빈 결과가 정답)
20
+ result: {columns: [order_id], rows: []}
21
+
22
+ - id: Q006 # → skipped_unverified (골든셋에서 verified: false)
23
+ result: {columns: [category, n], rows: [[가전, 410], [도서, 530], [의류, 300]]}