sqlverdict 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlverdict/__init__.py +5 -0
- sqlverdict/_version.py +1 -0
- sqlverdict/check.py +127 -0
- sqlverdict/cli.py +159 -0
- sqlverdict/compare.py +136 -0
- sqlverdict/hints.py +34 -0
- sqlverdict/loader.py +175 -0
- sqlverdict/model.py +55 -0
- sqlverdict/normalize.py +73 -0
- sqlverdict/output/__init__.py +0 -0
- sqlverdict/output/html_out.py +101 -0
- sqlverdict/output/json_out.py +5 -0
- sqlverdict/output/table.py +92 -0
- sqlverdict/score.py +125 -0
- sqlverdict/templates/AGENTS.md +41 -0
- sqlverdict/templates/golden.yaml +62 -0
- sqlverdict/templates/predictions.yaml +23 -0
- sqlverdict-0.1.0.dist-info/METADATA +298 -0
- sqlverdict-0.1.0.dist-info/RECORD +22 -0
- sqlverdict-0.1.0.dist-info/WHEEL +4 -0
- sqlverdict-0.1.0.dist-info/entry_points.txt +2 -0
- sqlverdict-0.1.0.dist-info/licenses/LICENSE +21 -0
sqlverdict/model.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass(frozen=True)
|
|
5
|
+
class Result:
|
|
6
|
+
columns: list[str]
|
|
7
|
+
rows: list[tuple]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Verdict:
|
|
12
|
+
label: str
|
|
13
|
+
detail: dict = field(default_factory=dict)
|
|
14
|
+
hints: list[str] = field(default_factory=list)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class GoldenItem:
|
|
19
|
+
id: str
|
|
20
|
+
question: str | None
|
|
21
|
+
expected: Result | None
|
|
22
|
+
level: str | None = None
|
|
23
|
+
trap: str | None = None
|
|
24
|
+
why: str | None = None
|
|
25
|
+
verified: bool = False
|
|
26
|
+
ordered: bool = False
|
|
27
|
+
compare_columns: list[str] | None = None
|
|
28
|
+
allow_extra_columns: bool = False
|
|
29
|
+
tolerance: float | None = None
|
|
30
|
+
empty_ok: bool = False
|
|
31
|
+
gold_sql: str | None = None
|
|
32
|
+
problem: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Golden:
|
|
37
|
+
dialect: str
|
|
38
|
+
snapshot: str | None
|
|
39
|
+
items: list[GoldenItem]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class PredItem:
|
|
44
|
+
id: str
|
|
45
|
+
result: Result | None = None
|
|
46
|
+
error: str | None = None
|
|
47
|
+
pred_sql: str | None = None
|
|
48
|
+
problem: str | None = None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Predictions:
|
|
53
|
+
run_name: str | None
|
|
54
|
+
dialect: str | None
|
|
55
|
+
items: list[PredItem]
|
sqlverdict/normalize.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import math
|
|
3
|
+
from datetime import date, datetime, time, timedelta, timezone
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
|
|
7
|
+
DIALECTS = ("sqlite", "mysql", "postgresql", "oracle")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _number(v):
|
|
11
|
+
if isinstance(v, bool):
|
|
12
|
+
return int(v)
|
|
13
|
+
if isinstance(v, Decimal):
|
|
14
|
+
if v.is_nan():
|
|
15
|
+
return "NaN"
|
|
16
|
+
if v.is_finite() and v == v.to_integral_value():
|
|
17
|
+
return int(v)
|
|
18
|
+
return float(v)
|
|
19
|
+
if isinstance(v, float):
|
|
20
|
+
if math.isnan(v):
|
|
21
|
+
return "NaN"
|
|
22
|
+
if v.is_integer():
|
|
23
|
+
return int(v)
|
|
24
|
+
return v
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _other(v):
|
|
28
|
+
if isinstance(v, UUID):
|
|
29
|
+
return str(v)
|
|
30
|
+
if isinstance(v, (dict, list)):
|
|
31
|
+
return json.dumps(v, sort_keys=True, ensure_ascii=False, default=str)
|
|
32
|
+
return v
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _clock(h, m, s, us):
|
|
36
|
+
frac = f".{us:06d}".rstrip("0") if us else ""
|
|
37
|
+
return f"{h:02d}:{m:02d}:{s:02d}{frac}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _day(v):
|
|
41
|
+
return f"{v.year:04d}-{v.month:02d}-{v.day:02d}"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _temporal(v):
|
|
45
|
+
if isinstance(v, datetime):
|
|
46
|
+
if v.utcoffset() is not None:
|
|
47
|
+
v = v.astimezone(timezone.utc)
|
|
48
|
+
return f"{_day(v)} {_clock(v.hour, v.minute, v.second, v.microsecond)}+00:00"
|
|
49
|
+
if v.time() == time(0):
|
|
50
|
+
return _day(v)
|
|
51
|
+
return f"{_day(v)} {_clock(v.hour, v.minute, v.second, v.microsecond)}"
|
|
52
|
+
if isinstance(v, date):
|
|
53
|
+
return _day(v)
|
|
54
|
+
if isinstance(v, time):
|
|
55
|
+
return _clock(v.hour, v.minute, v.second, v.microsecond)
|
|
56
|
+
if isinstance(v, timedelta):
|
|
57
|
+
if timedelta(0) <= v < timedelta(days=1):
|
|
58
|
+
h, rest = divmod(v.seconds, 3600)
|
|
59
|
+
return _clock(h, *divmod(rest, 60), v.microseconds)
|
|
60
|
+
return str(v)
|
|
61
|
+
return v
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def normalize_value(v, dialect):
|
|
65
|
+
if dialect not in DIALECTS:
|
|
66
|
+
raise ValueError(f"지원하지 않는 dialect: {dialect!r} (가능: {', '.join(DIALECTS)})")
|
|
67
|
+
if dialect == "oracle" and v == "":
|
|
68
|
+
return None
|
|
69
|
+
return _number(_other(_temporal(v)))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def normalize_rows(rows, dialect):
|
|
73
|
+
return [tuple(normalize_value(v, dialect) for v in row) for row in rows]
|
|
File without changes
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
from html import escape
|
|
2
|
+
|
|
3
|
+
from sqlverdict.output.table import _info, _pct
|
|
4
|
+
from sqlverdict.score import STATUSES, VERDICTS
|
|
5
|
+
|
|
6
|
+
_CSP = "default-src 'none'; style-src 'unsafe-inline'"
|
|
7
|
+
|
|
8
|
+
_CSS = """
|
|
9
|
+
:root{--bg:#fff;--fg:#1f2328;--muted:#656d76;--line:#d0d7de;--head:#f6f8fa;
|
|
10
|
+
--bad:#cf222e;--bad-bg:#ffebe9;--warn:#9a6700;--warn-bg:#fff8c5;--err:#bc4c00;--err-bg:#fff1e5}
|
|
11
|
+
@media (prefers-color-scheme:dark){:root{--bg:#0d1117;--fg:#e6edf3;--muted:#8d96a0;--line:#30363d;
|
|
12
|
+
--head:#161b22;--bad:#ff7b72;--bad-bg:#3c1618;--warn:#d29922;--warn-bg:#2e2a1a;--err:#f0883e;--err-bg:#3a2414}}
|
|
13
|
+
*{box-sizing:border-box}
|
|
14
|
+
body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 -apple-system,"Apple SD Gothic Neo","Noto Sans KR",sans-serif}
|
|
15
|
+
main{max-width:1100px;margin:0 auto;padding:24px 16px}
|
|
16
|
+
h1{font-size:20px;margin:0 0 4px}h2{font-size:16px;margin:28px 0 8px}
|
|
17
|
+
.meta{color:var(--muted)}
|
|
18
|
+
.warn{background:var(--warn-bg);color:var(--warn);border:1px solid var(--warn);border-radius:6px;padding:8px 12px;margin:12px 0}
|
|
19
|
+
.big{font-size:28px;font-weight:600}
|
|
20
|
+
.scroll{overflow-x:auto}
|
|
21
|
+
table{border-collapse:collapse;width:100%}
|
|
22
|
+
th,td{border-bottom:1px solid var(--line);padding:6px 8px;text-align:left;vertical-align:top}
|
|
23
|
+
th{background:var(--head);white-space:nowrap}
|
|
24
|
+
tr.silent_wrong td:nth-child(2),tr.unscorable td:nth-child(2){color:var(--bad);background:var(--bad-bg);font-weight:600}
|
|
25
|
+
tr.error td:nth-child(2),tr.missing_prediction td:nth-child(2){color:var(--err);background:var(--err-bg)}
|
|
26
|
+
tr.correct_approx td:nth-child(2),tr.extra_columns td:nth-child(2){color:var(--warn);background:var(--warn-bg)}
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def esc(v):
|
|
31
|
+
return escape("" if v is None else str(v), quote=True)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _page(title, body):
|
|
35
|
+
return (
|
|
36
|
+
'<!doctype html>\n<html lang="ko"><head><meta charset="utf-8">'
|
|
37
|
+
f'<meta http-equiv="Content-Security-Policy" content="{_CSP}">'
|
|
38
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
39
|
+
f"<title>{esc(title)}</title><style>{_CSS}</style></head>"
|
|
40
|
+
f"<body><main>{body}</main></body></html>"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _table(headers, rows, row_class=None):
|
|
45
|
+
head = "".join(f"<th>{esc(h)}</th>" for h in headers)
|
|
46
|
+
body = ""
|
|
47
|
+
for i, r in enumerate(rows):
|
|
48
|
+
cls = f' class="{esc(row_class[i])}"' if row_class else ""
|
|
49
|
+
body += f"<tr{cls}>" + "".join(f"<td>{esc(c)}</td>" for c in r) + "</tr>"
|
|
50
|
+
return f'<div class="scroll"><table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>'
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _values(title, v):
|
|
54
|
+
return f"<h3>{esc(title)}</h3>" + _table(v["columns"], v["rows"])
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def render_html(report):
|
|
58
|
+
run, s = report["run"], report["summary"]
|
|
59
|
+
dialect = run["dialect"]
|
|
60
|
+
if run.get("pred_dialect") and run["pred_dialect"] != dialect:
|
|
61
|
+
dialect += f" → {run['pred_dialect']}"
|
|
62
|
+
tol = "-" if run.get("tolerance") is None else f"{run['tolerance'] * 100:g}%"
|
|
63
|
+
acc = "-" if s["accuracy"] is None else f"{s['accuracy']:.1%}"
|
|
64
|
+
|
|
65
|
+
parts = [
|
|
66
|
+
"<h1>sqlverdict 보고서</h1>",
|
|
67
|
+
f'<p class="meta">{esc(run["run_name"] or "-")} · {esc(dialect)} · snapshot {esc(run["snapshot"] or "-")}'
|
|
68
|
+
f" · tolerance {esc(tol)} · v{esc(run['tool_version'])}</p>",
|
|
69
|
+
]
|
|
70
|
+
parts += [f'<div class="warn">{esc(w)}</div>' for w in report["warnings"]]
|
|
71
|
+
parts.append(f'<p><span class="big">{esc(acc)}</span> correct {s["correct"]} / 채점 {s["scored"]}'
|
|
72
|
+
f' / 전체 {s["total"]}</p>')
|
|
73
|
+
parts.append(_table(list(VERDICTS) + list(STATUSES), [[s[k] for k in VERDICTS + STATUSES]]))
|
|
74
|
+
if report["regressions"]:
|
|
75
|
+
parts.append(f"<p>새로 틀림: {esc(', '.join(report['regressions']))}</p>")
|
|
76
|
+
if report["improvements"]:
|
|
77
|
+
parts.append(f"<p>새로 맞음: {esc(', '.join(report['improvements']))}</p>")
|
|
78
|
+
|
|
79
|
+
for title, key in (("함정별", "by_trap"), ("난이도별", "by_level")):
|
|
80
|
+
if report[key]:
|
|
81
|
+
rows = [[k, g["total"], g["correct"], _pct(g["correct"], g["total"])] for k, g in report[key].items()]
|
|
82
|
+
parts += [f"<h2>{title}</h2>", _table([title[:-1], "채점", "정답", "정답률"], rows)]
|
|
83
|
+
|
|
84
|
+
shown = [r for r in report["items"] if r["label"] not in ("correct", "skipped_unverified")]
|
|
85
|
+
if shown:
|
|
86
|
+
rows = [[r["id"], r["label"], r["level"], r["trap"], r["question"], _info(r)] for r in shown]
|
|
87
|
+
parts += ["<h2>고칠 문항</h2>",
|
|
88
|
+
_table(["id", "판정", "level", "trap", "질문", "안내"], rows, [r["label"] for r in shown])]
|
|
89
|
+
|
|
90
|
+
with_values = [r for r in report["items"] if "got_head" in r or "expected" in r]
|
|
91
|
+
if with_values:
|
|
92
|
+
parts.append("<h2>결과 값 (옵션으로 포함됨)</h2>")
|
|
93
|
+
for r in with_values:
|
|
94
|
+
inner = ""
|
|
95
|
+
if "expected" in r:
|
|
96
|
+
inner += _values("정답", r["expected"])
|
|
97
|
+
if "got_head" in r:
|
|
98
|
+
inner += _values("예측 (앞부분)", r["got_head"])
|
|
99
|
+
parts.append(f"<details><summary>{esc(r['id'])} · {esc(r['label'])}</summary>{inner}</details>")
|
|
100
|
+
|
|
101
|
+
return _page(f"sqlverdict · {run['run_name'] or 'report'}", "".join(parts))
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import unicodedata
|
|
2
|
+
|
|
3
|
+
from sqlverdict.score import STATUSES, VERDICTS
|
|
4
|
+
|
|
5
|
+
QUESTION_W, INFO_W = 30, 70
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _width(s):
|
|
9
|
+
return sum(2 if unicodedata.east_asian_width(ch) in "WF" else 1 for ch in s)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def visible(s):
|
|
13
|
+
return "".join(ch if ch == "\n" or unicodedata.category(ch) not in ("Cc", "Cf", "Cs")
|
|
14
|
+
else ch.encode("unicode_escape").decode() for ch in s)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _cut(s, limit):
|
|
18
|
+
s = visible(" ".join(str(s).split()))
|
|
19
|
+
out, w = "", 0
|
|
20
|
+
for ch in s:
|
|
21
|
+
cw = _width(ch)
|
|
22
|
+
if w + cw > limit - 1:
|
|
23
|
+
return out + "…"
|
|
24
|
+
out, w = out + ch, w + cw
|
|
25
|
+
return out
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _pad(s, width, right=False):
|
|
29
|
+
fill = " " * (width - _width(s))
|
|
30
|
+
return fill + s if right else s + fill
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _table(headers, rows, right=()):
|
|
34
|
+
headers = [visible(str(h)) for h in headers]
|
|
35
|
+
rows = [[visible(str(c)) for c in r] for r in rows]
|
|
36
|
+
widths = [max(_width(x) for x in col) for col in zip(headers, *rows)]
|
|
37
|
+
|
|
38
|
+
def line(cells):
|
|
39
|
+
return " ".join(_pad(c, w, i in right) for i, (c, w) in enumerate(zip(cells, widths))).rstrip()
|
|
40
|
+
|
|
41
|
+
return "\n".join([line(headers), line(["-" * w for w in widths])] + [line(r) for r in rows])
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _pct(a, b):
|
|
45
|
+
return f"{a / b:.1%}" if b else "-"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _groups(title, groups):
|
|
49
|
+
rows = [[k, g["total"], g["correct"], _pct(g["correct"], g["total"])] for k, g in groups.items()]
|
|
50
|
+
return _table([title, "채점", "정답", "정답률"], rows, right={1, 2, 3})
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _info(r):
|
|
54
|
+
if r.get("reason"):
|
|
55
|
+
return r["reason"]
|
|
56
|
+
if r.get("error"):
|
|
57
|
+
return "실행 실패: " + r["error"]
|
|
58
|
+
return " / ".join(r["hints"])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def render_table(report):
|
|
62
|
+
run, s = report["run"], report["summary"]
|
|
63
|
+
dialect = run["dialect"]
|
|
64
|
+
if run.get("pred_dialect") and run["pred_dialect"] != dialect:
|
|
65
|
+
dialect += f" → {run['pred_dialect']}"
|
|
66
|
+
head = (f"sqlverdict {run['tool_version']} · {run['run_name'] or '-'} · {dialect}"
|
|
67
|
+
f" · snapshot {run['snapshot'] or '-'}")
|
|
68
|
+
blocks = [head]
|
|
69
|
+
if report["warnings"]:
|
|
70
|
+
blocks.append("\n".join(f"! {w}" for w in report["warnings"]))
|
|
71
|
+
|
|
72
|
+
acc = "-" if s["accuracy"] is None else f"{s['accuracy']:.1%}"
|
|
73
|
+
lines = [f"정확도 {acc} (correct {s['correct']} / 채점 {s['scored']} / 전체 {s['total']})",
|
|
74
|
+
"판정 " + " · ".join(f"{k} {s[k]}" for k in VERDICTS),
|
|
75
|
+
"제외 " + " · ".join(f"{k} {s[k]}" for k in STATUSES)]
|
|
76
|
+
if report["regressions"]:
|
|
77
|
+
lines.append("새로 틀림 " + ", ".join(report["regressions"]))
|
|
78
|
+
if report["improvements"]:
|
|
79
|
+
lines.append("새로 맞음 " + ", ".join(report["improvements"]))
|
|
80
|
+
blocks.append("\n".join(lines))
|
|
81
|
+
|
|
82
|
+
if report["by_trap"]:
|
|
83
|
+
blocks.append(_groups("함정", report["by_trap"]))
|
|
84
|
+
if report["by_level"]:
|
|
85
|
+
blocks.append(_groups("난이도", report["by_level"]))
|
|
86
|
+
|
|
87
|
+
shown = [r for r in report["items"] if r["label"] not in ("correct", "skipped_unverified")]
|
|
88
|
+
if shown:
|
|
89
|
+
rows = [[r["id"], r["label"], r["level"] or "-", r["trap"] or "-",
|
|
90
|
+
_cut(r["question"] or "-", QUESTION_W), _cut(_info(r), INFO_W)] for r in shown]
|
|
91
|
+
blocks.append(_table(["id", "판정", "level", "trap", "질문", "안내"], rows))
|
|
92
|
+
return visible("\n\n".join(blocks))
|
sqlverdict/score.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from datetime import date
|
|
2
|
+
|
|
3
|
+
from sqlverdict._version import __version__
|
|
4
|
+
from sqlverdict.check import STALE_DAYS
|
|
5
|
+
from sqlverdict.compare import DEFAULT_TOLERANCE, compare
|
|
6
|
+
from sqlverdict.normalize import normalize_rows
|
|
7
|
+
|
|
8
|
+
VERDICTS = ("correct", "correct_approx", "extra_columns", "silent_wrong", "error")
|
|
9
|
+
STATUSES = ("skipped_unverified", "unscorable", "missing_prediction")
|
|
10
|
+
ERROR_MAX = 300
|
|
11
|
+
MASS_FAIL_MIN, MASS_FAIL_RATIO = 3, 0.2
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _score_item(it, pred, *, golden_dialect, pred_dialect, tolerance,
|
|
15
|
+
include_unverified=False, head=0, include_expected=False):
|
|
16
|
+
rec = {"id": it.id, "question": it.question, "level": it.level, "trap": it.trap,
|
|
17
|
+
"label": None, "diff": {}, "hints": []}
|
|
18
|
+
if it.problem:
|
|
19
|
+
rec.update(label="unscorable", reason=it.problem)
|
|
20
|
+
elif not it.verified and not include_unverified:
|
|
21
|
+
rec["label"] = "skipped_unverified"
|
|
22
|
+
elif pred is None:
|
|
23
|
+
rec["label"] = "missing_prediction"
|
|
24
|
+
elif pred.problem:
|
|
25
|
+
rec.update(label="unscorable", reason=pred.problem)
|
|
26
|
+
elif pred.error is not None:
|
|
27
|
+
rec.update(label="error", error=pred.error[:ERROR_MAX])
|
|
28
|
+
else:
|
|
29
|
+
v = compare(it.expected, pred.result,
|
|
30
|
+
expected_dialect=golden_dialect, got_dialect=pred_dialect or golden_dialect,
|
|
31
|
+
ordered=it.ordered, compare_columns=it.compare_columns,
|
|
32
|
+
allow_extra_columns=it.allow_extra_columns,
|
|
33
|
+
tolerance=tolerance if it.tolerance is None else it.tolerance,
|
|
34
|
+
pred_sql=pred.pred_sql)
|
|
35
|
+
rec.update(label=v.label, diff=v.detail, hints=v.hints)
|
|
36
|
+
if head:
|
|
37
|
+
rows = normalize_rows(pred.result.rows[:head], pred_dialect or golden_dialect)
|
|
38
|
+
rec["got_head"] = {"columns": pred.result.columns, "rows": [list(r) for r in rows]}
|
|
39
|
+
if include_expected and it.expected:
|
|
40
|
+
rows = normalize_rows(it.expected.rows, golden_dialect)
|
|
41
|
+
rec["expected"] = {"columns": it.expected.columns, "rows": [list(r) for r in rows]}
|
|
42
|
+
return rec
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _summary(records):
|
|
46
|
+
summary = {"total": len(records)}
|
|
47
|
+
for name in STATUSES + VERDICTS:
|
|
48
|
+
summary[name] = sum(r["label"] == name for r in records)
|
|
49
|
+
scored = sum(summary[v] for v in VERDICTS)
|
|
50
|
+
summary["scored"] = scored
|
|
51
|
+
summary["accuracy"] = round(summary["correct"] / scored, 3) if scored else None
|
|
52
|
+
return summary
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _group(records, key):
|
|
56
|
+
out = {}
|
|
57
|
+
for r in records:
|
|
58
|
+
if r["label"] not in VERDICTS:
|
|
59
|
+
continue
|
|
60
|
+
g = out.setdefault(r[key] or "unknown", {"total": 0, "correct": 0})
|
|
61
|
+
g["total"] += 1
|
|
62
|
+
g["correct"] += r["label"] == "correct"
|
|
63
|
+
return dict(sorted(out.items()))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _baseline_diff(records, baseline):
|
|
67
|
+
before = {r["id"]: r["label"] for r in baseline.get("items", [])}
|
|
68
|
+
regressions, improvements = [], []
|
|
69
|
+
for r in records:
|
|
70
|
+
prev, now = before.get(r["id"]), r["label"]
|
|
71
|
+
if prev is None or now not in VERDICTS or prev not in VERDICTS:
|
|
72
|
+
continue
|
|
73
|
+
if prev == "correct" and now != "correct":
|
|
74
|
+
regressions.append(r["id"])
|
|
75
|
+
elif prev != "correct" and now == "correct":
|
|
76
|
+
improvements.append(r["id"])
|
|
77
|
+
return regressions, improvements
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _warnings(golden, preds, records, regressions, today):
|
|
81
|
+
out = []
|
|
82
|
+
if golden.snapshot:
|
|
83
|
+
try:
|
|
84
|
+
age = (today - date.fromisoformat(golden.snapshot)).days
|
|
85
|
+
except ValueError:
|
|
86
|
+
age = None
|
|
87
|
+
if age is not None and age > STALE_DAYS:
|
|
88
|
+
out.append(f"snapshot이 {age}일 지남: 정답 다시 뽑기 권장")
|
|
89
|
+
scored = sum(r["label"] in VERDICTS for r in records)
|
|
90
|
+
if len(regressions) >= max(MASS_FAIL_MIN, MASS_FAIL_RATIO * scored):
|
|
91
|
+
out.append(f"이전 실행 대비 {len(regressions)}문항이 한꺼번에 틀림: "
|
|
92
|
+
"SQL 문제가 아니라 DB 데이터가 바뀌어 정답이 낡았을 수 있음")
|
|
93
|
+
gold_ids = {it.id for it in golden.items}
|
|
94
|
+
unknown = [p.id for p in preds.items if p.id not in gold_ids]
|
|
95
|
+
if unknown:
|
|
96
|
+
out.append(f"골든셋에 없는 예측 id {len(unknown)}개는 채점하지 않음 (check로 확인)")
|
|
97
|
+
return out
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def score(golden, preds, *, tolerance=DEFAULT_TOLERANCE, include_unverified=False, head=0,
|
|
101
|
+
include_expected=False, baseline=None, today=None,
|
|
102
|
+
golden_path=None, preds_path=None):
|
|
103
|
+
today = today or date.today()
|
|
104
|
+
by_id = {}
|
|
105
|
+
for p in preds.items:
|
|
106
|
+
by_id.setdefault(p.id, p)
|
|
107
|
+
records = [_score_item(it, by_id.get(it.id), golden_dialect=golden.dialect,
|
|
108
|
+
pred_dialect=preds.dialect, tolerance=tolerance,
|
|
109
|
+
include_unverified=include_unverified, head=head,
|
|
110
|
+
include_expected=include_expected)
|
|
111
|
+
for it in golden.items]
|
|
112
|
+
regressions, improvements = _baseline_diff(records, baseline) if baseline else ([], [])
|
|
113
|
+
return {
|
|
114
|
+
"schema_version": "1",
|
|
115
|
+
"run": {"golden": golden_path, "predictions": preds_path, "run_name": preds.run_name,
|
|
116
|
+
"dialect": golden.dialect, "pred_dialect": preds.dialect,
|
|
117
|
+
"snapshot": golden.snapshot, "tolerance": tolerance, "tool_version": __version__},
|
|
118
|
+
"summary": _summary(records),
|
|
119
|
+
"by_trap": _group(records, "trap"),
|
|
120
|
+
"by_level": _group(records, "level"),
|
|
121
|
+
"regressions": regressions,
|
|
122
|
+
"improvements": improvements,
|
|
123
|
+
"warnings": _warnings(golden, preds, records, regressions, today),
|
|
124
|
+
"items": records,
|
|
125
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# 골든셋 작성 규칙 (sqlverdict)
|
|
2
|
+
|
|
3
|
+
이 폴더의 golden.yaml은 Text-to-SQL 채점 기준이다. 정답이 틀리면 채점 전체가 틀린다.
|
|
4
|
+
AI 에이전트는 이 폴더에서 작업할 때 아래 규칙을 그대로 따른다.
|
|
5
|
+
|
|
6
|
+
## 작업 순서 (이 폴더에서 실행)
|
|
7
|
+
1. 문항은 닫힌 과거 기간으로 쓴다 ("최근 30일" X → "2024년" O)
|
|
8
|
+
2. 채점 전에 gold_sql을 실행해 expected를 갱신하고 snapshot을 그날 날짜로 바꾼다
|
|
9
|
+
3. `sqlverdict check golden.yaml predictions.yaml --format json` → `summary.error`가 0이 될 때까지 issues를 고친다
|
|
10
|
+
4. `sqlverdict score golden.yaml predictions.yaml --format json`
|
|
11
|
+
5. warnings가 있으면 2번부터 다시
|
|
12
|
+
|
|
13
|
+
## 규칙
|
|
14
|
+
- AI가 추가하거나 고친 문항은 반드시 `verified: false`. true로 바꾸는 건 사람만 한다
|
|
15
|
+
- gold_sql을 고치면 verified를 false로 되돌린다
|
|
16
|
+
- 함정 문항을 섞는다: dup_join(중복 조인), unique_id(동명이인 등 식별자), biz_term(사내 용어 정의), empty_ok(빈 결과가 정답)
|
|
17
|
+
- trap이 none이 아니면 why에 "무엇을 시험하는지 + 정답 근거"를 쓴다. 사람은 why만 읽고 검증한다
|
|
18
|
+
- 난이도(L1~L4)를 고르게 섞는다
|
|
19
|
+
- 빈 결과가 정답이면 `empty_ok: true`
|
|
20
|
+
- id는 중복 금지. 예측 id는 골든셋 id와 같아야 한다
|
|
21
|
+
- gold_sql은 필요한 컬럼만 (SELECT * 금지), NOW()·RANDOM() 같은 함수 금지
|
|
22
|
+
- 상위 N 문항은 N번째와 N+1번째가 동점이 아니어야 한다
|
|
23
|
+
- 질문은 사람마다 다른 답이 나오지 않게 구체적으로 쓴다
|
|
24
|
+
|
|
25
|
+
## 하지 말 것
|
|
26
|
+
- 예측 SQL을 만들 때 golden.yaml의 expected·gold_sql을 보지 않는다 (시험 문제 유출)
|
|
27
|
+
- 예측을 맞추려고 expected를 고치지 않는다
|
|
28
|
+
- `--include-expected`, `--head`로 정답·결과 값을 보고서에 넣어 공유하지 않는다 (정답 유출·실무 데이터 노출)
|
|
29
|
+
|
|
30
|
+
## 필드
|
|
31
|
+
| 필드 | 필수 | 설명 |
|
|
32
|
+
|---|---|---|
|
|
33
|
+
| id, question, level, trap, expected | O | expected = {columns, rows} |
|
|
34
|
+
| why | trap≠none이면 O | 검증 근거 |
|
|
35
|
+
| verified | | 기본 false (채점 제외) |
|
|
36
|
+
| ordered | | 행 순서까지 비교 (ORDER BY 문항) |
|
|
37
|
+
| compare_columns | | 이 컬럼만 비교 |
|
|
38
|
+
| allow_extra_columns | | 예측의 추가 컬럼 허용 |
|
|
39
|
+
| tolerance | | 이 문항의 상대 오차 (기본 0.001 = 0.1%) |
|
|
40
|
+
| empty_ok | | 빈 결과가 정답 |
|
|
41
|
+
| gold_sql | | 정답 SQL (다시 뽑기·검사용) |
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# sqlverdict 골든셋 예시 — 숫자는 가상 데이터입니다.
|
|
2
|
+
# 실제 DB에서 gold_sql을 실행해 expected를 다시 뽑고 snapshot을 그날 날짜로 바꾸기 전에는 채점에 쓰지 마세요.
|
|
3
|
+
# 작성 규칙: 같은 폴더의 AGENTS.md
|
|
4
|
+
dialect: sqlite # sqlite | mysql | postgresql | oracle
|
|
5
|
+
snapshot: 2025-01-02 # expected를 DB에서 뽑은 날짜 (30일 지나면 경고)
|
|
6
|
+
items:
|
|
7
|
+
- id: Q001
|
|
8
|
+
question: 2024년 주문 건수는?
|
|
9
|
+
level: L1
|
|
10
|
+
trap: none
|
|
11
|
+
verified: true
|
|
12
|
+
gold_sql: SELECT COUNT(*) AS n FROM orders WHERE order_date >= '2024-01-01' AND order_date < '2025-01-01'
|
|
13
|
+
expected: {columns: [n], rows: [[1240]]}
|
|
14
|
+
|
|
15
|
+
- id: Q002
|
|
16
|
+
question: 2024년에 주문한 고객은 몇 명?
|
|
17
|
+
level: L2
|
|
18
|
+
trap: dup_join
|
|
19
|
+
why: orders-order_items 조인 시 고객이 상품 수만큼 중복됨. COUNT(DISTINCT customer_id)여야 함
|
|
20
|
+
verified: true
|
|
21
|
+
gold_sql: SELECT COUNT(DISTINCT customer_id) AS n FROM orders WHERE order_date >= '2024-01-01' AND order_date < '2025-01-01'
|
|
22
|
+
expected: {columns: [n], rows: [[312]]}
|
|
23
|
+
|
|
24
|
+
- id: Q003
|
|
25
|
+
question: 2024년 매출 상위 3명 고객의 id, 이름, 매출은?
|
|
26
|
+
level: L3
|
|
27
|
+
trap: unique_id
|
|
28
|
+
why: 이름이 같은 고객(김민준)이 둘. 이름이 아니라 customer_id로 묶어야 함. 4위와 동점 없음 확인
|
|
29
|
+
verified: true
|
|
30
|
+
ordered: true
|
|
31
|
+
expected:
|
|
32
|
+
columns: [customer_id, name, revenue]
|
|
33
|
+
rows:
|
|
34
|
+
- [17, 김민준, 4820000]
|
|
35
|
+
- [203, 이서연, 4510000]
|
|
36
|
+
- [88, 김민준, 4390000]
|
|
37
|
+
|
|
38
|
+
- id: Q004
|
|
39
|
+
question: 2024년 활성 고객의 평균 주문 금액은?
|
|
40
|
+
level: L3
|
|
41
|
+
trap: biz_term
|
|
42
|
+
why: 사내 정의상 '활성 고객' = 해당 연도 주문 2회 이상. 전체 고객 평균이 아님
|
|
43
|
+
verified: true
|
|
44
|
+
expected: {columns: [avg_amount], rows: [[152340.3333]]}
|
|
45
|
+
|
|
46
|
+
- id: Q005
|
|
47
|
+
question: 2024년 환불 주문 중 금액이 1천만 원 이상인 주문 id는?
|
|
48
|
+
level: L4
|
|
49
|
+
trap: empty_ok
|
|
50
|
+
why: 2024년 환불 최대 금액은 320만 원. 결과가 없는 게 정답
|
|
51
|
+
verified: true
|
|
52
|
+
empty_ok: true
|
|
53
|
+
expected: {columns: [order_id], rows: []}
|
|
54
|
+
|
|
55
|
+
- id: Q006
|
|
56
|
+
question: 2024년 카테고리별 주문 건수는?
|
|
57
|
+
level: L2
|
|
58
|
+
trap: none
|
|
59
|
+
verified: false # AI가 추가한 문항: 사람이 확인하고 true로 바꾸기 전까지 채점 제외
|
|
60
|
+
expected:
|
|
61
|
+
columns: [category, n]
|
|
62
|
+
rows: [[가전, 410], [도서, 530], [의류, 300]]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# 예측 결과 예시 — Text-to-SQL이 만든 SQL을 실행한 결과를 문항 id별로 적습니다.
|
|
2
|
+
# 각 문항은 result(실행 결과) 또는 error(실행 실패 메시지) 중 정확히 하나.
|
|
3
|
+
# dialect를 생략하면 골든셋의 dialect를 따릅니다.
|
|
4
|
+
run_name: example
|
|
5
|
+
items:
|
|
6
|
+
- id: Q001 # → correct
|
|
7
|
+
result: {columns: [n], rows: [[1240]]}
|
|
8
|
+
|
|
9
|
+
- id: Q002 # → silent_wrong (3배: 중복 조인 의심)
|
|
10
|
+
pred_sql: SELECT COUNT(o.customer_id) AS n FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.order_date >= '2024-01-01' AND o.order_date < '2025-01-01'
|
|
11
|
+
result: {columns: [n], rows: [[936]]}
|
|
12
|
+
|
|
13
|
+
- id: Q003 # → error
|
|
14
|
+
error: "no such column: c.nm"
|
|
15
|
+
|
|
16
|
+
- id: Q004 # → correct_approx (반올림 차이)
|
|
17
|
+
result: {columns: [avg_amount], rows: [[152340.33]]}
|
|
18
|
+
|
|
19
|
+
- id: Q005 # → correct (빈 결과가 정답)
|
|
20
|
+
result: {columns: [order_id], rows: []}
|
|
21
|
+
|
|
22
|
+
- id: Q006 # → skipped_unverified (골든셋에서 verified: false)
|
|
23
|
+
result: {columns: [category, n], rows: [[가전, 410], [도서, 530], [의류, 300]]}
|