sqlverdict 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/PKG-INFO +2 -2
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/README.ko.md +1 -1
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/README.md +1 -1
- sqlverdict-0.1.2/src/sqlverdict/_version.py +1 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/compare.py +9 -8
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/AGENTS.md +1 -1
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_compare.py +21 -0
- sqlverdict-0.1.1/src/sqlverdict/_version.py +0 -1
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.github/workflows/ci.yml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.github/workflows/release.yml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.gitignore +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/AGENTS.md +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/LICENSE +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/SECURITY.md +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/pyproject.toml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/__init__.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/check.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/cli.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/hints.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/loader.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/model.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/normalize.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/__init__.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/html_out.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/json_out.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/table.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/score.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/golden.yaml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/predictions.yaml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/docker-compose.yml +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/probe.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/test_real_drivers.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_check.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_cli.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_hints.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_html.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_loader.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_normalize.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_output.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_score.py +0 -0
- {sqlverdict-0.1.1 → sqlverdict-0.1.2}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sqlverdict
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Rule-based scorer for Text-to-SQL results that catches silent_wrong SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/songwookun/sqlverdict
|
|
6
6
|
Author: songwookun
|
|
@@ -151,7 +151,7 @@ items:
|
|
|
151
151
|
| `compare_columns` | | Compare only these columns |
|
|
152
152
|
| `allow_extra_columns` | | Allow extra columns in the prediction |
|
|
153
153
|
| `tolerance` | | Per-item tolerance (relative; 0 = strict only) |
|
|
154
|
-
| `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows,
|
|
154
|
+
| `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, it is correct as long as every golden column can be matched (by name or position); extra columns only produce a hint |
|
|
155
155
|
| `gold_sql` | | The golden SQL. Never executed; used by `check` and for re-pulling answers |
|
|
156
156
|
|
|
157
157
|
Authoring rules (mix in trap items, ask only about closed past periods, …) are in the `evals/AGENTS.md` that `init` creates; `check` enforces what can be checked mechanically.
|
|
@@ -124,7 +124,7 @@ items:
|
|
|
124
124
|
| `compare_columns` | | 이 컬럼만 비교 |
|
|
125
125
|
| `allow_extra_columns` | | 예측의 추가 컬럼 허용 |
|
|
126
126
|
| `tolerance` | | 이 문항의 허용 오차 (상대값, 0이면 엄격 비교만) |
|
|
127
|
-
| `empty_ok` | | 빈 결과가 정답. 정답·예측 모두 0
|
|
127
|
+
| `empty_ok` | | 빈 결과가 정답. 정답·예측 모두 0행이면, 정답 컬럼이 (이름 또는 위치로) 모두 대응될 때 correct (추가 컬럼은 힌트로만) |
|
|
128
128
|
| `gold_sql` | | 정답 SQL. 실행하지 않고 `check`의 검사·다시 뽑기용 |
|
|
129
129
|
|
|
130
130
|
작성 규칙(함정 섞기, 닫힌 과거 기간만 묻기 등)은 `init`이 만드는 `evals/AGENTS.md`에 있고, 기계로 확인 가능한 것은 `check`가 검사한다.
|
|
@@ -128,7 +128,7 @@ items:
|
|
|
128
128
|
| `compare_columns` | | Compare only these columns |
|
|
129
129
|
| `allow_extra_columns` | | Allow extra columns in the prediction |
|
|
130
130
|
| `tolerance` | | Per-item tolerance (relative; 0 = strict only) |
|
|
131
|
-
| `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows,
|
|
131
|
+
| `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, it is correct as long as every golden column can be matched (by name or position); extra columns only produce a hint |
|
|
132
132
|
| `gold_sql` | | The golden SQL. Never executed; used by `check` and for re-pulling answers |
|
|
133
133
|
|
|
134
134
|
Authoring rules (mix in trap items, ask only about closed past periods, …) are in the `evals/AGENTS.md` that `init` creates; `check` enforces what can be checked mechanically.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.2"
|
|
@@ -32,7 +32,9 @@ def match_columns(exp_cols, got_cols, want):
|
|
|
32
32
|
lower = [c.lower() for c in got_cols]
|
|
33
33
|
mapping = {c: lower.index(c.lower()) for c in want if c.lower() in lower}
|
|
34
34
|
by_name = len(mapping)
|
|
35
|
-
|
|
35
|
+
names = {c.lower() for c in exp_cols}
|
|
36
|
+
free = iter(i for i in range(len(got_cols))
|
|
37
|
+
if i not in mapping.values() and lower[i] not in names)
|
|
36
38
|
for c in want:
|
|
37
39
|
if c not in mapping:
|
|
38
40
|
mapping[c] = next(free, None)
|
|
@@ -93,13 +95,6 @@ def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=
|
|
|
93
95
|
if exp_d is None:
|
|
94
96
|
raise ValueError("dialect 또는 expected_dialect가 필요합니다")
|
|
95
97
|
exp, got = _as_result(expected), _as_result(got)
|
|
96
|
-
if empty_ok and not exp.rows and not got.rows:
|
|
97
|
-
names = {c.lower() for c in exp.columns}
|
|
98
|
-
detail = {"expected_rows": 0, "got_rows": 0, "empty_ok": True}
|
|
99
|
-
extra = [c for c in got.columns if c.lower() not in names]
|
|
100
|
-
if extra:
|
|
101
|
-
detail["extra_columns"] = extra
|
|
102
|
-
return Verdict("correct", detail)
|
|
103
98
|
want = list(compare_columns or exp.columns)
|
|
104
99
|
mapping, matched_by = match_columns(exp.columns, got.columns, want)
|
|
105
100
|
detail = {
|
|
@@ -111,6 +106,12 @@ def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=
|
|
|
111
106
|
if missing:
|
|
112
107
|
detail["missing_columns"] = missing
|
|
113
108
|
return Verdict("silent_wrong", detail)
|
|
109
|
+
if empty_ok and not exp.rows and not got.rows:
|
|
110
|
+
detail["empty_ok"] = True
|
|
111
|
+
extra = [col for i, col in enumerate(got.columns) if i not in mapping.values()]
|
|
112
|
+
if extra:
|
|
113
|
+
detail["extra_columns"] = extra
|
|
114
|
+
return Verdict("correct", detail)
|
|
114
115
|
e = _project(normalize_rows(exp.rows, exp_d), [exp.columns.index(c) for c in want])
|
|
115
116
|
g = _project(normalize_rows(got.rows, got_d), [mapping[c] for c in want])
|
|
116
117
|
if len(e) != len(g):
|
|
@@ -37,5 +37,5 @@ AI 에이전트는 이 폴더에서 작업할 때 아래 규칙을 그대로 따
|
|
|
37
37
|
| compare_columns | | 이 컬럼만 비교 |
|
|
38
38
|
| allow_extra_columns | | 예측의 추가 컬럼 허용 |
|
|
39
39
|
| tolerance | | 이 문항의 상대 오차 (기본 0.001 = 0.1%) |
|
|
40
|
-
| empty_ok | | 빈 결과가 정답. 정답·예측 모두 0
|
|
40
|
+
| empty_ok | | 빈 결과가 정답. 정답·예측 모두 0행이면, 정답 컬럼이 (이름 또는 위치로) 모두 대응될 때 정답 (추가 컬럼은 힌트로만) |
|
|
41
41
|
| gold_sql | | 정답 SQL (다시 뽑기·검사용) |
|
|
@@ -333,3 +333,24 @@ def test_empty_ok_but_prediction_has_rows():
|
|
|
333
333
|
got = {"columns": ["order_id"], "rows": [[1]]}
|
|
334
334
|
assert compare(EMPTY_EXP, got, dialect="sqlite", empty_ok=True).label == "silent_wrong"
|
|
335
335
|
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def test_report5_compare_columns_skips_other_golden_names():
|
|
339
|
+
exp = {"columns": ["state", "n", "avg"], "rows": [["SP", 10, 1.5], ["RJ", 5, 2.5]]}
|
|
340
|
+
got = {"columns": ["state", "avg", "cnt"], "rows": [["SP", 1.5, 10], ["RJ", 2.5, 5]]}
|
|
341
|
+
v = compare(exp, got, dialect="sqlite", compare_columns=["state", "n"])
|
|
342
|
+
assert v.label == "correct" and v.detail["column_map"] == {"n": "cnt"}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def test_report6_empty_ok_renamed_column_matches_by_position():
|
|
346
|
+
got = {"columns": ["cnt"], "rows": []}
|
|
347
|
+
v = compare(EMPTY_EXP, got, dialect="sqlite", empty_ok=True)
|
|
348
|
+
assert v.label == "correct" and v.detail["matched_by"] == "position"
|
|
349
|
+
assert "extra_columns" not in v.detail
|
|
350
|
+
assert any("위치로 비교함" in h for h in v.hints)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def test_empty_ok_missing_golden_column_is_silent_wrong():
|
|
354
|
+
exp = {"columns": ["a", "b"], "rows": []}
|
|
355
|
+
v = compare(exp, {"columns": ["x"], "rows": []}, dialect="sqlite", empty_ok=True)
|
|
356
|
+
assert v.label == "silent_wrong" and v.detail["missing_columns"] == ["b"]
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|