sqlverdict 0.1.1__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/PKG-INFO +2 -2
  2. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/README.ko.md +1 -1
  3. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/README.md +1 -1
  4. sqlverdict-0.1.2/src/sqlverdict/_version.py +1 -0
  5. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/compare.py +9 -8
  6. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/AGENTS.md +1 -1
  7. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_compare.py +21 -0
  8. sqlverdict-0.1.1/src/sqlverdict/_version.py +0 -1
  9. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.github/workflows/ci.yml +0 -0
  10. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.github/workflows/release.yml +0 -0
  11. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/.gitignore +0 -0
  12. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/AGENTS.md +0 -0
  13. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/LICENSE +0 -0
  14. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/SECURITY.md +0 -0
  15. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/pyproject.toml +0 -0
  16. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/__init__.py +0 -0
  17. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/check.py +0 -0
  18. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/cli.py +0 -0
  19. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/hints.py +0 -0
  20. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/loader.py +0 -0
  21. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/model.py +0 -0
  22. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/normalize.py +0 -0
  23. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/__init__.py +0 -0
  24. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/html_out.py +0 -0
  25. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/json_out.py +0 -0
  26. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/output/table.py +0 -0
  27. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/score.py +0 -0
  28. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/golden.yaml +0 -0
  29. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/src/sqlverdict/templates/predictions.yaml +0 -0
  30. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/docker-compose.yml +0 -0
  31. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/probe.py +0 -0
  32. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/db/test_real_drivers.py +0 -0
  33. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_check.py +0 -0
  34. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_cli.py +0 -0
  35. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_hints.py +0 -0
  36. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_html.py +0 -0
  37. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_loader.py +0 -0
  38. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_normalize.py +0 -0
  39. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_output.py +0 -0
  40. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/tests/test_score.py +0 -0
  41. {sqlverdict-0.1.1 → sqlverdict-0.1.2}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sqlverdict
3
- Version: 0.1.1
3
+ Version: 0.1.2
4
4
  Summary: Rule-based scorer for Text-to-SQL results that catches silent_wrong SQL
5
5
  Project-URL: Repository, https://github.com/songwookun/sqlverdict
6
6
  Author: songwookun
@@ -151,7 +151,7 @@ items:
151
151
  | `compare_columns` | | Compare only these columns |
152
152
  | `allow_extra_columns` | | Allow extra columns in the prediction |
153
153
  | `tolerance` | | Per-item tolerance (relative; 0 = strict only) |
154
- | `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, columns are not compared (extra columns only produce a hint) |
154
+ | `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, it is correct as long as every golden column can be matched (by name or position); extra columns only produce a hint |
155
155
  | `gold_sql` | | The golden SQL. Never executed; used by `check` and for re-pulling answers |
156
156
 
157
157
  Authoring rules (mix in trap items, ask only about closed past periods, …) are in the `evals/AGENTS.md` that `init` creates; `check` enforces what can be checked mechanically.
@@ -124,7 +124,7 @@ items:
124
124
  | `compare_columns` | | 이 컬럼만 비교 |
125
125
  | `allow_extra_columns` | | 예측의 추가 컬럼 허용 |
126
126
  | `tolerance` | | 이 문항의 허용 오차 (상대값, 0이면 엄격 비교만) |
127
- | `empty_ok` | | 빈 결과가 정답. 정답·예측 모두 0행이면 컬럼은 비교하지 않음 (추가 컬럼은 힌트로만) |
127
+ | `empty_ok` | | 빈 결과가 정답. 정답·예측 모두 0행이면, 정답 컬럼이 (이름 또는 위치로) 모두 대응될 때 correct (추가 컬럼은 힌트로만) |
128
128
  | `gold_sql` | | 정답 SQL. 실행하지 않고 `check`의 검사·다시 뽑기용 |
129
129
 
130
130
  작성 규칙(함정 섞기, 닫힌 과거 기간만 묻기 등)은 `init`이 만드는 `evals/AGENTS.md`에 있고, 기계로 확인 가능한 것은 `check`가 검사한다.
@@ -128,7 +128,7 @@ items:
128
128
  | `compare_columns` | | Compare only these columns |
129
129
  | `allow_extra_columns` | | Allow extra columns in the prediction |
130
130
  | `tolerance` | | Per-item tolerance (relative; 0 = strict only) |
131
- | `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, columns are not compared (extra columns only produce a hint) |
131
+ | `empty_ok` | | An empty result is the correct answer. If golden and prediction both have 0 rows, it is correct as long as every golden column can be matched (by name or position); extra columns only produce a hint |
132
132
  | `gold_sql` | | The golden SQL. Never executed; used by `check` and for re-pulling answers |
133
133
 
134
134
  Authoring rules (mix in trap items, ask only about closed past periods, …) are in the `evals/AGENTS.md` that `init` creates; `check` enforces what can be checked mechanically.
@@ -0,0 +1 @@
1
+ __version__ = "0.1.2"
@@ -32,7 +32,9 @@ def match_columns(exp_cols, got_cols, want):
32
32
  lower = [c.lower() for c in got_cols]
33
33
  mapping = {c: lower.index(c.lower()) for c in want if c.lower() in lower}
34
34
  by_name = len(mapping)
35
- free = iter(i for i in range(len(got_cols)) if i not in mapping.values())
35
+ names = {c.lower() for c in exp_cols}
36
+ free = iter(i for i in range(len(got_cols))
37
+ if i not in mapping.values() and lower[i] not in names)
36
38
  for c in want:
37
39
  if c not in mapping:
38
40
  mapping[c] = next(free, None)
@@ -93,13 +95,6 @@ def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=
93
95
  if exp_d is None:
94
96
  raise ValueError("dialect 또는 expected_dialect가 필요합니다")
95
97
  exp, got = _as_result(expected), _as_result(got)
96
- if empty_ok and not exp.rows and not got.rows:
97
- names = {c.lower() for c in exp.columns}
98
- detail = {"expected_rows": 0, "got_rows": 0, "empty_ok": True}
99
- extra = [c for c in got.columns if c.lower() not in names]
100
- if extra:
101
- detail["extra_columns"] = extra
102
- return Verdict("correct", detail)
103
98
  want = list(compare_columns or exp.columns)
104
99
  mapping, matched_by = match_columns(exp.columns, got.columns, want)
105
100
  detail = {
@@ -111,6 +106,12 @@ def _compare(expected, got, *, dialect=None, expected_dialect=None, got_dialect=
111
106
  if missing:
112
107
  detail["missing_columns"] = missing
113
108
  return Verdict("silent_wrong", detail)
109
+ if empty_ok and not exp.rows and not got.rows:
110
+ detail["empty_ok"] = True
111
+ extra = [col for i, col in enumerate(got.columns) if i not in mapping.values()]
112
+ if extra:
113
+ detail["extra_columns"] = extra
114
+ return Verdict("correct", detail)
114
115
  e = _project(normalize_rows(exp.rows, exp_d), [exp.columns.index(c) for c in want])
115
116
  g = _project(normalize_rows(got.rows, got_d), [mapping[c] for c in want])
116
117
  if len(e) != len(g):
@@ -37,5 +37,5 @@ AI 에이전트는 이 폴더에서 작업할 때 아래 규칙을 그대로 따
37
37
  | compare_columns | | 이 컬럼만 비교 |
38
38
  | allow_extra_columns | | 예측의 추가 컬럼 허용 |
39
39
  | tolerance | | 이 문항의 상대 오차 (기본 0.001 = 0.1%) |
40
- | empty_ok | | 빈 결과가 정답. 정답·예측 모두 0행이면 컬럼은 비교하지 않음 |
40
+ | empty_ok | | 빈 결과가 정답. 정답·예측 모두 0행이면, 정답 컬럼이 (이름 또는 위치로) 모두 대응될 때 정답 (추가 컬럼은 힌트로만) |
41
41
  | gold_sql | | 정답 SQL (다시 뽑기·검사용) |
@@ -333,3 +333,24 @@ def test_empty_ok_but_prediction_has_rows():
333
333
  got = {"columns": ["order_id"], "rows": [[1]]}
334
334
  assert compare(EMPTY_EXP, got, dialect="sqlite", empty_ok=True).label == "silent_wrong"
335
335
 
336
+
337
+
338
+ def test_report5_compare_columns_skips_other_golden_names():
339
+ exp = {"columns": ["state", "n", "avg"], "rows": [["SP", 10, 1.5], ["RJ", 5, 2.5]]}
340
+ got = {"columns": ["state", "avg", "cnt"], "rows": [["SP", 1.5, 10], ["RJ", 2.5, 5]]}
341
+ v = compare(exp, got, dialect="sqlite", compare_columns=["state", "n"])
342
+ assert v.label == "correct" and v.detail["column_map"] == {"n": "cnt"}
343
+
344
+
345
+ def test_report6_empty_ok_renamed_column_matches_by_position():
346
+ got = {"columns": ["cnt"], "rows": []}
347
+ v = compare(EMPTY_EXP, got, dialect="sqlite", empty_ok=True)
348
+ assert v.label == "correct" and v.detail["matched_by"] == "position"
349
+ assert "extra_columns" not in v.detail
350
+ assert any("위치로 비교함" in h for h in v.hints)
351
+
352
+
353
+ def test_empty_ok_missing_golden_column_is_silent_wrong():
354
+ exp = {"columns": ["a", "b"], "rows": []}
355
+ v = compare(exp, {"columns": ["x"], "rows": []}, dialect="sqlite", empty_ok=True)
356
+ assert v.label == "silent_wrong" and v.detail["missing_columns"] == ["b"]
@@ -1 +0,0 @@
1
- __version__ = "0.1.1"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes