de-augur 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- augur/__init__.py +14 -0
- augur/catalog.py +104 -0
- augur/cli.py +156 -0
- augur/evaluate.py +198 -0
- augur/fixture.py +70 -0
- augur/generate.py +156 -0
- augur/keys.py +116 -0
- augur/llm.py +161 -0
- augur/py.typed +0 -0
- augur/retrieve.py +70 -0
- de_augur-0.4.0.dist-info/METADATA +95 -0
- de_augur-0.4.0.dist-info/RECORD +15 -0
- de_augur-0.4.0.dist-info/WHEEL +4 -0
- de_augur-0.4.0.dist-info/entry_points.txt +2 -0
- de_augur-0.4.0.dist-info/licenses/LICENSE +201 -0
augur/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Augur — 자연어 → SQL (스키마 RAG) + eval 하네스.
|
|
2
|
+
|
|
3
|
+
파이프라인: catalog(스키마 색인) → retrieve(관련 테이블 선별) → generate(LLM → SQL)
|
|
4
|
+
→ execute(DuckDB) → trace(JSONL). evaluate는 이 트레이스를 골든 SQL과 대조해
|
|
5
|
+
실패 모드를 자동 분류한다. 엔진은 빌린다 — LLM은 HTTP로 호출, 검색은 어휘 매칭.
|
|
6
|
+
벡터 DB 없음. 필요해지면 retrieve.py의 Scorer 하나만 바꾼다.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
__version__ = version("de-augur")
|
|
13
|
+
except PackageNotFoundError:
|
|
14
|
+
__version__ = "0.4.0"
|
augur/catalog.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""스키마 카탈로그 — Parquet 레이크에서 테이블/컬럼/샘플값을 뽑아 검색 단위로 만든다.
|
|
2
|
+
|
|
3
|
+
RAG의 "R"은 여기서 만든 SchemaDoc을 대상으로 한다. 샘플값을 넣는 이유: LLM이
|
|
4
|
+
`status = 'PAID'`처럼 존재하지 않는 리터럴을 지어내는 실패(HALLUCINATED_LITERAL)를
|
|
5
|
+
줄이기 위해. 카탈로그는 JSON으로 저장되며 질문마다 재스캔하지 않는다.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import typing as t
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import duckdb
|
|
13
|
+
from pydantic import BaseModel, ConfigDict
|
|
14
|
+
|
|
15
|
+
from arsenal_core.errors import FatalError
|
|
16
|
+
|
|
17
|
+
MAX_SAMPLE_VALUES = 5
|
|
18
|
+
SAMPLE_ROW_LIMIT = 10_000
|
|
19
|
+
TEXT_TYPES = ("VARCHAR", "TEXT", "STRING")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ColumnDoc(BaseModel):
|
|
23
|
+
model_config = ConfigDict(frozen=True)
|
|
24
|
+
name: str
|
|
25
|
+
dtype: str
|
|
26
|
+
samples: tuple[str, ...] = ()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SchemaDoc(BaseModel):
|
|
30
|
+
model_config = ConfigDict(frozen=True)
|
|
31
|
+
table: str
|
|
32
|
+
path: str # DuckDB에서 FROM에 쓸 경로 (glob 허용)
|
|
33
|
+
row_count: int
|
|
34
|
+
columns: tuple[ColumnDoc, ...]
|
|
35
|
+
|
|
36
|
+
def render(self) -> str:
|
|
37
|
+
"""프롬프트에 들어가는 형태. 짧고 결정적이어야 캐시·diff가 가능하다."""
|
|
38
|
+
cols = ", ".join(
|
|
39
|
+
f"{c.name} {c.dtype}" + (f" e.g. {list(c.samples)}" if c.samples else "")
|
|
40
|
+
for c in self.columns
|
|
41
|
+
)
|
|
42
|
+
return f"TABLE {self.table} (source: '{self.path}', rows={self.row_count})\n {cols}"
|
|
43
|
+
|
|
44
|
+
def column_names(self) -> frozenset[str]:
|
|
45
|
+
return frozenset(c.name.lower() for c in self.columns)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Catalog(BaseModel):
|
|
49
|
+
model_config = ConfigDict(frozen=True)
|
|
50
|
+
tables: tuple[SchemaDoc, ...]
|
|
51
|
+
|
|
52
|
+
def get(self, table: str) -> SchemaDoc | None:
|
|
53
|
+
return next((d for d in self.tables if d.table == table), None)
|
|
54
|
+
|
|
55
|
+
def save(self, path: Path) -> None:
|
|
56
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
path.write_text(self.model_dump_json(indent=2), encoding="utf-8")
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def load(cls, path: Path) -> "Catalog":
|
|
61
|
+
if not path.exists():
|
|
62
|
+
raise FatalError(f"catalog not found: {path} (run `augur index` first)")
|
|
63
|
+
return cls.model_validate(json.loads(path.read_text(encoding="utf-8")))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _sample_values(
|
|
67
|
+
con: duckdb.DuckDBPyConnection, path: str, column: str, dtype: str
|
|
68
|
+
) -> tuple[str, ...]:
|
|
69
|
+
if not any(tt in dtype.upper() for tt in TEXT_TYPES):
|
|
70
|
+
return ()
|
|
71
|
+
sql = (
|
|
72
|
+
f'SELECT "{column}" AS v, count(*) AS n '
|
|
73
|
+
f"FROM (SELECT * FROM '{path}' LIMIT {SAMPLE_ROW_LIMIT}) "
|
|
74
|
+
f'WHERE "{column}" IS NOT NULL GROUP BY 1 ORDER BY n DESC LIMIT {MAX_SAMPLE_VALUES}'
|
|
75
|
+
)
|
|
76
|
+
rows = t.cast(list[tuple[t.Any, int]], con.execute(sql).fetchall())
|
|
77
|
+
return tuple(str(r[0]) for r in rows)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def describe(table: str, path: str) -> SchemaDoc:
|
|
81
|
+
"""Parquet 경로 하나를 SchemaDoc으로. DuckDB 에러는 FatalError로 변환."""
|
|
82
|
+
con = duckdb.connect()
|
|
83
|
+
try:
|
|
84
|
+
desc = con.execute(f"DESCRIBE SELECT * FROM '{path}'").fetchall()
|
|
85
|
+
count_row = t.cast(tuple[int], con.execute(f"SELECT count(*) FROM '{path}'").fetchone())
|
|
86
|
+
columns = tuple(
|
|
87
|
+
ColumnDoc(
|
|
88
|
+
name=str(r[0]),
|
|
89
|
+
dtype=str(r[1]),
|
|
90
|
+
samples=_sample_values(con, path, str(r[0]), str(r[1])),
|
|
91
|
+
)
|
|
92
|
+
for r in desc
|
|
93
|
+
)
|
|
94
|
+
return SchemaDoc(table=table, path=path, row_count=int(count_row[0]), columns=columns)
|
|
95
|
+
except duckdb.Error as e:
|
|
96
|
+
raise FatalError(f"describe failed for {table} ({path}): {e}") from e
|
|
97
|
+
finally:
|
|
98
|
+
con.close()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build_catalog(sources: dict[str, str]) -> Catalog:
|
|
102
|
+
"""{table_name: parquet_path_or_glob} → Catalog. 테이블명 순으로 정렬해 결정적."""
|
|
103
|
+
docs = tuple(describe(name, sources[name]) for name in sorted(sources))
|
|
104
|
+
return Catalog(tables=docs)
|
augur/cli.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""augur CLI — auth / index / ask / eval / report. 러너 층: 여기서만 예외를 잡는다."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
import typing as t
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated
|
|
9
|
+
|
|
10
|
+
import duckdb
|
|
11
|
+
import typer
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from arsenal_core.errors import ArsenalError
|
|
15
|
+
from augur.catalog import Catalog, build_catalog
|
|
16
|
+
from augur.evaluate import load_cases, run_eval, summarize
|
|
17
|
+
from augur.generate import append_trace, ask
|
|
18
|
+
from augur.keys import (
|
|
19
|
+
clear_credential,
|
|
20
|
+
credentials_path,
|
|
21
|
+
key_source,
|
|
22
|
+
mask,
|
|
23
|
+
resolve_api_key,
|
|
24
|
+
save_credential,
|
|
25
|
+
)
|
|
26
|
+
from augur.llm import provider_from_name
|
|
27
|
+
|
|
28
|
+
app = typer.Typer(help="Augur — 자연어 → SQL (schema RAG) + eval")
|
|
29
|
+
auth_app = typer.Typer(help="API 키 설정 — 환경변수 → ~/.config/augur → 프롬프트 순으로 찾는다.")
|
|
30
|
+
app.add_typer(auth_app, name="auth")
|
|
31
|
+
DEFAULT_CATALOG = Path("./data/augur/catalog.json")
|
|
32
|
+
DEFAULT_TRACES = Path("./data/augur/traces.jsonl")
|
|
33
|
+
DEFAULT_EVAL_OUT = Path("./data/augur/eval")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _fail(e: Exception) -> t.NoReturn:
|
|
37
|
+
typer.echo(f"error: {e}", err=True)
|
|
38
|
+
raise typer.Exit(1) from e
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@app.command()
|
|
42
|
+
def index(
|
|
43
|
+
sources: Annotated[Path, typer.Argument(help="YAML: {tables: {name: './data/x/*.parquet'}}")],
|
|
44
|
+
catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Parquet 소스를 스캔해 스키마 카탈로그를 만든다."""
|
|
47
|
+
try:
|
|
48
|
+
data = t.cast(dict[str, t.Any], yaml.safe_load(sources.read_text(encoding="utf-8")))
|
|
49
|
+
cat = build_catalog({str(k): str(v) for k, v in data["tables"].items()})
|
|
50
|
+
cat.save(catalog)
|
|
51
|
+
except (ArsenalError, OSError, KeyError) as e:
|
|
52
|
+
_fail(e)
|
|
53
|
+
typer.echo(f"indexed {len(cat.tables)} tables → {catalog}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@app.command(name="ask")
|
|
57
|
+
def ask_cmd(
|
|
58
|
+
question: str,
|
|
59
|
+
catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
|
|
60
|
+
provider: str = typer.Option("anthropic", "--provider", help="anthropic | openai"),
|
|
61
|
+
model: Annotated[str | None, typer.Option("--model")] = None,
|
|
62
|
+
top_k: int = typer.Option(3, "--top-k"),
|
|
63
|
+
traces: Annotated[Path, typer.Option("--traces")] = DEFAULT_TRACES,
|
|
64
|
+
show_sql: bool = typer.Option(True, "--show-sql/--no-show-sql"),
|
|
65
|
+
) -> None:
|
|
66
|
+
"""질문 → SQL → 실행. 모든 호출은 traces.jsonl에 남는다."""
|
|
67
|
+
try:
|
|
68
|
+
cat = Catalog.load(catalog)
|
|
69
|
+
prov = provider_from_name(provider, model, resolve_api_key(provider))
|
|
70
|
+
trace, rows = ask(question, cat, prov, top_k=top_k)
|
|
71
|
+
append_trace(traces, trace)
|
|
72
|
+
except ArsenalError as e:
|
|
73
|
+
_fail(e)
|
|
74
|
+
if show_sql:
|
|
75
|
+
typer.echo(f"-- retrieved: {trace.retrieved_tables}\n{trace.sql}\n", err=True)
|
|
76
|
+
if trace.error:
|
|
77
|
+
typer.echo(f"error: {trace.error}", err=True)
|
|
78
|
+
raise typer.Exit(1)
|
|
79
|
+
for row in rows:
|
|
80
|
+
typer.echo(json.dumps([str(v) for v in row], ensure_ascii=False))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@app.command(name="eval")
|
|
84
|
+
def eval_cmd(
|
|
85
|
+
cases: Path,
|
|
86
|
+
catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
|
|
87
|
+
provider: str = typer.Option("anthropic", "--provider"),
|
|
88
|
+
model: Annotated[str | None, typer.Option("--model")] = None,
|
|
89
|
+
top_k: int = typer.Option(3, "--top-k"),
|
|
90
|
+
out: Annotated[Path, typer.Option("--out")] = DEFAULT_EVAL_OUT,
|
|
91
|
+
run_id: Annotated[str | None, typer.Option("--run-id")] = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
"""케이스 파일 전체를 실행하고 실패 모드별 집계를 출력한다."""
|
|
94
|
+
rid = run_id or time.strftime("%Y%m%dT%H%M%S")
|
|
95
|
+
try:
|
|
96
|
+
cat = Catalog.load(catalog)
|
|
97
|
+
prov = provider_from_name(provider, model, resolve_api_key(provider))
|
|
98
|
+
results = run_eval(load_cases(cases), cat, prov, top_k=top_k, run_id=rid, out_dir=out)
|
|
99
|
+
except ArsenalError as e:
|
|
100
|
+
_fail(e)
|
|
101
|
+
summary = summarize(results)
|
|
102
|
+
total = len(results)
|
|
103
|
+
typer.echo(f"run_id={rid} provider={provider} top_k={top_k} n={total}")
|
|
104
|
+
for mode, n in summary.items():
|
|
105
|
+
typer.echo(f" {mode:<20} {n:>4} {n / total:6.1%}")
|
|
106
|
+
typer.echo(f"→ {out / (rid + '.parquet')}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@app.command()
|
|
110
|
+
def report(out: Annotated[Path, typer.Option("--out")] = DEFAULT_EVAL_OUT) -> None:
|
|
111
|
+
"""모든 run의 실패 모드 추이 — 개선 전후 비교용."""
|
|
112
|
+
pattern = str(out / "*.parquet")
|
|
113
|
+
try:
|
|
114
|
+
duckdb.sql( # pyright: ignore[reportUnknownMemberType]
|
|
115
|
+
f"SELECT run_id, provider, failure_mode, count(*) AS n FROM '{pattern}' "
|
|
116
|
+
"GROUP BY 1,2,3 ORDER BY 1,3"
|
|
117
|
+
).show()
|
|
118
|
+
except duckdb.Error as e:
|
|
119
|
+
_fail(e)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@auth_app.command(name="set")
|
|
123
|
+
def auth_set(provider: Annotated[str, typer.Argument(help="anthropic | openai")]) -> None:
|
|
124
|
+
"""키를 숨김 입력으로 받아 사용자 설정 파일(0600)에 저장한다. 인자로는 받지 않는다."""
|
|
125
|
+
api_key = str(typer.prompt(f"{provider} API key", hide_input=True))
|
|
126
|
+
try:
|
|
127
|
+
path = save_credential(provider, api_key)
|
|
128
|
+
except ArsenalError as e:
|
|
129
|
+
_fail(e)
|
|
130
|
+
typer.echo(f"saved {provider} key → {path}")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@auth_app.command(name="status")
|
|
134
|
+
def auth_status() -> None:
|
|
135
|
+
"""프로바이더별로 키가 어디서 오는지(env | file | none) 보여준다. 값은 앞 4자리만."""
|
|
136
|
+
for name in ("anthropic", "openai"):
|
|
137
|
+
found = key_source(name)
|
|
138
|
+
if found is None:
|
|
139
|
+
typer.echo(f"{name:<10} none")
|
|
140
|
+
else:
|
|
141
|
+
typer.echo(f"{name:<10} {found[0]:<5} {mask(found[1])}")
|
|
142
|
+
typer.echo(f"file: {credentials_path()}")
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@auth_app.command(name="clear")
|
|
146
|
+
def auth_clear(provider: Annotated[str, typer.Argument(help="anthropic | openai")]) -> None:
|
|
147
|
+
"""설정 파일에서 키를 지운다 (환경변수는 건드리지 않는다)."""
|
|
148
|
+
try:
|
|
149
|
+
removed = clear_credential(provider)
|
|
150
|
+
except ArsenalError as e:
|
|
151
|
+
_fail(e)
|
|
152
|
+
typer.echo(f"{'cleared' if removed else 'nothing stored for'} {provider}")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == "__main__":
|
|
156
|
+
sys.exit(app())
|
augur/evaluate.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Eval 하네스 — 골든 SQL과 생성 SQL의 실행 결과를 대조하고 실패 모드를 자동 분류한다.
|
|
2
|
+
|
|
3
|
+
실패 모드는 파이프라인 단계 순서로 판정한다(앞 단계 실패가 뒷 단계 실패의 원인이므로):
|
|
4
|
+
RETRIEVAL_MISS 골든이 쓰는 테이블이 검색 결과에 없음 → retrieve.py 문제
|
|
5
|
+
NON_SELECT LLM이 SELECT 아닌 것을 냄 → 프롬프트/모델 문제
|
|
6
|
+
UNANSWERABLE 모델이 답 불가를 선언(정답이 answerable이면 오류)
|
|
7
|
+
HALLUCINATED_COLUMN 카탈로그에 없는 컬럼 참조 → 스키마 렌더/모델 문제
|
|
8
|
+
HALLUCINATED_TABLE 카탈로그에 없는 경로/테이블 참조
|
|
9
|
+
EXEC_ERROR 위 어느 것도 아닌데 DuckDB 실행 실패(문법 등)
|
|
10
|
+
WRONG_SHAPE 실행됐지만 컬럼 수/행 수가 골든과 다름 → 질문 해석 오류(집계 축·필터)
|
|
11
|
+
WRONG_RESULT 모양은 같은데 값이 다름 → 의미 오류(조건·리터럴)
|
|
12
|
+
CORRECT
|
|
13
|
+
분류가 자동이어야 50건 이상을 반복 측정할 수 있다. 수동 라벨은 notes 컬럼에.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import typing as t
|
|
19
|
+
from dataclasses import asdict, dataclass
|
|
20
|
+
from decimal import Decimal
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import duckdb
|
|
24
|
+
import pyarrow as pa
|
|
25
|
+
import pyarrow.parquet as pq
|
|
26
|
+
import yaml
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
from arsenal_core.errors import FatalError
|
|
30
|
+
from augur.catalog import Catalog
|
|
31
|
+
from augur.generate import Trace, ask, execute
|
|
32
|
+
from augur.llm import Provider
|
|
33
|
+
|
|
34
|
+
FAILURE_MODES = (
|
|
35
|
+
"RETRIEVAL_MISS",
|
|
36
|
+
"NON_SELECT",
|
|
37
|
+
"UNANSWERABLE",
|
|
38
|
+
"HALLUCINATED_COLUMN",
|
|
39
|
+
"HALLUCINATED_TABLE",
|
|
40
|
+
"EXEC_ERROR",
|
|
41
|
+
"WRONG_RESULT",
|
|
42
|
+
"WRONG_SHAPE",
|
|
43
|
+
"CORRECT",
|
|
44
|
+
)
|
|
45
|
+
PATH_RE = re.compile(r"'([^']+\.parquet)'")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class EvalCase(BaseModel):
|
|
49
|
+
model_config = ConfigDict(frozen=True)
|
|
50
|
+
id: str
|
|
51
|
+
question: str
|
|
52
|
+
golden_sql: str
|
|
53
|
+
tables: tuple[str, ...] # 골든이 필요로 하는 카탈로그 테이블명
|
|
54
|
+
answerable: bool = True
|
|
55
|
+
notes: str = ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def load_cases(path: Path) -> tuple[EvalCase, ...]:
|
|
59
|
+
if not path.exists():
|
|
60
|
+
raise FatalError(f"cases file not found: {path}")
|
|
61
|
+
data = t.cast(dict[str, t.Any], yaml.safe_load(path.read_text(encoding="utf-8")))
|
|
62
|
+
cases = tuple(EvalCase.model_validate(c) for c in data.get("cases", []))
|
|
63
|
+
ids = [c.id for c in cases]
|
|
64
|
+
if len(ids) != len(set(ids)):
|
|
65
|
+
raise FatalError("duplicate case ids")
|
|
66
|
+
return cases
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _normalize(v: t.Any) -> t.Any:
|
|
70
|
+
if isinstance(v, Decimal):
|
|
71
|
+
return float(v)
|
|
72
|
+
if isinstance(v, float):
|
|
73
|
+
return round(v, 6)
|
|
74
|
+
return v
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def rows_equal(a: list[tuple[t.Any, ...]], b: list[tuple[t.Any, ...]]) -> bool:
|
|
78
|
+
"""순서 무시, 값 정규화 후 비교. ORDER BY가 질문의 일부인 케이스는 notes로 표시하고
|
|
79
|
+
골든에 순서를 넣되, 여기서는 집합 비교만 한다(P0 한계)."""
|
|
80
|
+
na = sorted((tuple(_normalize(x) for x in r) for r in a), key=repr)
|
|
81
|
+
nb = sorted((tuple(_normalize(x) for x in r) for r in b), key=repr)
|
|
82
|
+
return na == nb
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def classify(
|
|
86
|
+
case: EvalCase,
|
|
87
|
+
trace: Trace,
|
|
88
|
+
catalog: Catalog,
|
|
89
|
+
golden_rows: list[tuple[t.Any, ...]],
|
|
90
|
+
generated_rows: list[tuple[t.Any, ...]],
|
|
91
|
+
) -> str:
|
|
92
|
+
missing = [tb for tb in case.tables if tb not in trace.retrieved_tables]
|
|
93
|
+
if missing:
|
|
94
|
+
return "RETRIEVAL_MISS"
|
|
95
|
+
if trace.error and trace.error.startswith("NON_SELECT"):
|
|
96
|
+
return "NON_SELECT"
|
|
97
|
+
if "UNANSWERABLE" in trace.sql.upper():
|
|
98
|
+
return "CORRECT" if not case.answerable else "UNANSWERABLE"
|
|
99
|
+
known_paths = {d.path for d in catalog.tables}
|
|
100
|
+
if any(p not in known_paths for p in PATH_RE.findall(trace.sql)):
|
|
101
|
+
return "HALLUCINATED_TABLE"
|
|
102
|
+
if not trace.executed:
|
|
103
|
+
if trace.error and "not found in FROM clause" in trace.error:
|
|
104
|
+
return "HALLUCINATED_COLUMN"
|
|
105
|
+
return "EXEC_ERROR"
|
|
106
|
+
if not case.answerable:
|
|
107
|
+
return "WRONG_RESULT" # 답 못 해야 하는데 답함
|
|
108
|
+
if rows_equal(golden_rows, generated_rows):
|
|
109
|
+
return "CORRECT"
|
|
110
|
+
same_shape = len(golden_rows) == len(generated_rows) and (
|
|
111
|
+
not golden_rows or len(golden_rows[0]) == len(generated_rows[0])
|
|
112
|
+
)
|
|
113
|
+
return "WRONG_RESULT" if same_shape else "WRONG_SHAPE"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(frozen=True)
|
|
117
|
+
class EvalResult:
|
|
118
|
+
case_id: str
|
|
119
|
+
question: str
|
|
120
|
+
failure_mode: str
|
|
121
|
+
retrieved_tables: str
|
|
122
|
+
golden_sql: str
|
|
123
|
+
generated_sql: str
|
|
124
|
+
error: str | None
|
|
125
|
+
repair_attempts: int
|
|
126
|
+
latency_ms: int
|
|
127
|
+
input_tokens: int
|
|
128
|
+
output_tokens: int
|
|
129
|
+
provider: str
|
|
130
|
+
model: str
|
|
131
|
+
notes: str
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def run_case(
|
|
135
|
+
case: EvalCase, catalog: Catalog, provider: Provider, *, top_k: int
|
|
136
|
+
) -> tuple[EvalResult, Trace]:
|
|
137
|
+
try:
|
|
138
|
+
_, golden_rows = execute(case.golden_sql)
|
|
139
|
+
except duckdb.Error as e:
|
|
140
|
+
raise FatalError(f"golden SQL failed for {case.id}: {e}") from e
|
|
141
|
+
trace, generated_rows = ask(case.question, catalog, provider, top_k=top_k)
|
|
142
|
+
mode = classify(case, trace, catalog, golden_rows, generated_rows)
|
|
143
|
+
result = EvalResult(
|
|
144
|
+
case.id,
|
|
145
|
+
case.question,
|
|
146
|
+
mode,
|
|
147
|
+
",".join(trace.retrieved_tables),
|
|
148
|
+
case.golden_sql,
|
|
149
|
+
trace.sql,
|
|
150
|
+
trace.error,
|
|
151
|
+
trace.repair_attempts,
|
|
152
|
+
trace.latency_ms,
|
|
153
|
+
trace.input_tokens,
|
|
154
|
+
trace.output_tokens,
|
|
155
|
+
trace.provider,
|
|
156
|
+
trace.model,
|
|
157
|
+
case.notes,
|
|
158
|
+
)
|
|
159
|
+
return result, trace
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def run_eval(
|
|
163
|
+
cases: tuple[EvalCase, ...],
|
|
164
|
+
catalog: Catalog,
|
|
165
|
+
provider: Provider,
|
|
166
|
+
*,
|
|
167
|
+
top_k: int = 3,
|
|
168
|
+
run_id: str,
|
|
169
|
+
out_dir: Path,
|
|
170
|
+
) -> list[EvalResult]:
|
|
171
|
+
"""결과는 out_dir/run_id.parquet + traces/run_id.jsonl. Parquet으로 남기는 이유:
|
|
172
|
+
`gladius query`로 바로 집계 — 도구가 도구를 평가한다(dogfooding)."""
|
|
173
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
trace_path = out_dir / "traces" / f"{run_id}.jsonl"
|
|
175
|
+
trace_path.parent.mkdir(parents=True, exist_ok=True)
|
|
176
|
+
results: list[EvalResult] = []
|
|
177
|
+
with trace_path.open("w", encoding="utf-8") as tf:
|
|
178
|
+
for case in cases:
|
|
179
|
+
result, trace = run_case(case, catalog, provider, top_k=top_k)
|
|
180
|
+
results.append(result)
|
|
181
|
+
tf.write(
|
|
182
|
+
json.dumps(
|
|
183
|
+
{"run_id": run_id, "case_id": case.id, **asdict(trace)}, ensure_ascii=False
|
|
184
|
+
)
|
|
185
|
+
+ "\n"
|
|
186
|
+
)
|
|
187
|
+
table = pa.Table.from_pylist([{"run_id": run_id, **asdict(r)} for r in results])
|
|
188
|
+
pq.write_table( # pyright: ignore[reportUnknownMemberType]
|
|
189
|
+
table, out_dir / f"{run_id}.parquet", compression="zstd"
|
|
190
|
+
)
|
|
191
|
+
return results
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def summarize(results: list[EvalResult]) -> dict[str, int]:
|
|
195
|
+
counts = {m: 0 for m in FAILURE_MODES}
|
|
196
|
+
for r in results:
|
|
197
|
+
counts[r.failure_mode] += 1
|
|
198
|
+
return {k: v for k, v in counts.items() if v or k == "CORRECT"}
|
augur/fixture.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""결정적 합성 데이터 — eval 케이스의 골든 SQL이 항상 같은 결과를 내도록 seed 고정.
|
|
2
|
+
|
|
3
|
+
생성: customers / products / orders 세 테이블 Parquet. 스크립트 진입점은
|
|
4
|
+
`packages/augur/eval/make_fixture.py`(이 모듈을 호출), 테스트 fixture도 같은 함수를 쓴다.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import random
|
|
8
|
+
from datetime import date, timedelta
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import pyarrow as pa
|
|
12
|
+
import pyarrow.parquet as pq
|
|
13
|
+
|
|
14
|
+
SEED = 20260908
|
|
15
|
+
N_CUSTOMERS = 40
|
|
16
|
+
N_ORDERS = 400
|
|
17
|
+
CITIES = ["Seoul", "Busan", "Incheon", "Daegu", "Daejeon"]
|
|
18
|
+
SEGMENTS = ["consumer", "smb", "enterprise"]
|
|
19
|
+
STATUSES = ["paid", "pending", "refunded", "cancelled"]
|
|
20
|
+
PRODUCTS: list[tuple[int, str, str, float]] = [
|
|
21
|
+
(1, "Keyboard", "hardware", 49.0),
|
|
22
|
+
(2, "Monitor", "hardware", 199.0),
|
|
23
|
+
(3, "Mouse", "hardware", 19.0),
|
|
24
|
+
(4, "IDE License", "software", 89.0),
|
|
25
|
+
(5, "Cloud Credits", "software", 120.0),
|
|
26
|
+
(6, "Headset", "hardware", 59.0),
|
|
27
|
+
(7, "VPN Subscription", "software", 9.0),
|
|
28
|
+
]
|
|
29
|
+
TABLE_NAMES = ("customers", "products", "orders")
|
|
30
|
+
|
|
31
|
+
Row = dict[str, object]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def make(out: Path) -> None:
|
|
35
|
+
rng = random.Random(SEED)
|
|
36
|
+
customers: list[Row] = [
|
|
37
|
+
{
|
|
38
|
+
"customer_id": i,
|
|
39
|
+
"name": f"cust_{i:03d}",
|
|
40
|
+
"city": rng.choice(CITIES),
|
|
41
|
+
"segment": rng.choice(SEGMENTS),
|
|
42
|
+
"signup_date": date(2025, 1, 1) + timedelta(days=rng.randint(0, 400)),
|
|
43
|
+
}
|
|
44
|
+
for i in range(1, N_CUSTOMERS + 1)
|
|
45
|
+
]
|
|
46
|
+
products: list[Row] = [
|
|
47
|
+
{"product_id": p[0], "product_name": p[1], "category": p[2], "price": p[3]}
|
|
48
|
+
for p in PRODUCTS
|
|
49
|
+
]
|
|
50
|
+
orders: list[Row] = []
|
|
51
|
+
for i in range(1, N_ORDERS + 1):
|
|
52
|
+
p = rng.choice(PRODUCTS)
|
|
53
|
+
qty = rng.randint(1, 5)
|
|
54
|
+
orders.append(
|
|
55
|
+
{
|
|
56
|
+
"order_id": i,
|
|
57
|
+
"customer_id": rng.randint(1, N_CUSTOMERS),
|
|
58
|
+
"product_id": p[0],
|
|
59
|
+
"quantity": qty,
|
|
60
|
+
"amount": round(qty * p[3], 2),
|
|
61
|
+
"status": rng.choice(STATUSES),
|
|
62
|
+
"ordered_at": date(2026, 1, 1) + timedelta(days=rng.randint(0, 240)),
|
|
63
|
+
}
|
|
64
|
+
)
|
|
65
|
+
for name, rows in (("customers", customers), ("products", products), ("orders", orders)):
|
|
66
|
+
d = out / name
|
|
67
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
pq.write_table( # pyright: ignore[reportUnknownMemberType]
|
|
69
|
+
pa.Table.from_pylist(rows), d / "part-0.parquet"
|
|
70
|
+
)
|