de-augur 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
augur/__init__.py ADDED
@@ -0,0 +1,14 @@
1
+ """Augur — 자연어 → SQL (스키마 RAG) + eval 하네스.
2
+
3
+ 파이프라인: catalog(스키마 색인) → retrieve(관련 테이블 선별) → generate(LLM → SQL)
4
+ → execute(DuckDB) → trace(JSONL). evaluate는 이 트레이스를 골든 SQL과 대조해
5
+ 실패 모드를 자동 분류한다. 엔진은 빌린다 — LLM은 HTTP로 호출, 검색은 어휘 매칭.
6
+ 벡터 DB 없음. 필요해지면 retrieve.py의 Scorer 하나만 바꾼다.
7
+ """
8
+
9
+ from importlib.metadata import PackageNotFoundError, version
10
+
11
+ try:
12
+ __version__ = version("de-augur")
13
+ except PackageNotFoundError:
14
+ __version__ = "0.4.0"
augur/catalog.py ADDED
@@ -0,0 +1,104 @@
1
+ """스키마 카탈로그 — Parquet 레이크에서 테이블/컬럼/샘플값을 뽑아 검색 단위로 만든다.
2
+
3
+ RAG의 "R"은 여기서 만든 SchemaDoc을 대상으로 한다. 샘플값을 넣는 이유: LLM이
4
+ `status = 'PAID'`처럼 존재하지 않는 리터럴을 지어내는 실패(HALLUCINATED_LITERAL)를
5
+ 줄이기 위해. 카탈로그는 JSON으로 저장되며 질문마다 재스캔하지 않는다.
6
+ """
7
+
8
+ import json
9
+ import typing as t
10
+ from pathlib import Path
11
+
12
+ import duckdb
13
+ from pydantic import BaseModel, ConfigDict
14
+
15
+ from arsenal_core.errors import FatalError
16
+
17
+ MAX_SAMPLE_VALUES = 5
18
+ SAMPLE_ROW_LIMIT = 10_000
19
+ TEXT_TYPES = ("VARCHAR", "TEXT", "STRING")
20
+
21
+
22
+ class ColumnDoc(BaseModel):
23
+ model_config = ConfigDict(frozen=True)
24
+ name: str
25
+ dtype: str
26
+ samples: tuple[str, ...] = ()
27
+
28
+
29
+ class SchemaDoc(BaseModel):
30
+ model_config = ConfigDict(frozen=True)
31
+ table: str
32
+ path: str # DuckDB에서 FROM에 쓸 경로 (glob 허용)
33
+ row_count: int
34
+ columns: tuple[ColumnDoc, ...]
35
+
36
+ def render(self) -> str:
37
+ """프롬프트에 들어가는 형태. 짧고 결정적이어야 캐시·diff가 가능하다."""
38
+ cols = ", ".join(
39
+ f"{c.name} {c.dtype}" + (f" e.g. {list(c.samples)}" if c.samples else "")
40
+ for c in self.columns
41
+ )
42
+ return f"TABLE {self.table} (source: '{self.path}', rows={self.row_count})\n {cols}"
43
+
44
+ def column_names(self) -> frozenset[str]:
45
+ return frozenset(c.name.lower() for c in self.columns)
46
+
47
+
48
+ class Catalog(BaseModel):
49
+ model_config = ConfigDict(frozen=True)
50
+ tables: tuple[SchemaDoc, ...]
51
+
52
+ def get(self, table: str) -> SchemaDoc | None:
53
+ return next((d for d in self.tables if d.table == table), None)
54
+
55
+ def save(self, path: Path) -> None:
56
+ path.parent.mkdir(parents=True, exist_ok=True)
57
+ path.write_text(self.model_dump_json(indent=2), encoding="utf-8")
58
+
59
+ @classmethod
60
+ def load(cls, path: Path) -> "Catalog":
61
+ if not path.exists():
62
+ raise FatalError(f"catalog not found: {path} (run `augur index` first)")
63
+ return cls.model_validate(json.loads(path.read_text(encoding="utf-8")))
64
+
65
+
66
+ def _sample_values(
67
+ con: duckdb.DuckDBPyConnection, path: str, column: str, dtype: str
68
+ ) -> tuple[str, ...]:
69
+ if not any(tt in dtype.upper() for tt in TEXT_TYPES):
70
+ return ()
71
+ sql = (
72
+ f'SELECT "{column}" AS v, count(*) AS n '
73
+ f"FROM (SELECT * FROM '{path}' LIMIT {SAMPLE_ROW_LIMIT}) "
74
+ f'WHERE "{column}" IS NOT NULL GROUP BY 1 ORDER BY n DESC LIMIT {MAX_SAMPLE_VALUES}'
75
+ )
76
+ rows = t.cast(list[tuple[t.Any, int]], con.execute(sql).fetchall())
77
+ return tuple(str(r[0]) for r in rows)
78
+
79
+
80
+ def describe(table: str, path: str) -> SchemaDoc:
81
+ """Parquet 경로 하나를 SchemaDoc으로. DuckDB 에러는 FatalError로 변환."""
82
+ con = duckdb.connect()
83
+ try:
84
+ desc = con.execute(f"DESCRIBE SELECT * FROM '{path}'").fetchall()
85
+ count_row = t.cast(tuple[int], con.execute(f"SELECT count(*) FROM '{path}'").fetchone())
86
+ columns = tuple(
87
+ ColumnDoc(
88
+ name=str(r[0]),
89
+ dtype=str(r[1]),
90
+ samples=_sample_values(con, path, str(r[0]), str(r[1])),
91
+ )
92
+ for r in desc
93
+ )
94
+ return SchemaDoc(table=table, path=path, row_count=int(count_row[0]), columns=columns)
95
+ except duckdb.Error as e:
96
+ raise FatalError(f"describe failed for {table} ({path}): {e}") from e
97
+ finally:
98
+ con.close()
99
+
100
+
101
+ def build_catalog(sources: dict[str, str]) -> Catalog:
102
+ """{table_name: parquet_path_or_glob} → Catalog. 테이블명 순으로 정렬해 결정적."""
103
+ docs = tuple(describe(name, sources[name]) for name in sorted(sources))
104
+ return Catalog(tables=docs)
augur/cli.py ADDED
@@ -0,0 +1,156 @@
1
+ """augur CLI — auth / index / ask / eval / report. 러너 층: 여기서만 예외를 잡는다."""
2
+
3
+ import json
4
+ import sys
5
+ import time
6
+ import typing as t
7
+ from pathlib import Path
8
+ from typing import Annotated
9
+
10
+ import duckdb
11
+ import typer
12
+ import yaml
13
+
14
+ from arsenal_core.errors import ArsenalError
15
+ from augur.catalog import Catalog, build_catalog
16
+ from augur.evaluate import load_cases, run_eval, summarize
17
+ from augur.generate import append_trace, ask
18
+ from augur.keys import (
19
+ clear_credential,
20
+ credentials_path,
21
+ key_source,
22
+ mask,
23
+ resolve_api_key,
24
+ save_credential,
25
+ )
26
+ from augur.llm import provider_from_name
27
+
28
+ app = typer.Typer(help="Augur — 자연어 → SQL (schema RAG) + eval")
29
+ auth_app = typer.Typer(help="API 키 설정 — 환경변수 → ~/.config/augur → 프롬프트 순으로 찾는다.")
30
+ app.add_typer(auth_app, name="auth")
31
+ DEFAULT_CATALOG = Path("./data/augur/catalog.json")
32
+ DEFAULT_TRACES = Path("./data/augur/traces.jsonl")
33
+ DEFAULT_EVAL_OUT = Path("./data/augur/eval")
34
+
35
+
36
+ def _fail(e: Exception) -> t.NoReturn:
37
+ typer.echo(f"error: {e}", err=True)
38
+ raise typer.Exit(1) from e
39
+
40
+
41
+ @app.command()
42
+ def index(
43
+ sources: Annotated[Path, typer.Argument(help="YAML: {tables: {name: './data/x/*.parquet'}}")],
44
+ catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
45
+ ) -> None:
46
+ """Parquet 소스를 스캔해 스키마 카탈로그를 만든다."""
47
+ try:
48
+ data = t.cast(dict[str, t.Any], yaml.safe_load(sources.read_text(encoding="utf-8")))
49
+ cat = build_catalog({str(k): str(v) for k, v in data["tables"].items()})
50
+ cat.save(catalog)
51
+ except (ArsenalError, OSError, KeyError) as e:
52
+ _fail(e)
53
+ typer.echo(f"indexed {len(cat.tables)} tables → {catalog}")
54
+
55
+
56
+ @app.command(name="ask")
57
+ def ask_cmd(
58
+ question: str,
59
+ catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
60
+ provider: str = typer.Option("anthropic", "--provider", help="anthropic | openai"),
61
+ model: Annotated[str | None, typer.Option("--model")] = None,
62
+ top_k: int = typer.Option(3, "--top-k"),
63
+ traces: Annotated[Path, typer.Option("--traces")] = DEFAULT_TRACES,
64
+ show_sql: bool = typer.Option(True, "--show-sql/--no-show-sql"),
65
+ ) -> None:
66
+ """질문 → SQL → 실행. 모든 호출은 traces.jsonl에 남는다."""
67
+ try:
68
+ cat = Catalog.load(catalog)
69
+ prov = provider_from_name(provider, model, resolve_api_key(provider))
70
+ trace, rows = ask(question, cat, prov, top_k=top_k)
71
+ append_trace(traces, trace)
72
+ except ArsenalError as e:
73
+ _fail(e)
74
+ if show_sql:
75
+ typer.echo(f"-- retrieved: {trace.retrieved_tables}\n{trace.sql}\n", err=True)
76
+ if trace.error:
77
+ typer.echo(f"error: {trace.error}", err=True)
78
+ raise typer.Exit(1)
79
+ for row in rows:
80
+ typer.echo(json.dumps([str(v) for v in row], ensure_ascii=False))
81
+
82
+
83
+ @app.command(name="eval")
84
+ def eval_cmd(
85
+ cases: Path,
86
+ catalog: Annotated[Path, typer.Option("--catalog")] = DEFAULT_CATALOG,
87
+ provider: str = typer.Option("anthropic", "--provider"),
88
+ model: Annotated[str | None, typer.Option("--model")] = None,
89
+ top_k: int = typer.Option(3, "--top-k"),
90
+ out: Annotated[Path, typer.Option("--out")] = DEFAULT_EVAL_OUT,
91
+ run_id: Annotated[str | None, typer.Option("--run-id")] = None,
92
+ ) -> None:
93
+ """케이스 파일 전체를 실행하고 실패 모드별 집계를 출력한다."""
94
+ rid = run_id or time.strftime("%Y%m%dT%H%M%S")
95
+ try:
96
+ cat = Catalog.load(catalog)
97
+ prov = provider_from_name(provider, model, resolve_api_key(provider))
98
+ results = run_eval(load_cases(cases), cat, prov, top_k=top_k, run_id=rid, out_dir=out)
99
+ except ArsenalError as e:
100
+ _fail(e)
101
+ summary = summarize(results)
102
+ total = len(results)
103
+ typer.echo(f"run_id={rid} provider={provider} top_k={top_k} n={total}")
104
+ for mode, n in summary.items():
105
+ typer.echo(f" {mode:<20} {n:>4} {n / total:6.1%}")
106
+ typer.echo(f"→ {out / (rid + '.parquet')}")
107
+
108
+
109
+ @app.command()
110
+ def report(out: Annotated[Path, typer.Option("--out")] = DEFAULT_EVAL_OUT) -> None:
111
+ """모든 run의 실패 모드 추이 — 개선 전후 비교용."""
112
+ pattern = str(out / "*.parquet")
113
+ try:
114
+ duckdb.sql( # pyright: ignore[reportUnknownMemberType]
115
+ f"SELECT run_id, provider, failure_mode, count(*) AS n FROM '{pattern}' "
116
+ "GROUP BY 1,2,3 ORDER BY 1,3"
117
+ ).show()
118
+ except duckdb.Error as e:
119
+ _fail(e)
120
+
121
+
122
+ @auth_app.command(name="set")
123
+ def auth_set(provider: Annotated[str, typer.Argument(help="anthropic | openai")]) -> None:
124
+ """키를 숨김 입력으로 받아 사용자 설정 파일(0600)에 저장한다. 인자로는 받지 않는다."""
125
+ api_key = str(typer.prompt(f"{provider} API key", hide_input=True))
126
+ try:
127
+ path = save_credential(provider, api_key)
128
+ except ArsenalError as e:
129
+ _fail(e)
130
+ typer.echo(f"saved {provider} key → {path}")
131
+
132
+
133
+ @auth_app.command(name="status")
134
+ def auth_status() -> None:
135
+ """프로바이더별로 키가 어디서 오는지(env | file | none) 보여준다. 값은 앞 4자리만."""
136
+ for name in ("anthropic", "openai"):
137
+ found = key_source(name)
138
+ if found is None:
139
+ typer.echo(f"{name:<10} none")
140
+ else:
141
+ typer.echo(f"{name:<10} {found[0]:<5} {mask(found[1])}")
142
+ typer.echo(f"file: {credentials_path()}")
143
+
144
+
145
+ @auth_app.command(name="clear")
146
+ def auth_clear(provider: Annotated[str, typer.Argument(help="anthropic | openai")]) -> None:
147
+ """설정 파일에서 키를 지운다 (환경변수는 건드리지 않는다)."""
148
+ try:
149
+ removed = clear_credential(provider)
150
+ except ArsenalError as e:
151
+ _fail(e)
152
+ typer.echo(f"{'cleared' if removed else 'nothing stored for'} {provider}")
153
+
154
+
155
+ if __name__ == "__main__":
156
+ sys.exit(app())
augur/evaluate.py ADDED
@@ -0,0 +1,198 @@
1
+ """Eval 하네스 — 골든 SQL과 생성 SQL의 실행 결과를 대조하고 실패 모드를 자동 분류한다.
2
+
3
+ 실패 모드는 파이프라인 단계 순서로 판정한다(앞 단계 실패가 뒷 단계 실패의 원인이므로):
4
+ RETRIEVAL_MISS 골든이 쓰는 테이블이 검색 결과에 없음 → retrieve.py 문제
5
+ NON_SELECT LLM이 SELECT 아닌 것을 냄 → 프롬프트/모델 문제
6
+ UNANSWERABLE 모델이 답 불가를 선언(정답이 answerable이면 오류)
7
+ HALLUCINATED_COLUMN 카탈로그에 없는 컬럼 참조 → 스키마 렌더/모델 문제
8
+ HALLUCINATED_TABLE 카탈로그에 없는 경로/테이블 참조
9
+ EXEC_ERROR 위 어느 것도 아닌데 DuckDB 실행 실패(문법 등)
10
+ WRONG_SHAPE 실행됐지만 컬럼 수/행 수가 골든과 다름 → 질문 해석 오류(집계 축·필터)
11
+ WRONG_RESULT 모양은 같은데 값이 다름 → 의미 오류(조건·리터럴)
12
+ CORRECT
13
+ 분류가 자동이어야 50건 이상을 반복 측정할 수 있다. 수동 라벨은 notes 컬럼에.
14
+ """
15
+
16
+ import json
17
+ import re
18
+ import typing as t
19
+ from dataclasses import asdict, dataclass
20
+ from decimal import Decimal
21
+ from pathlib import Path
22
+
23
+ import duckdb
24
+ import pyarrow as pa
25
+ import pyarrow.parquet as pq
26
+ import yaml
27
+ from pydantic import BaseModel, ConfigDict
28
+
29
+ from arsenal_core.errors import FatalError
30
+ from augur.catalog import Catalog
31
+ from augur.generate import Trace, ask, execute
32
+ from augur.llm import Provider
33
+
34
+ FAILURE_MODES = (
35
+ "RETRIEVAL_MISS",
36
+ "NON_SELECT",
37
+ "UNANSWERABLE",
38
+ "HALLUCINATED_COLUMN",
39
+ "HALLUCINATED_TABLE",
40
+ "EXEC_ERROR",
41
+ "WRONG_RESULT",
42
+ "WRONG_SHAPE",
43
+ "CORRECT",
44
+ )
45
+ PATH_RE = re.compile(r"'([^']+\.parquet)'")
46
+
47
+
48
+ class EvalCase(BaseModel):
49
+ model_config = ConfigDict(frozen=True)
50
+ id: str
51
+ question: str
52
+ golden_sql: str
53
+ tables: tuple[str, ...] # 골든이 필요로 하는 카탈로그 테이블명
54
+ answerable: bool = True
55
+ notes: str = ""
56
+
57
+
58
+ def load_cases(path: Path) -> tuple[EvalCase, ...]:
59
+ if not path.exists():
60
+ raise FatalError(f"cases file not found: {path}")
61
+ data = t.cast(dict[str, t.Any], yaml.safe_load(path.read_text(encoding="utf-8")))
62
+ cases = tuple(EvalCase.model_validate(c) for c in data.get("cases", []))
63
+ ids = [c.id for c in cases]
64
+ if len(ids) != len(set(ids)):
65
+ raise FatalError("duplicate case ids")
66
+ return cases
67
+
68
+
69
+ def _normalize(v: t.Any) -> t.Any:
70
+ if isinstance(v, Decimal):
71
+ return float(v)
72
+ if isinstance(v, float):
73
+ return round(v, 6)
74
+ return v
75
+
76
+
77
+ def rows_equal(a: list[tuple[t.Any, ...]], b: list[tuple[t.Any, ...]]) -> bool:
78
+ """순서 무시, 값 정규화 후 비교. ORDER BY가 질문의 일부인 케이스는 notes로 표시하고
79
+ 골든에 순서를 넣되, 여기서는 집합 비교만 한다(P0 한계)."""
80
+ na = sorted((tuple(_normalize(x) for x in r) for r in a), key=repr)
81
+ nb = sorted((tuple(_normalize(x) for x in r) for r in b), key=repr)
82
+ return na == nb
83
+
84
+
85
+ def classify(
86
+ case: EvalCase,
87
+ trace: Trace,
88
+ catalog: Catalog,
89
+ golden_rows: list[tuple[t.Any, ...]],
90
+ generated_rows: list[tuple[t.Any, ...]],
91
+ ) -> str:
92
+ missing = [tb for tb in case.tables if tb not in trace.retrieved_tables]
93
+ if missing:
94
+ return "RETRIEVAL_MISS"
95
+ if trace.error and trace.error.startswith("NON_SELECT"):
96
+ return "NON_SELECT"
97
+ if "UNANSWERABLE" in trace.sql.upper():
98
+ return "CORRECT" if not case.answerable else "UNANSWERABLE"
99
+ known_paths = {d.path for d in catalog.tables}
100
+ if any(p not in known_paths for p in PATH_RE.findall(trace.sql)):
101
+ return "HALLUCINATED_TABLE"
102
+ if not trace.executed:
103
+ if trace.error and "not found in FROM clause" in trace.error:
104
+ return "HALLUCINATED_COLUMN"
105
+ return "EXEC_ERROR"
106
+ if not case.answerable:
107
+ return "WRONG_RESULT" # 답 못 해야 하는데 답함
108
+ if rows_equal(golden_rows, generated_rows):
109
+ return "CORRECT"
110
+ same_shape = len(golden_rows) == len(generated_rows) and (
111
+ not golden_rows or len(golden_rows[0]) == len(generated_rows[0])
112
+ )
113
+ return "WRONG_RESULT" if same_shape else "WRONG_SHAPE"
114
+
115
+
116
+ @dataclass(frozen=True)
117
+ class EvalResult:
118
+ case_id: str
119
+ question: str
120
+ failure_mode: str
121
+ retrieved_tables: str
122
+ golden_sql: str
123
+ generated_sql: str
124
+ error: str | None
125
+ repair_attempts: int
126
+ latency_ms: int
127
+ input_tokens: int
128
+ output_tokens: int
129
+ provider: str
130
+ model: str
131
+ notes: str
132
+
133
+
134
+ def run_case(
135
+ case: EvalCase, catalog: Catalog, provider: Provider, *, top_k: int
136
+ ) -> tuple[EvalResult, Trace]:
137
+ try:
138
+ _, golden_rows = execute(case.golden_sql)
139
+ except duckdb.Error as e:
140
+ raise FatalError(f"golden SQL failed for {case.id}: {e}") from e
141
+ trace, generated_rows = ask(case.question, catalog, provider, top_k=top_k)
142
+ mode = classify(case, trace, catalog, golden_rows, generated_rows)
143
+ result = EvalResult(
144
+ case.id,
145
+ case.question,
146
+ mode,
147
+ ",".join(trace.retrieved_tables),
148
+ case.golden_sql,
149
+ trace.sql,
150
+ trace.error,
151
+ trace.repair_attempts,
152
+ trace.latency_ms,
153
+ trace.input_tokens,
154
+ trace.output_tokens,
155
+ trace.provider,
156
+ trace.model,
157
+ case.notes,
158
+ )
159
+ return result, trace
160
+
161
+
162
+ def run_eval(
163
+ cases: tuple[EvalCase, ...],
164
+ catalog: Catalog,
165
+ provider: Provider,
166
+ *,
167
+ top_k: int = 3,
168
+ run_id: str,
169
+ out_dir: Path,
170
+ ) -> list[EvalResult]:
171
+ """결과는 out_dir/run_id.parquet + traces/run_id.jsonl. Parquet으로 남기는 이유:
172
+ `gladius query`로 바로 집계 — 도구가 도구를 평가한다(dogfooding)."""
173
+ out_dir.mkdir(parents=True, exist_ok=True)
174
+ trace_path = out_dir / "traces" / f"{run_id}.jsonl"
175
+ trace_path.parent.mkdir(parents=True, exist_ok=True)
176
+ results: list[EvalResult] = []
177
+ with trace_path.open("w", encoding="utf-8") as tf:
178
+ for case in cases:
179
+ result, trace = run_case(case, catalog, provider, top_k=top_k)
180
+ results.append(result)
181
+ tf.write(
182
+ json.dumps(
183
+ {"run_id": run_id, "case_id": case.id, **asdict(trace)}, ensure_ascii=False
184
+ )
185
+ + "\n"
186
+ )
187
+ table = pa.Table.from_pylist([{"run_id": run_id, **asdict(r)} for r in results])
188
+ pq.write_table( # pyright: ignore[reportUnknownMemberType]
189
+ table, out_dir / f"{run_id}.parquet", compression="zstd"
190
+ )
191
+ return results
192
+
193
+
194
+ def summarize(results: list[EvalResult]) -> dict[str, int]:
195
+ counts = {m: 0 for m in FAILURE_MODES}
196
+ for r in results:
197
+ counts[r.failure_mode] += 1
198
+ return {k: v for k, v in counts.items() if v or k == "CORRECT"}
augur/fixture.py ADDED
@@ -0,0 +1,70 @@
1
+ """결정적 합성 데이터 — eval 케이스의 골든 SQL이 항상 같은 결과를 내도록 seed 고정.
2
+
3
+ 생성: customers / products / orders 세 테이블 Parquet. 스크립트 진입점은
4
+ `packages/augur/eval/make_fixture.py`(이 모듈을 호출), 테스트 fixture도 같은 함수를 쓴다.
5
+ """
6
+
7
+ import random
8
+ from datetime import date, timedelta
9
+ from pathlib import Path
10
+
11
+ import pyarrow as pa
12
+ import pyarrow.parquet as pq
13
+
14
+ SEED = 20260908
15
+ N_CUSTOMERS = 40
16
+ N_ORDERS = 400
17
+ CITIES = ["Seoul", "Busan", "Incheon", "Daegu", "Daejeon"]
18
+ SEGMENTS = ["consumer", "smb", "enterprise"]
19
+ STATUSES = ["paid", "pending", "refunded", "cancelled"]
20
+ PRODUCTS: list[tuple[int, str, str, float]] = [
21
+ (1, "Keyboard", "hardware", 49.0),
22
+ (2, "Monitor", "hardware", 199.0),
23
+ (3, "Mouse", "hardware", 19.0),
24
+ (4, "IDE License", "software", 89.0),
25
+ (5, "Cloud Credits", "software", 120.0),
26
+ (6, "Headset", "hardware", 59.0),
27
+ (7, "VPN Subscription", "software", 9.0),
28
+ ]
29
+ TABLE_NAMES = ("customers", "products", "orders")
30
+
31
+ Row = dict[str, object]
32
+
33
+
34
+ def make(out: Path) -> None:
35
+ rng = random.Random(SEED)
36
+ customers: list[Row] = [
37
+ {
38
+ "customer_id": i,
39
+ "name": f"cust_{i:03d}",
40
+ "city": rng.choice(CITIES),
41
+ "segment": rng.choice(SEGMENTS),
42
+ "signup_date": date(2025, 1, 1) + timedelta(days=rng.randint(0, 400)),
43
+ }
44
+ for i in range(1, N_CUSTOMERS + 1)
45
+ ]
46
+ products: list[Row] = [
47
+ {"product_id": p[0], "product_name": p[1], "category": p[2], "price": p[3]}
48
+ for p in PRODUCTS
49
+ ]
50
+ orders: list[Row] = []
51
+ for i in range(1, N_ORDERS + 1):
52
+ p = rng.choice(PRODUCTS)
53
+ qty = rng.randint(1, 5)
54
+ orders.append(
55
+ {
56
+ "order_id": i,
57
+ "customer_id": rng.randint(1, N_CUSTOMERS),
58
+ "product_id": p[0],
59
+ "quantity": qty,
60
+ "amount": round(qty * p[3], 2),
61
+ "status": rng.choice(STATUSES),
62
+ "ordered_at": date(2026, 1, 1) + timedelta(days=rng.randint(0, 240)),
63
+ }
64
+ )
65
+ for name, rows in (("customers", customers), ("products", products), ("orders", orders)):
66
+ d = out / name
67
+ d.mkdir(parents=True, exist_ok=True)
68
+ pq.write_table( # pyright: ignore[reportUnknownMemberType]
69
+ pa.Table.from_pylist(rows), d / "part-0.parquet"
70
+ )