de-gladius 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ *.egg-info/
5
+ dist/
6
+ .coverage
7
+ coverage.xml
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .arsenal/
11
+ data/
12
+ .DS_Store
@@ -0,0 +1,68 @@
1
+ Metadata-Version: 2.5
2
+ Name: de-gladius
3
+ Version: 0.2.0
4
+ Summary: Gladius — 변환·쿼리. map/steps 선언을 SQL로 컴파일해 DuckDB에서 실행
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ Project-URL: PyPI, https://pypi.org/project/de-gladius/
8
+ License-Expression: Apache-2.0
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
+ Requires-Python: >=3.11
17
+ Requires-Dist: arsenal-core<0.3,>=0.2
18
+ Requires-Dist: duckdb>=1.0
19
+ Requires-Dist: pyarrow>=17
20
+ Requires-Dist: typer>=0.12
21
+ Provides-Extra: python
22
+ Requires-Dist: numpy>=1.26; extra == 'python'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # Gladius (`de-gladius`)
26
+
27
+ > 군단병 주력검. 가장 자주 쓰는 변환·쿼리 도구.
28
+
29
+ 선언형 변환(`map`/`steps`)을 SQL로 컴파일해 DuckDB에서 실행하는 엔진. 필터·
30
+ 리네임·캐스팅·정렬·중복 제거 같은 흔한 변환을 YAML 레시피로 표현하면, 손으로
31
+ 쓴 SQL과 동일한 결과를 벡터화 엔진으로 빠르게 낸다. `gladius compile`로 생성된
32
+ SQL을 그대로 확인할 수 있어 마법이 없다. `gladius query`로 수집 결과 Parquet에
33
+ 바로 SQL을 던지는 미니 DWH로도 쓴다.
34
+
35
+ ## 설치
36
+
37
+ ```bash
38
+ pip install de-gladius
39
+ ```
40
+
41
+ 배포명은 `de-gladius`지만 Python import와 CLI는 기존처럼 `gladius`다.
42
+
43
+ ## 사용 예
44
+
45
+ ```yaml
46
+ # transform.yaml
47
+ name: clean-issues
48
+ input: ./data/issues
49
+ steps:
50
+ - filter: "state = 'open'"
51
+ - rename: { created_at: opened_at }
52
+ - cast: { number: bigint }
53
+ - select: [number, title, opened_at, user]
54
+ output: ./data/issues_clean
55
+ ```
56
+
57
+ ```bash
58
+ gladius compile transform.yaml # 생성될 SQL을 미리 확인
59
+ gladius run transform.yaml # Parquet in → DuckDB → Parquet out
60
+ gladius query "SELECT count(*) FROM './data/issues_clean/*.parquet'"
61
+ ```
62
+
63
+ ## 문서
64
+
65
+ - [docs/01-scope.md](../../docs/01-scope.md) — Gladius가 다루는 범위(M3)
66
+ - [docs/08-limits.md](../../docs/08-limits.md) — 단일 노드 처리 envelope(벤치마크 실측)
67
+ - [benchmarks/RESULTS.md](../../benchmarks/RESULTS.md) — 변환 벤치마크 수치
68
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,44 @@
1
+ # Gladius (`de-gladius`)
2
+
3
+ > 군단병 주력검. 가장 자주 쓰는 변환·쿼리 도구.
4
+
5
+ 선언형 변환(`map`/`steps`)을 SQL로 컴파일해 DuckDB에서 실행하는 엔진. 필터·
6
+ 리네임·캐스팅·정렬·중복 제거 같은 흔한 변환을 YAML 레시피로 표현하면, 손으로
7
+ 쓴 SQL과 동일한 결과를 벡터화 엔진으로 빠르게 낸다. `gladius compile`로 생성된
8
+ SQL을 그대로 확인할 수 있어 마법이 없다. `gladius query`로 수집 결과 Parquet에
9
+ 바로 SQL을 던지는 미니 DWH로도 쓴다.
10
+
11
+ ## 설치
12
+
13
+ ```bash
14
+ pip install de-gladius
15
+ ```
16
+
17
+ 배포명은 `de-gladius`지만 Python import와 CLI는 기존처럼 `gladius`다.
18
+
19
+ ## 사용 예
20
+
21
+ ```yaml
22
+ # transform.yaml
23
+ name: clean-issues
24
+ input: ./data/issues
25
+ steps:
26
+ - filter: "state = 'open'"
27
+ - rename: { created_at: opened_at }
28
+ - cast: { number: bigint }
29
+ - select: [number, title, opened_at, user]
30
+ output: ./data/issues_clean
31
+ ```
32
+
33
+ ```bash
34
+ gladius compile transform.yaml # 생성될 SQL을 미리 확인
35
+ gladius run transform.yaml # Parquet in → DuckDB → Parquet out
36
+ gladius query "SELECT count(*) FROM './data/issues_clean/*.parquet'"
37
+ ```
38
+
39
+ ## 문서
40
+
41
+ - [docs/01-scope.md](../../docs/01-scope.md) — Gladius가 다루는 범위(M3)
42
+ - [docs/08-limits.md](../../docs/08-limits.md) — 단일 노드 처리 envelope(벤치마크 실측)
43
+ - [benchmarks/RESULTS.md](../../benchmarks/RESULTS.md) — 변환 벤치마크 수치
44
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,35 @@
1
+ [project]
2
+ name = "de-gladius"
3
+ version = "0.2.0"
4
+ description = "Gladius — 변환·쿼리. map/steps 선언을 SQL로 컴파일해 DuckDB에서 실행"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ license = "Apache-2.0"
8
+ dependencies = ["arsenal-core>=0.2,<0.3", "duckdb>=1.0", "pyarrow>=17", "typer>=0.12"]
9
+ classifiers = [
10
+ "Programming Language :: Python :: 3",
11
+ "Programming Language :: Python :: 3.11",
12
+ "Programming Language :: Python :: 3.12",
13
+ "License :: OSI Approved :: Apache Software License",
14
+ "Operating System :: OS Independent",
15
+ "Intended Audience :: Developers",
16
+ "Topic :: Software Development :: Libraries :: Python Modules",
17
+ ]
18
+
19
+ [project.urls]
20
+ Homepage = "https://github.com/KooEric/de-arsenal"
21
+ Repository = "https://github.com/KooEric/de-arsenal"
22
+ PyPI = "https://pypi.org/project/de-gladius/"
23
+
24
+ [project.scripts]
25
+ gladius = "gladius.cli:app"
26
+
27
+ [project.optional-dependencies]
28
+ python = ["numpy>=1.26"]
29
+
30
+ [build-system]
31
+ requires = ["hatchling"]
32
+ build-backend = "hatchling.build"
33
+
34
+ [tool.hatch.build.targets.wheel]
35
+ packages = ["src/gladius"]
@@ -0,0 +1,12 @@
1
+ """Gladius — 변환·쿼리 (핵심 처리).
2
+
3
+ map/steps 선언 → SQL 컴파일 → DuckDB 벡터화 실행. 단일 노드 수백 GB.
4
+ 구조: spec.py compile/ engine.py cli.py (docs/02-architecture.md)
5
+ """
6
+
7
+ from importlib.metadata import PackageNotFoundError, version
8
+
9
+ try:
10
+ __version__ = version("de-gladius")
11
+ except PackageNotFoundError:
12
+ __version__ = "0.2.0"
@@ -0,0 +1,78 @@
1
+ """Gladius CLI (M3 Task 3.6).
2
+
3
+ compile: 생성될 SQL을 그대로 출력 — 투명성 커맨드, 마법 없음.
4
+ run: 변환 실행.
5
+ query: 수집·변환 결과 Parquet에 즉석 SQL — 미니 DWH의 "아하 모먼트".
6
+ """
7
+
8
+ import json
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ import duckdb
13
+ import pyarrow.csv as pa_csv
14
+ import typer
15
+
16
+ from arsenal_core.errors import ArsenalError
17
+ from gladius.compile import compile_sql
18
+ from gladius.engine import query as run_query
19
+ from gladius.engine import run_transform
20
+ from gladius.loader import load_transform
21
+
22
+ app = typer.Typer(help="Gladius — 변환·쿼리. 선언이 SQL로 컴파일된다.")
23
+
24
+
25
+ @app.command(name="compile")
26
+ def compile_cmd(spec_path: Path) -> None:
27
+ """변환 스펙이 생성할 SQL을 출력한다 (실행하지 않음)."""
28
+ try:
29
+ spec = load_transform(spec_path)
30
+ sql = compile_sql(spec)
31
+ except ArsenalError as e:
32
+ typer.echo(f"error: {e}", err=True)
33
+ raise typer.Exit(1) from e
34
+ typer.echo(sql)
35
+
36
+
37
+ @app.command(name="run")
38
+ def run_cmd(spec_path: Path) -> None:
39
+ """변환 실행: Parquet in → DuckDB → Parquet out."""
40
+ try:
41
+ spec = load_transform(spec_path)
42
+ out = run_transform(spec)
43
+ except ArsenalError as e:
44
+ typer.echo(f"error: {e}", err=True)
45
+ raise typer.Exit(1) from e
46
+ typer.echo(f"done: {out}")
47
+
48
+
49
+ @app.command(name="query")
50
+ def query_cmd(
51
+ sql: str,
52
+ fmt: str = typer.Option("table", "--format", help="table | csv | jsonl"),
53
+ limit: int = typer.Option(0, help="0 = 제한 없음"),
54
+ ) -> None:
55
+ """수집·변환 결과 Parquet에 즉석 SQL (미니 DWH)."""
56
+ effective_sql = sql if limit <= 0 else f"SELECT * FROM ({sql}) AS _gladius_query LIMIT {limit}"
57
+ try:
58
+ result = run_query(effective_sql)
59
+ except ArsenalError as e:
60
+ typer.echo(f"error: {e}", err=True)
61
+ raise typer.Exit(1) from e
62
+
63
+ if fmt == "table":
64
+ duckdb.sql("SELECT * FROM result").show() # pyright: ignore[reportUnknownMemberType]
65
+ elif fmt == "csv":
66
+ pa_csv.write_csv( # pyright: ignore[reportPrivateImportUsage, reportUnknownMemberType]
67
+ result, sys.stdout.buffer
68
+ )
69
+ elif fmt == "jsonl":
70
+ for row in result.to_pylist():
71
+ typer.echo(json.dumps(row))
72
+ else:
73
+ typer.echo(f"error: unsupported --format: {fmt!r} (table | csv | jsonl)", err=True)
74
+ raise typer.Exit(1)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ app()
@@ -0,0 +1,3 @@
1
+ from gladius.compile.transpiler import compile_sql
2
+
3
+ __all__ = ["compile_sql"]
@@ -0,0 +1,40 @@
1
+ """식별자 안전 처리 — quoting + cast 타입 화이트리스트 (M3 Task 3.2).
2
+
3
+ 경계: filter/derive/map의 expression은 사용자가 작성하는 SQL 조각이다 — 사용자
4
+ 자신의 쿼리이므로 인젝션 방어 대상이 아니라 문서화 대상(dbt와 동일한 신뢰 모델).
5
+ 방어 대상은 식별자·타입처럼 구조를 깨는 위치다.
6
+ """
7
+
8
+ from arsenal_core.errors import FatalError
9
+
10
+
11
+ def quote_ident(name: str) -> str:
12
+ return '"' + name.replace('"', '""') + '"'
13
+
14
+
15
+ def quote_str_literal(value: str) -> str:
16
+ """SQL 문자열 리터럴로 안전하게 인용한다 (작은따옴표 이스케이프).
17
+
18
+ 경로(input/output) 등 우리가 직접 SQL 리터럴에 보간하는 값에 사용한다 — 사용자가
19
+ 작성한 filter/derive 표현식(위 모듈 docstring 참조)과는 다른 신뢰 경계다.
20
+ """
21
+ return "'" + value.replace("'", "''") + "'"
22
+
23
+
24
+ _CAST_TYPES = { # cast step 타입 화이트리스트 — 임의 문자열을 타입 위치에 두지 않는다
25
+ "bigint": "BIGINT",
26
+ "integer": "INTEGER",
27
+ "double": "DOUBLE",
28
+ "varchar": "VARCHAR",
29
+ "boolean": "BOOLEAN",
30
+ "date": "DATE",
31
+ "timestamp": "TIMESTAMP",
32
+ "decimal": "DECIMAL(38, 9)",
33
+ }
34
+
35
+
36
+ def normalize_type(t: str) -> str:
37
+ try:
38
+ return _CAST_TYPES[t.lower()]
39
+ except KeyError:
40
+ raise FatalError(f"unsupported cast type: {t!r}") from None
@@ -0,0 +1,108 @@
1
+ """steps → SQL 트랜스파일러 (M3 Task 3.3).
2
+
3
+ 각 step이 CTE 하나로 컴파일된다:
4
+ WITH s0 AS (SELECT * FROM read_parquet('...')),
5
+ s1 AS (SELECT * FROM s0 WHERE ...), -- filter
6
+ s2 AS (SELECT * EXCLUDE (a), a AS b FROM s1) -- rename
7
+ SELECT ... FROM s2 -- select
8
+
9
+ 원칙:
10
+ - 생성 SQL은 `gladius compile`로 그대로 노출 (투명성 — 마법 없음)
11
+ - 식별자는 화이트리스트 + 인용, 값은 바인딩 — 인젝션 안전 (hypothesis 퍼징)
12
+ - golden test: 손으로 쓴 SQL과 결과 동치 (docs/05-testing-plan.md)
13
+
14
+ select step은 CTE가 아니라 최종 SELECT로 처리한다. 마지막 위치가 아닌 select는
15
+ 그 지점에서 실제 projection을 수행해(중간 select 허용) 이후 step이 좁혀진 컬럼
16
+ 집합 위에서 동작하게 한다.
17
+ """
18
+
19
+ from collections.abc import Sequence
20
+ from pathlib import Path
21
+
22
+ from gladius.compile.ident import normalize_type, quote_ident, quote_str_literal
23
+ from gladius.spec import (
24
+ CastStep,
25
+ DedupStep,
26
+ DeriveStep,
27
+ FilterStep,
28
+ PythonUdfStep,
29
+ RenameStep,
30
+ SelectStep,
31
+ SqlStep,
32
+ Step,
33
+ TransformSpec,
34
+ )
35
+ from gladius.udf import udf_name
36
+
37
+
38
+ def compile_sql(spec: TransformSpec, input_files: Sequence[Path] | None = None) -> str:
39
+ """TransformSpec을 실행 가능한 DuckDB SQL 문자열로 컴파일한다."""
40
+ source = _input_source(spec, input_files)
41
+ ctes = [f"s0 AS (SELECT * FROM read_parquet({source}, union_by_name=true))"]
42
+ idx = 0
43
+ if spec.map:
44
+ idx += 1
45
+ cols = ", ".join(f"{expr} AS {quote_ident(new)}" for new, expr in spec.map.items())
46
+ ctes.append(f"s{idx} AS (SELECT {cols} FROM s{idx - 1})")
47
+ steps = spec.steps
48
+ for i, step in enumerate(steps):
49
+ is_last = i == len(steps) - 1
50
+ if isinstance(step, SelectStep) and is_last:
51
+ continue # 마지막 select는 CTE가 아니라 최종 SELECT에서 처리
52
+ idx += 1
53
+ ctes.append(f"s{idx} AS ({_compile_step(step, f's{idx - 1}', i)})")
54
+ final = _final_select(steps, f"s{idx}")
55
+ return "WITH " + ",\n".join(ctes) + "\n" + final
56
+
57
+
58
+ def _input_source(spec: TransformSpec, input_files: Sequence[Path] | None) -> str:
59
+ if input_files is None:
60
+ src = str(spec.input).rstrip("/")
61
+ return quote_str_literal(f"{src}/**/*.parquet")
62
+ if len(input_files) == 1:
63
+ return quote_str_literal(str(input_files[0]))
64
+ values = ", ".join(quote_str_literal(str(path)) for path in input_files)
65
+ return "[" + values + "]"
66
+
67
+
68
+ def _compile_step(step: Step, prev: str, step_index: int) -> str:
69
+ match step:
70
+ case FilterStep(filter=cond):
71
+ return f"SELECT * FROM {prev} WHERE {cond}"
72
+ case RenameStep(rename=m):
73
+ ex = ", ".join(quote_ident(o) for o in m)
74
+ al = ", ".join(f"{quote_ident(o)} AS {quote_ident(n)}" for o, n in m.items())
75
+ return f"SELECT * EXCLUDE ({ex}), {al} FROM {prev}"
76
+ case CastStep(cast=m):
77
+ rep = ", ".join(
78
+ f"CAST({quote_ident(c)} AS {normalize_type(t)}) AS {quote_ident(c)}"
79
+ for c, t in m.items()
80
+ )
81
+ return f"SELECT * REPLACE ({rep}) FROM {prev}"
82
+ case DedupStep(dedup=keys):
83
+ part = ", ".join(quote_ident(k) for k in keys)
84
+ return (
85
+ f"SELECT * FROM {prev} QUALIFY row_number() "
86
+ f"OVER (PARTITION BY {part} ORDER BY {part}) = 1"
87
+ )
88
+ case DeriveStep(derive=m):
89
+ dv = ", ".join(f"{expr} AS {quote_ident(n)}" for n, expr in m.items())
90
+ return f"SELECT *, {dv} FROM {prev}"
91
+ case SelectStep(select=cols):
92
+ projected = ", ".join(quote_ident(c) for c in cols)
93
+ return f"SELECT {projected} FROM {prev}"
94
+ case SqlStep(sql=expression):
95
+ return expression.replace("{input}", prev)
96
+ case PythonUdfStep(args=args, output=output):
97
+ arguments = ", ".join(quote_ident(argument) for argument in args)
98
+ return (
99
+ f"SELECT *, {quote_ident(udf_name(step_index))}({arguments}) "
100
+ f"AS {quote_ident(output)} FROM {prev}"
101
+ )
102
+
103
+
104
+ def _final_select(steps: list[Step], last: str) -> str:
105
+ if steps and isinstance(steps[-1], SelectStep):
106
+ cols = ", ".join(quote_ident(c) for c in steps[-1].select)
107
+ return f"SELECT {cols} FROM {last}"
108
+ return f"SELECT * FROM {last}"
@@ -0,0 +1,182 @@
1
+ """DuckDB 실행 엔진 (M3 Task 3.3~3.6). 엔진은 빌린다 — 우리는 컴파일과 신뢰성만.
2
+
3
+ Parquet in → (compile_sql 결과 실행) → Parquet out. Arrow 경유 zero-copy.
4
+ query()는 즉석 SQL(미니 DWH) — 원클릭 경험의 "아하 모먼트".
5
+ """
6
+
7
+ import hashlib
8
+ import os
9
+ import shutil
10
+ import typing as t
11
+ from pathlib import Path
12
+
13
+ import duckdb
14
+ import pyarrow as pa
15
+
16
+ from arsenal_core.errors import FatalError
17
+ from gladius.compile import compile_sql
18
+ from gladius.compile.ident import quote_str_literal
19
+ from gladius.incremental import (
20
+ IncrementalStore,
21
+ InputFile,
22
+ atomic_replace_directory,
23
+ discover_input_files,
24
+ reset_output,
25
+ state_path,
26
+ transform_hash,
27
+ )
28
+ from gladius.spec import TransformSpec
29
+ from gladius.udf import register_python_udfs
30
+
31
+
32
+ def run_transform(spec: TransformSpec) -> Path:
33
+ """변환을 실행하고 output 경로를 반환한다.
34
+
35
+ 전체 재계산 의미론(P0): 매 실행은 output을 통째로 다시 쓴다. 임시 디렉터리에
36
+ 쓰고 os.replace로 원자 교체 — 실패한 실행이 이전 성공 출력을 훼손하지 않는다.
37
+ """
38
+ if spec.incremental is not None:
39
+ return _run_incremental(spec)
40
+ sql = compile_sql(spec)
41
+ con = duckdb.connect() # in-memory, 상태 없음
42
+ register_python_udfs(con, spec)
43
+ con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
44
+ # spec.output은 str(URI 스킴 보존용) — 파일시스템 연산이 필요한 여기서만 Path로 감싼다.
45
+ output = Path(spec.output)
46
+ tmp = output.with_name(output.name + ".tmp")
47
+ tmp.mkdir(parents=True, exist_ok=True)
48
+ try:
49
+ out_path = quote_str_literal(str(tmp / "part-0.parquet"))
50
+ con.execute(f"COPY ({sql}) TO {out_path} (FORMAT PARQUET, COMPRESSION ZSTD)")
51
+ except duckdb.Error as e:
52
+ raise FatalError(f"transform failed: {e}\n--- compiled SQL ---\n{sql}") from e
53
+ finally:
54
+ con.close()
55
+ if output.exists():
56
+ shutil.rmtree(output) # 변환 출력은 전체 재계산 의미론 (P0)
57
+ os.replace(tmp, output)
58
+ return output
59
+
60
+
61
+ def _run_incremental(spec: TransformSpec) -> Path:
62
+ incremental = spec.incremental
63
+ if incremental is None: # pragma: no cover - guarded by run_transform
64
+ raise FatalError("incremental execution requires an incremental spec")
65
+ files = discover_input_files(spec)
66
+ output = Path(spec.output)
67
+ store = IncrementalStore(state_path(spec))
68
+ try:
69
+ current_hash = transform_hash(spec)
70
+ known = store.signatures()
71
+ current = {item.key: item.signature for item in files}
72
+ changed_existing = any(
73
+ key in known and known[key] != signature for key, signature in current.items()
74
+ )
75
+ removed_input = any(key not in current for key in known)
76
+ if (
77
+ store.spec_hash() != current_hash
78
+ or (not output.exists() and known)
79
+ or changed_existing
80
+ or removed_input
81
+ ):
82
+ reset_output(output)
83
+ store.reset(current_hash)
84
+ known = {}
85
+ pending = [item for item in files if known.get(item.key) != item.signature]
86
+ if not pending:
87
+ output.mkdir(parents=True, exist_ok=True)
88
+ return output
89
+ if not files:
90
+ output.mkdir(parents=True, exist_ok=True)
91
+ return output
92
+
93
+ sql = compile_sql(spec, [item.path for item in pending])
94
+ con = duckdb.connect()
95
+ register_python_udfs(con, spec)
96
+ con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
97
+ tmp_input = output.with_name(output.name + ".input.tmp")
98
+ tmp_output = output.with_name(output.name + ".tmp")
99
+ try:
100
+ if tmp_input.exists():
101
+ shutil.rmtree(tmp_input)
102
+ tmp_input.mkdir(parents=True)
103
+ new_part = tmp_input / "new.parquet"
104
+ con.execute(
105
+ f"COPY ({sql}) TO {quote_str_literal(str(new_part))} "
106
+ "(FORMAT PARQUET, COMPRESSION ZSTD)"
107
+ )
108
+ if incremental.mode == "by_unit":
109
+ _append_incremental_part(output, new_part, pending)
110
+ else:
111
+ _merge_incremental_output(con, output, new_part, tmp_output, incremental.key or [])
112
+ store.mark_processed(pending)
113
+ except duckdb.Error as e:
114
+ raise FatalError(
115
+ f"incremental transform failed: {e}\n--- compiled SQL ---\n{sql}"
116
+ ) from e
117
+ finally:
118
+ con.close()
119
+ if tmp_input.exists():
120
+ shutil.rmtree(tmp_input)
121
+ if tmp_output.exists():
122
+ shutil.rmtree(tmp_output)
123
+ return output
124
+ finally:
125
+ store.close()
126
+
127
+
128
+ def _append_incremental_part(output: Path, new_part: Path, pending: list[InputFile]) -> None:
129
+ output.mkdir(parents=True, exist_ok=True)
130
+ batch_id = hashlib.sha256(
131
+ "|".join(f"{item.key}:{item.signature}" for item in pending).encode()
132
+ ).hexdigest()[:16]
133
+ target = output / f"part-{batch_id}.parquet"
134
+ os.replace(new_part, target)
135
+
136
+
137
+ def _merge_incremental_output(
138
+ con: duckdb.DuckDBPyConnection,
139
+ output: Path,
140
+ new_part: Path,
141
+ tmp_output: Path,
142
+ keys: list[str],
143
+ ) -> None:
144
+ new_relation = quote_str_literal(str(new_part))
145
+ if output.exists() and any(output.glob("**/*.parquet")):
146
+ old_relation = quote_str_literal(str(output / "**/*.parquet"))
147
+ partition = ", ".join('"' + key.replace('"', '""') + '"' for key in keys)
148
+ query = (
149
+ "SELECT * EXCLUDE (__gladius_precedence) FROM ("
150
+ f"SELECT *, 0 AS __gladius_precedence FROM "
151
+ f"read_parquet({new_relation}, union_by_name=true) "
152
+ "UNION ALL "
153
+ f"SELECT *, 1 AS __gladius_precedence FROM "
154
+ f"read_parquet({old_relation}, union_by_name=true)"
155
+ f") QUALIFY row_number() OVER (PARTITION BY {partition} "
156
+ "ORDER BY __gladius_precedence) = 1"
157
+ )
158
+ else:
159
+ query = f"SELECT * FROM read_parquet({new_relation}, union_by_name=true)"
160
+ tmp_output.mkdir(parents=True, exist_ok=True)
161
+ con.execute(
162
+ f"COPY ({query}) TO {quote_str_literal(str(tmp_output / 'part-0.parquet'))} "
163
+ "(FORMAT PARQUET, COMPRESSION ZSTD)"
164
+ )
165
+ atomic_replace_directory(tmp_output, output)
166
+
167
+
168
+ def query(sql: str) -> pa.Table:
169
+ """Parquet 레이크에 즉석 SQL — FROM './data/*.parquet' 경로 직접 참조.
170
+
171
+ in-memory DuckDB, 상태 없음. DuckDB 에러는 FatalError로 변환.
172
+ """
173
+ con = duckdb.connect() # in-memory, 상태 없음
174
+ try:
175
+ arrow_table = con.sql( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
176
+ sql
177
+ ).to_arrow_table()
178
+ return t.cast(pa.Table, arrow_table)
179
+ except duckdb.Error as e:
180
+ raise FatalError(str(e)) from e
181
+ finally:
182
+ con.close()