de-gladius 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,68 @@
1
+ Metadata-Version: 2.5
2
+ Name: de-gladius
3
+ Version: 0.2.2
4
+ Summary: Gladius — 변환·쿼리. map/steps 선언을 SQL로 컴파일해 DuckDB에서 실행
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ Project-URL: PyPI, https://pypi.org/project/de-gladius/
8
+ License-Expression: Apache-2.0
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
+ Requires-Python: >=3.11
17
+ Requires-Dist: arsenal-core<0.3,>=0.2
18
+ Requires-Dist: duckdb>=1.0
19
+ Requires-Dist: pyarrow>=17
20
+ Requires-Dist: typer>=0.12
21
+ Provides-Extra: python
22
+ Requires-Dist: numpy>=1.26; extra == 'python'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # Gladius (`de-gladius`)
26
+
27
+ > 군단병 주력검. 가장 자주 쓰는 변환·쿼리 도구.
28
+
29
+ 선언형 변환(`map`/`steps`)을 SQL로 컴파일해 DuckDB에서 실행하는 엔진. 필터·
30
+ 리네임·캐스팅·정렬·중복 제거 같은 흔한 변환을 YAML 레시피로 표현하면, 손으로
31
+ 쓴 SQL과 동일한 결과를 벡터화 엔진으로 빠르게 낸다. `gladius compile`로 생성된
32
+ SQL을 그대로 확인할 수 있어 마법이 없다. `gladius query`로 수집 결과 Parquet에
33
+ 바로 SQL을 던지는 미니 DWH로도 쓴다.
34
+
35
+ ## 설치
36
+
37
+ ```bash
38
+ pip install de-gladius
39
+ ```
40
+
41
+ 배포명은 `de-gladius`지만 Python import와 CLI는 기존처럼 `gladius`다.
42
+
43
+ ## 사용 예
44
+
45
+ ```yaml
46
+ # transform.yaml
47
+ name: clean-issues
48
+ input: ./data/issues
49
+ steps:
50
+ - filter: "state = 'open'"
51
+ - rename: { created_at: opened_at }
52
+ - cast: { number: bigint }
53
+ - select: [number, title, opened_at, user]
54
+ output: ./data/issues_clean
55
+ ```
56
+
57
+ ```bash
58
+ gladius compile transform.yaml # 생성될 SQL을 미리 확인
59
+ gladius run transform.yaml # Parquet in → DuckDB → Parquet out
60
+ gladius query "SELECT count(*) FROM './data/issues_clean/*.parquet'"
61
+ ```
62
+
63
+ ## 문서
64
+
65
+ - [docs/01-scope.md](../../docs/01-scope.md) — Gladius가 다루는 범위(M3)
66
+ - [docs/08-limits.md](../../docs/08-limits.md) — 단일 노드 처리 envelope(벤치마크 실측)
67
+ - [benchmarks/RESULTS.md](../../benchmarks/RESULTS.md) — 변환 벤치마크 수치
68
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,15 @@
1
+ gladius/__init__.py,sha256=2fcA-JDV1MgDW_8xJz2vgJrVL5k8gxPvg50Gt5sG5cw,372
2
+ gladius/cli.py,sha256=ExWjSlfKrRnmBw8qv5jtv5Rr5U1bhztZV7KRcli-CKY,2447
3
+ gladius/engine.py,sha256=Bt6b2RhxsMhRZRHTyFG4yKoTiQNJxPWLo27bRn0RR40,6888
4
+ gladius/incremental.py,sha256=tViLkG8xWZC8cpL9YR1ns5owA4p_gAstf1t9P2NRBRI,3332
5
+ gladius/loader.py,sha256=erH-BTzPhyHB2ofNMofIKTQwmNR_14zobKSIxRpM6vo,1354
6
+ gladius/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ gladius/spec.py,sha256=RzEIO1RdccXmdvjt5NU2cFcTPVflr8CDUeClIqc3GVk,5006
8
+ gladius/udf.py,sha256=eEyPrrDJaeD4TVYB0wpuZA29H4OPxtrUz2dIYBuD4dE,1914
9
+ gladius/compile/__init__.py,sha256=_d4vxzVOi_e2R04-U7HH4VkHAZxrE0GGX5C_yj9IjVU,78
10
+ gladius/compile/ident.py,sha256=Gdb8ApQXWu3IhhJbwLHlNQTRwu_GbOotA2sRSlcSj8s,1408
11
+ gladius/compile/transpiler.py,sha256=0YwAhiiFbUoMCdoh5j1mZjEJlEwRzD-v7gRX-kAOcp4,4454
12
+ de_gladius-0.2.2.dist-info/METADATA,sha256=RPGm5kRHhxTd3AyN17B-lt8H8gjx8fqYfeJ_ol_Mg6g,2485
13
+ de_gladius-0.2.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
14
+ de_gladius-0.2.2.dist-info/entry_points.txt,sha256=P7moccpxo_Mx0BY2iuEjv7b5Q1Es_V_EE7CH_bUh6ms,44
15
+ de_gladius-0.2.2.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ gladius = gladius.cli:app
gladius/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ """Gladius — 변환·쿼리 (핵심 처리).
2
+
3
+ map/steps 선언 → SQL 컴파일 → DuckDB 벡터화 실행. 단일 노드 수백 GB.
4
+ 구조: spec.py compile/ engine.py cli.py (docs/02-architecture.md)
5
+ """
6
+
7
+ from importlib.metadata import PackageNotFoundError, version
8
+
9
+ try:
10
+ __version__ = version("de-gladius")
11
+ except PackageNotFoundError:
12
+ __version__ = "0.2.2"
gladius/cli.py ADDED
@@ -0,0 +1,78 @@
1
+ """Gladius CLI (M3 Task 3.6).
2
+
3
+ compile: 생성될 SQL을 그대로 출력 — 투명성 커맨드, 마법 없음.
4
+ run: 변환 실행.
5
+ query: 수집·변환 결과 Parquet에 즉석 SQL — 미니 DWH의 "아하 모먼트".
6
+ """
7
+
8
+ import json
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ import duckdb
13
+ import pyarrow.csv as pa_csv
14
+ import typer
15
+
16
+ from arsenal_core.errors import ArsenalError
17
+ from gladius.compile import compile_sql
18
+ from gladius.engine import query as run_query
19
+ from gladius.engine import run_transform
20
+ from gladius.loader import load_transform
21
+
22
+ app = typer.Typer(help="Gladius — 변환·쿼리. 선언이 SQL로 컴파일된다.")
23
+
24
+
25
+ @app.command(name="compile")
26
+ def compile_cmd(spec_path: Path) -> None:
27
+ """변환 스펙이 생성할 SQL을 출력한다 (실행하지 않음)."""
28
+ try:
29
+ spec = load_transform(spec_path)
30
+ sql = compile_sql(spec)
31
+ except ArsenalError as e:
32
+ typer.echo(f"error: {e}", err=True)
33
+ raise typer.Exit(1) from e
34
+ typer.echo(sql)
35
+
36
+
37
+ @app.command(name="run")
38
+ def run_cmd(spec_path: Path) -> None:
39
+ """변환 실행: Parquet in → DuckDB → Parquet out."""
40
+ try:
41
+ spec = load_transform(spec_path)
42
+ out = run_transform(spec)
43
+ except ArsenalError as e:
44
+ typer.echo(f"error: {e}", err=True)
45
+ raise typer.Exit(1) from e
46
+ typer.echo(f"done: {out}")
47
+
48
+
49
+ @app.command(name="query")
50
+ def query_cmd(
51
+ sql: str,
52
+ fmt: str = typer.Option("table", "--format", help="table | csv | jsonl"),
53
+ limit: int = typer.Option(0, help="0 = 제한 없음"),
54
+ ) -> None:
55
+ """수집·변환 결과 Parquet에 즉석 SQL (미니 DWH)."""
56
+ effective_sql = sql if limit <= 0 else f"SELECT * FROM ({sql}) AS _gladius_query LIMIT {limit}"
57
+ try:
58
+ result = run_query(effective_sql)
59
+ except ArsenalError as e:
60
+ typer.echo(f"error: {e}", err=True)
61
+ raise typer.Exit(1) from e
62
+
63
+ if fmt == "table":
64
+ duckdb.sql("SELECT * FROM result").show() # pyright: ignore[reportUnknownMemberType]
65
+ elif fmt == "csv":
66
+ pa_csv.write_csv( # pyright: ignore[reportPrivateImportUsage, reportUnknownMemberType]
67
+ result, sys.stdout.buffer
68
+ )
69
+ elif fmt == "jsonl":
70
+ for row in result.to_pylist():
71
+ typer.echo(json.dumps(row))
72
+ else:
73
+ typer.echo(f"error: unsupported --format: {fmt!r} (table | csv | jsonl)", err=True)
74
+ raise typer.Exit(1)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ app()
@@ -0,0 +1,3 @@
1
+ from gladius.compile.transpiler import compile_sql
2
+
3
+ __all__ = ["compile_sql"]
@@ -0,0 +1,40 @@
1
+ """식별자 안전 처리 — quoting + cast 타입 화이트리스트 (M3 Task 3.2).
2
+
3
+ 경계: filter/derive/map의 expression은 사용자가 작성하는 SQL 조각이다 — 사용자
4
+ 자신의 쿼리이므로 인젝션 방어 대상이 아니라 문서화 대상(dbt와 동일한 신뢰 모델).
5
+ 방어 대상은 식별자·타입처럼 구조를 깨는 위치다.
6
+ """
7
+
8
+ from arsenal_core.errors import FatalError
9
+
10
+
11
+ def quote_ident(name: str) -> str:
12
+ return '"' + name.replace('"', '""') + '"'
13
+
14
+
15
+ def quote_str_literal(value: str) -> str:
16
+ """SQL 문자열 리터럴로 안전하게 인용한다 (작은따옴표 이스케이프).
17
+
18
+ 경로(input/output) 등 우리가 직접 SQL 리터럴에 보간하는 값에 사용한다 — 사용자가
19
+ 작성한 filter/derive 표현식(위 모듈 docstring 참조)과는 다른 신뢰 경계다.
20
+ """
21
+ return "'" + value.replace("'", "''") + "'"
22
+
23
+
24
+ _CAST_TYPES = { # cast step 타입 화이트리스트 — 임의 문자열을 타입 위치에 두지 않는다
25
+ "bigint": "BIGINT",
26
+ "integer": "INTEGER",
27
+ "double": "DOUBLE",
28
+ "varchar": "VARCHAR",
29
+ "boolean": "BOOLEAN",
30
+ "date": "DATE",
31
+ "timestamp": "TIMESTAMP",
32
+ "decimal": "DECIMAL(38, 9)",
33
+ }
34
+
35
+
36
+ def normalize_type(t: str) -> str:
37
+ try:
38
+ return _CAST_TYPES[t.lower()]
39
+ except KeyError:
40
+ raise FatalError(f"unsupported cast type: {t!r}") from None
@@ -0,0 +1,108 @@
1
+ """steps → SQL 트랜스파일러 (M3 Task 3.3).
2
+
3
+ 각 step이 CTE 하나로 컴파일된다:
4
+ WITH s0 AS (SELECT * FROM read_parquet('...')),
5
+ s1 AS (SELECT * FROM s0 WHERE ...), -- filter
6
+ s2 AS (SELECT * EXCLUDE (a), a AS b FROM s1) -- rename
7
+ SELECT ... FROM s2 -- select
8
+
9
+ 원칙:
10
+ - 생성 SQL은 `gladius compile`로 그대로 노출 (투명성 — 마법 없음)
11
+ - 식별자는 화이트리스트 + 인용, 값은 바인딩 — 인젝션 안전 (hypothesis 퍼징)
12
+ - golden test: 손으로 쓴 SQL과 결과 동치 (docs/05-testing-plan.md)
13
+
14
+ select step은 CTE가 아니라 최종 SELECT로 처리한다. 마지막 위치가 아닌 select는
15
+ 그 지점에서 실제 projection을 수행해(중간 select 허용) 이후 step이 좁혀진 컬럼
16
+ 집합 위에서 동작하게 한다.
17
+ """
18
+
19
+ from collections.abc import Sequence
20
+ from pathlib import Path
21
+
22
+ from gladius.compile.ident import normalize_type, quote_ident, quote_str_literal
23
+ from gladius.spec import (
24
+ CastStep,
25
+ DedupStep,
26
+ DeriveStep,
27
+ FilterStep,
28
+ PythonUdfStep,
29
+ RenameStep,
30
+ SelectStep,
31
+ SqlStep,
32
+ Step,
33
+ TransformSpec,
34
+ )
35
+ from gladius.udf import udf_name
36
+
37
+
38
+ def compile_sql(spec: TransformSpec, input_files: Sequence[Path] | None = None) -> str:
39
+ """TransformSpec을 실행 가능한 DuckDB SQL 문자열로 컴파일한다."""
40
+ source = _input_source(spec, input_files)
41
+ ctes = [f"s0 AS (SELECT * FROM read_parquet({source}, union_by_name=true))"]
42
+ idx = 0
43
+ if spec.map:
44
+ idx += 1
45
+ cols = ", ".join(f"{expr} AS {quote_ident(new)}" for new, expr in spec.map.items())
46
+ ctes.append(f"s{idx} AS (SELECT {cols} FROM s{idx - 1})")
47
+ steps = spec.steps
48
+ for i, step in enumerate(steps):
49
+ is_last = i == len(steps) - 1
50
+ if isinstance(step, SelectStep) and is_last:
51
+ continue # 마지막 select는 CTE가 아니라 최종 SELECT에서 처리
52
+ idx += 1
53
+ ctes.append(f"s{idx} AS ({_compile_step(step, f's{idx - 1}', i)})")
54
+ final = _final_select(steps, f"s{idx}")
55
+ return "WITH " + ",\n".join(ctes) + "\n" + final
56
+
57
+
58
+ def _input_source(spec: TransformSpec, input_files: Sequence[Path] | None) -> str:
59
+ if input_files is None:
60
+ src = str(spec.input).rstrip("/")
61
+ return quote_str_literal(f"{src}/**/*.parquet")
62
+ if len(input_files) == 1:
63
+ return quote_str_literal(str(input_files[0]))
64
+ values = ", ".join(quote_str_literal(str(path)) for path in input_files)
65
+ return "[" + values + "]"
66
+
67
+
68
+ def _compile_step(step: Step, prev: str, step_index: int) -> str:
69
+ match step:
70
+ case FilterStep(filter=cond):
71
+ return f"SELECT * FROM {prev} WHERE {cond}"
72
+ case RenameStep(rename=m):
73
+ ex = ", ".join(quote_ident(o) for o in m)
74
+ al = ", ".join(f"{quote_ident(o)} AS {quote_ident(n)}" for o, n in m.items())
75
+ return f"SELECT * EXCLUDE ({ex}), {al} FROM {prev}"
76
+ case CastStep(cast=m):
77
+ rep = ", ".join(
78
+ f"CAST({quote_ident(c)} AS {normalize_type(t)}) AS {quote_ident(c)}"
79
+ for c, t in m.items()
80
+ )
81
+ return f"SELECT * REPLACE ({rep}) FROM {prev}"
82
+ case DedupStep(dedup=keys):
83
+ part = ", ".join(quote_ident(k) for k in keys)
84
+ return (
85
+ f"SELECT * FROM {prev} QUALIFY row_number() "
86
+ f"OVER (PARTITION BY {part} ORDER BY {part}) = 1"
87
+ )
88
+ case DeriveStep(derive=m):
89
+ dv = ", ".join(f"{expr} AS {quote_ident(n)}" for n, expr in m.items())
90
+ return f"SELECT *, {dv} FROM {prev}"
91
+ case SelectStep(select=cols):
92
+ projected = ", ".join(quote_ident(c) for c in cols)
93
+ return f"SELECT {projected} FROM {prev}"
94
+ case SqlStep(sql=expression):
95
+ return expression.replace("{input}", prev)
96
+ case PythonUdfStep(args=args, output=output):
97
+ arguments = ", ".join(quote_ident(argument) for argument in args)
98
+ return (
99
+ f"SELECT *, {quote_ident(udf_name(step_index))}({arguments}) "
100
+ f"AS {quote_ident(output)} FROM {prev}"
101
+ )
102
+
103
+
104
+ def _final_select(steps: list[Step], last: str) -> str:
105
+ if steps and isinstance(steps[-1], SelectStep):
106
+ cols = ", ".join(quote_ident(c) for c in steps[-1].select)
107
+ return f"SELECT {cols} FROM {last}"
108
+ return f"SELECT * FROM {last}"
gladius/engine.py ADDED
@@ -0,0 +1,182 @@
1
+ """DuckDB 실행 엔진 (M3 Task 3.3~3.6). 엔진은 빌린다 — 우리는 컴파일과 신뢰성만.
2
+
3
+ Parquet in → (compile_sql 결과 실행) → Parquet out. Arrow 경유 zero-copy.
4
+ query()는 즉석 SQL(미니 DWH) — 원클릭 경험의 "아하 모먼트".
5
+ """
6
+
7
+ import hashlib
8
+ import os
9
+ import shutil
10
+ import typing as t
11
+ from pathlib import Path
12
+
13
+ import duckdb
14
+ import pyarrow as pa
15
+
16
+ from arsenal_core.errors import FatalError
17
+ from gladius.compile import compile_sql
18
+ from gladius.compile.ident import quote_str_literal
19
+ from gladius.incremental import (
20
+ IncrementalStore,
21
+ InputFile,
22
+ atomic_replace_directory,
23
+ discover_input_files,
24
+ reset_output,
25
+ state_path,
26
+ transform_hash,
27
+ )
28
+ from gladius.spec import TransformSpec
29
+ from gladius.udf import register_python_udfs
30
+
31
+
32
+ def run_transform(spec: TransformSpec) -> Path:
33
+ """변환을 실행하고 output 경로를 반환한다.
34
+
35
+ 전체 재계산 의미론(P0): 매 실행은 output을 통째로 다시 쓴다. 임시 디렉터리에
36
+ 쓰고 os.replace로 원자 교체 — 실패한 실행이 이전 성공 출력을 훼손하지 않는다.
37
+ """
38
+ if spec.incremental is not None:
39
+ return _run_incremental(spec)
40
+ sql = compile_sql(spec)
41
+ con = duckdb.connect() # in-memory, 상태 없음
42
+ register_python_udfs(con, spec)
43
+ con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
44
+ # spec.output은 str(URI 스킴 보존용) — 파일시스템 연산이 필요한 여기서만 Path로 감싼다.
45
+ output = Path(spec.output)
46
+ tmp = output.with_name(output.name + ".tmp")
47
+ tmp.mkdir(parents=True, exist_ok=True)
48
+ try:
49
+ out_path = quote_str_literal(str(tmp / "part-0.parquet"))
50
+ con.execute(f"COPY ({sql}) TO {out_path} (FORMAT PARQUET, COMPRESSION ZSTD)")
51
+ except duckdb.Error as e:
52
+ raise FatalError(f"transform failed: {e}\n--- compiled SQL ---\n{sql}") from e
53
+ finally:
54
+ con.close()
55
+ if output.exists():
56
+ shutil.rmtree(output) # 변환 출력은 전체 재계산 의미론 (P0)
57
+ os.replace(tmp, output)
58
+ return output
59
+
60
+
61
+ def _run_incremental(spec: TransformSpec) -> Path:
62
+ incremental = spec.incremental
63
+ if incremental is None: # pragma: no cover - guarded by run_transform
64
+ raise FatalError("incremental execution requires an incremental spec")
65
+ files = discover_input_files(spec)
66
+ output = Path(spec.output)
67
+ store = IncrementalStore(state_path(spec))
68
+ try:
69
+ current_hash = transform_hash(spec)
70
+ known = store.signatures()
71
+ current = {item.key: item.signature for item in files}
72
+ changed_existing = any(
73
+ key in known and known[key] != signature for key, signature in current.items()
74
+ )
75
+ removed_input = any(key not in current for key in known)
76
+ if (
77
+ store.spec_hash() != current_hash
78
+ or (not output.exists() and known)
79
+ or changed_existing
80
+ or removed_input
81
+ ):
82
+ reset_output(output)
83
+ store.reset(current_hash)
84
+ known = {}
85
+ pending = [item for item in files if known.get(item.key) != item.signature]
86
+ if not pending:
87
+ output.mkdir(parents=True, exist_ok=True)
88
+ return output
89
+ if not files:
90
+ output.mkdir(parents=True, exist_ok=True)
91
+ return output
92
+
93
+ sql = compile_sql(spec, [item.path for item in pending])
94
+ con = duckdb.connect()
95
+ register_python_udfs(con, spec)
96
+ con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
97
+ tmp_input = output.with_name(output.name + ".input.tmp")
98
+ tmp_output = output.with_name(output.name + ".tmp")
99
+ try:
100
+ if tmp_input.exists():
101
+ shutil.rmtree(tmp_input)
102
+ tmp_input.mkdir(parents=True)
103
+ new_part = tmp_input / "new.parquet"
104
+ con.execute(
105
+ f"COPY ({sql}) TO {quote_str_literal(str(new_part))} "
106
+ "(FORMAT PARQUET, COMPRESSION ZSTD)"
107
+ )
108
+ if incremental.mode == "by_unit":
109
+ _append_incremental_part(output, new_part, pending)
110
+ else:
111
+ _merge_incremental_output(con, output, new_part, tmp_output, incremental.key or [])
112
+ store.mark_processed(pending)
113
+ except duckdb.Error as e:
114
+ raise FatalError(
115
+ f"incremental transform failed: {e}\n--- compiled SQL ---\n{sql}"
116
+ ) from e
117
+ finally:
118
+ con.close()
119
+ if tmp_input.exists():
120
+ shutil.rmtree(tmp_input)
121
+ if tmp_output.exists():
122
+ shutil.rmtree(tmp_output)
123
+ return output
124
+ finally:
125
+ store.close()
126
+
127
+
128
+ def _append_incremental_part(output: Path, new_part: Path, pending: list[InputFile]) -> None:
129
+ output.mkdir(parents=True, exist_ok=True)
130
+ batch_id = hashlib.sha256(
131
+ "|".join(f"{item.key}:{item.signature}" for item in pending).encode()
132
+ ).hexdigest()[:16]
133
+ target = output / f"part-{batch_id}.parquet"
134
+ os.replace(new_part, target)
135
+
136
+
137
+ def _merge_incremental_output(
138
+ con: duckdb.DuckDBPyConnection,
139
+ output: Path,
140
+ new_part: Path,
141
+ tmp_output: Path,
142
+ keys: list[str],
143
+ ) -> None:
144
+ new_relation = quote_str_literal(str(new_part))
145
+ if output.exists() and any(output.glob("**/*.parquet")):
146
+ old_relation = quote_str_literal(str(output / "**/*.parquet"))
147
+ partition = ", ".join('"' + key.replace('"', '""') + '"' for key in keys)
148
+ query = (
149
+ "SELECT * EXCLUDE (__gladius_precedence) FROM ("
150
+ f"SELECT *, 0 AS __gladius_precedence FROM "
151
+ f"read_parquet({new_relation}, union_by_name=true) "
152
+ "UNION ALL "
153
+ f"SELECT *, 1 AS __gladius_precedence FROM "
154
+ f"read_parquet({old_relation}, union_by_name=true)"
155
+ f") QUALIFY row_number() OVER (PARTITION BY {partition} "
156
+ "ORDER BY __gladius_precedence) = 1"
157
+ )
158
+ else:
159
+ query = f"SELECT * FROM read_parquet({new_relation}, union_by_name=true)"
160
+ tmp_output.mkdir(parents=True, exist_ok=True)
161
+ con.execute(
162
+ f"COPY ({query}) TO {quote_str_literal(str(tmp_output / 'part-0.parquet'))} "
163
+ "(FORMAT PARQUET, COMPRESSION ZSTD)"
164
+ )
165
+ atomic_replace_directory(tmp_output, output)
166
+
167
+
168
+ def query(sql: str) -> pa.Table:
169
+ """Parquet 레이크에 즉석 SQL — FROM './data/*.parquet' 경로 직접 참조.
170
+
171
+ in-memory DuckDB, 상태 없음. DuckDB 에러는 FatalError로 변환.
172
+ """
173
+ con = duckdb.connect() # in-memory, 상태 없음
174
+ try:
175
+ arrow_table = con.sql( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
176
+ sql
177
+ ).to_arrow_table()
178
+ return t.cast(pa.Table, arrow_table)
179
+ except duckdb.Error as e:
180
+ raise FatalError(str(e)) from e
181
+ finally:
182
+ con.close()
gladius/incremental.py ADDED
@@ -0,0 +1,106 @@
1
+ """Persistent bookkeeping and file discovery for incremental transforms."""
2
+
3
+ import hashlib
4
+ import json
5
+ import os
6
+ import shutil
7
+ import sqlite3
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+ from gladius.spec import TransformSpec
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class InputFile:
16
+ path: Path
17
+ key: str
18
+ signature: str
19
+
20
+
21
+ class IncrementalStore:
22
+ """Small SQLite journal making incremental output restart-safe."""
23
+
24
+ def __init__(self, path: Path) -> None:
25
+ path.parent.mkdir(parents=True, exist_ok=True)
26
+ self._conn = sqlite3.connect(path)
27
+ self._conn.executescript(
28
+ """
29
+ CREATE TABLE IF NOT EXISTS metadata (
30
+ key TEXT PRIMARY KEY,
31
+ value TEXT NOT NULL
32
+ );
33
+ CREATE TABLE IF NOT EXISTS input_files (
34
+ path TEXT PRIMARY KEY,
35
+ signature TEXT NOT NULL,
36
+ updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
37
+ );
38
+ """
39
+ )
40
+ self._conn.commit()
41
+
42
+ def spec_hash(self) -> str | None:
43
+ row = self._conn.execute("SELECT value FROM metadata WHERE key='spec_hash'").fetchone()
44
+ return str(row[0]) if row else None
45
+
46
+ def reset(self, spec_hash: str) -> None:
47
+ self._conn.execute("DELETE FROM input_files")
48
+ self._conn.execute(
49
+ "INSERT INTO metadata(key, value) VALUES ('spec_hash', ?) "
50
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
51
+ (spec_hash,),
52
+ )
53
+ self._conn.commit()
54
+
55
+ def signatures(self) -> dict[str, str]:
56
+ rows = self._conn.execute("SELECT path, signature FROM input_files").fetchall()
57
+ return {str(path): str(signature) for path, signature in rows}
58
+
59
+ def mark_processed(self, files: list[InputFile]) -> None:
60
+ self._conn.executemany(
61
+ "INSERT INTO input_files(path, signature) VALUES (?, ?) "
62
+ "ON CONFLICT(path) DO UPDATE SET signature=excluded.signature, "
63
+ "updated_at=CURRENT_TIMESTAMP",
64
+ [(item.key, item.signature) for item in files],
65
+ )
66
+ self._conn.commit()
67
+
68
+ def close(self) -> None:
69
+ self._conn.close()
70
+
71
+
72
+ def transform_hash(spec: TransformSpec) -> str:
73
+ payload = spec.model_dump(mode="json")
74
+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
75
+ return hashlib.sha256(encoded).hexdigest()
76
+
77
+
78
+ def discover_input_files(spec: TransformSpec) -> list[InputFile]:
79
+ root = Path(spec.input)
80
+ output = Path(spec.output).absolute()
81
+ candidates = [root] if root.is_file() else sorted(root.glob("**/*.parquet"))
82
+ result: list[InputFile] = []
83
+ for path in candidates:
84
+ resolved = path.absolute()
85
+ if resolved == output or output in resolved.parents:
86
+ continue
87
+ stat = path.stat()
88
+ key = str(resolved)
89
+ signature = f"{stat.st_size}:{stat.st_mtime_ns}"
90
+ result.append(InputFile(path, key, signature))
91
+ return result
92
+
93
+
94
+ def state_path(spec: TransformSpec) -> Path:
95
+ return Path(spec.state_dir) / f"{spec.name}.db"
96
+
97
+
98
+ def reset_output(output: Path) -> None:
99
+ if output.exists():
100
+ shutil.rmtree(output)
101
+
102
+
103
+ def atomic_replace_directory(tmp: Path, output: Path) -> None:
104
+ if output.exists():
105
+ shutil.rmtree(output)
106
+ os.replace(tmp, output)
gladius/loader.py ADDED
@@ -0,0 +1,43 @@
1
+ """YAML → 검증된 TransformSpec (M3 Task 3.5). arsenal-core 로더 패턴 재사용.
2
+
3
+ - ${VAR} 환경변수 치환 (비밀은 YAML에 두지 않는다)
4
+ - 검증 실패 시 필드 경로가 담긴 FatalError
5
+ """
6
+
7
+ import os
8
+ import re
9
+ from pathlib import Path
10
+
11
+ import yaml
12
+ from pydantic import ValidationError
13
+
14
+ from arsenal_core.errors import FatalError
15
+ from gladius.spec import TransformSpec
16
+
17
+ _ENV_PATTERN = re.compile(r"\$\{(\w+)\}")
18
+
19
+
20
+ def _substitute_env(text: str) -> str:
21
+ def repl(m: re.Match[str]) -> str:
22
+ value = os.environ.get(m.group(1))
23
+ if value is None:
24
+ raise FatalError(f"environment variable not set: {m.group(1)}")
25
+ return value
26
+
27
+ return _ENV_PATTERN.sub(repl, text)
28
+
29
+
30
+ def load_transform(path: Path) -> TransformSpec:
31
+ try:
32
+ text = path.read_text(encoding="utf-8")
33
+ except FileNotFoundError as e:
34
+ raise FatalError(f"spec file not found: {path}") from e
35
+ except OSError as e:
36
+ raise FatalError(f"cannot read spec file {path}: {e}") from e
37
+
38
+ raw = yaml.safe_load(_substitute_env(text))
39
+ try:
40
+ return TransformSpec.model_validate(raw)
41
+ except ValidationError as e:
42
+ lines = [f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in e.errors()]
43
+ raise FatalError(f"invalid transform spec {path}:\n " + "\n ".join(lines)) from e
gladius/py.typed ADDED
File without changes
gladius/spec.py ADDED
@@ -0,0 +1,149 @@
1
+ """변환 YAML의 Pydantic 모델 — map/steps 선언 (M3 Task 3.1).
2
+
3
+ 하강 경로 (설계 원칙 5): map → steps → SQL(`sql:` step) → Python(`python:` UDF).
4
+ 각 step은 트랜스파일러에서 CTE 하나로 컴파일된다 (docs/02-architecture.md).
5
+ 증분 변환(P1): `incremental: {mode: by_unit|by_key}`로 신규 입력만 재계산한다 (docs/07).
6
+ """
7
+
8
+ from pathlib import Path
9
+ from typing import Literal
10
+
11
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
12
+
13
+
14
+ class _Frozen(BaseModel):
15
+ model_config = ConfigDict(frozen=True, extra="forbid")
16
+
17
+
18
+ class FilterStep(_Frozen):
19
+ filter: str # SQL WHERE 식 (예: "state = 'open'")
20
+
21
+
22
+ class RenameStep(_Frozen):
23
+ rename: dict[str, str] # {old: new}
24
+
25
+
26
+ class CastStep(_Frozen):
27
+ cast: dict[str, str] # {column: type} — DuckDB 타입명
28
+
29
+
30
+ class SelectStep(_Frozen):
31
+ select: list[str] # 최종 컬럼 목록
32
+
33
+
34
+ class DedupStep(_Frozen):
35
+ dedup: list[str] # 중복 판정 키 컬럼
36
+
37
+
38
+ class DeriveStep(_Frozen):
39
+ derive: dict[str, str] # {new_column: SQL 식}
40
+
41
+
42
+ class SqlStep(_Frozen):
43
+ """Raw SQL escape hatch; ``{input}`` is replaced with the previous CTE."""
44
+
45
+ sql: str
46
+
47
+ @field_validator("sql")
48
+ @classmethod
49
+ def _must_reference_input(cls, value: str) -> str:
50
+ if not value.strip():
51
+ raise ValueError("sql step must not be empty")
52
+ if value.count("{input}") != 1:
53
+ raise ValueError("sql step must contain exactly one {input} placeholder")
54
+ if ";" in value:
55
+ raise ValueError("sql step must contain one SELECT expression without ';'")
56
+ return value
57
+
58
+
59
+ class PythonUdfStep(_Frozen):
60
+ """Register a user function and append its result as a new column."""
61
+
62
+ python: str
63
+ args: list[str] = Field(min_length=1)
64
+ output: str
65
+ arg_types: list[str] | None = None
66
+ return_type: str = "VARCHAR"
67
+
68
+ @field_validator("python")
69
+ @classmethod
70
+ def _target_is_importable_shape(cls, value: str) -> str:
71
+ if value.count(":") != 1:
72
+ raise ValueError("python UDF target must use module:function form")
73
+ module, function = value.split(":")
74
+ if not module or not function:
75
+ raise ValueError("python UDF target must use module:function form")
76
+ return value
77
+
78
+ @model_validator(mode="after")
79
+ def _validate_types(self) -> "PythonUdfStep":
80
+ if self.arg_types is not None and len(self.arg_types) != len(self.args):
81
+ raise ValueError("python UDF arg_types must match args length")
82
+ if not self.output:
83
+ raise ValueError("python UDF output must not be empty")
84
+ return self
85
+
86
+
87
+ Step = (
88
+ FilterStep
89
+ | RenameStep
90
+ | CastStep
91
+ | SelectStep
92
+ | DedupStep
93
+ | DeriveStep
94
+ | SqlStep
95
+ | PythonUdfStep
96
+ )
97
+
98
+
99
+ class IncrementalSpec(_Frozen):
100
+ """Incremental execution mode and optional upsert key."""
101
+
102
+ mode: Literal["by_unit", "by_key"]
103
+ key: list[str] | None = Field(
104
+ default=None,
105
+ description="Columns used to keep the newest transformed row in by_key mode.",
106
+ )
107
+
108
+ @model_validator(mode="after")
109
+ def _key_required_for_by_key(self) -> "IncrementalSpec":
110
+ if self.mode == "by_key" and not self.key:
111
+ raise ValueError("incremental.by_key requires a non-empty key")
112
+ return self
113
+
114
+
115
+ class TransformSpec(_Frozen):
116
+ name: str
117
+ # Parquet 디렉터리/파일 경로 (Pugio 산출물 — Arrow/Parquet 허브).
118
+ # str로 보관 — Path로 파싱하면 "./in" 같은 상대경로의 선행 "./"가 정규화로
119
+ # 사라져 컴파일된 SQL 리터럴이 원본 스펙과 달라진다(트랜스파일러 투명성 원칙 위반).
120
+ input: str
121
+ map: dict[str, str] | None = None # {new_column: source_expr} — steps보다 먼저 적용
122
+ steps: list[Step] = []
123
+ incremental: IncrementalSpec | None = None
124
+ state_dir: str = ".gladius"
125
+ # 출력 디렉터리. str로 보관 — Path로 파싱하면 "s3://bucket/x" 같은 URI 스킴이
126
+ # "s3:/bucket/x"로 정규화되어 훼손된다 (P1 httpfs/S3 싱크가 이 필드를 그대로
127
+ # 쓴다). 엔진이 mkdir/os.replace/rmtree 같은 파일시스템 연산이 필요한 지점에서
128
+ # Path(...)로 감싸 해석한다.
129
+ output: str
130
+
131
+ @field_validator("input", "output", "state_dir", mode="before")
132
+ @classmethod
133
+ def _coerce_path_fields_to_str(cls, v: object) -> object:
134
+ if isinstance(v, Path):
135
+ return str(v)
136
+ return v
137
+
138
+ @model_validator(mode="after")
139
+ def _require_map_or_steps(self) -> "TransformSpec":
140
+ if not self.map and not self.steps:
141
+ raise ValueError("transform spec requires at least one of 'map' or 'steps'")
142
+ return self
143
+
144
+ @field_validator("name")
145
+ @classmethod
146
+ def _name_is_path_safe(cls, value: str) -> str:
147
+ if "/" in value or "\\" in value or ".." in value:
148
+ raise ValueError("transform name must not contain '/', '\\\\', or '..'")
149
+ return value
gladius/udf.py ADDED
@@ -0,0 +1,52 @@
1
+ """Python UDF loading and DuckDB registration for Gladius."""
2
+
3
+ import importlib
4
+ from collections.abc import Callable
5
+ from typing import Any
6
+
7
+ import duckdb
8
+
9
+ from arsenal_core.errors import FatalError
10
+ from gladius.compile.ident import normalize_type
11
+ from gladius.spec import PythonUdfStep, TransformSpec
12
+
13
+
14
+ def udf_name(index: int) -> str:
15
+ """Return the deterministic, private SQL name for a Python UDF step."""
16
+ return f"__gladius_python_udf_{index}"
17
+
18
+
19
+ def register_python_udfs(con: duckdb.DuckDBPyConnection, spec: TransformSpec) -> None:
20
+ """Load and register all Python UDFs declared in ``spec``."""
21
+ for index, step in enumerate(spec.steps):
22
+ if not isinstance(step, PythonUdfStep):
23
+ continue
24
+ function = _load_function(step.python)
25
+ parameters = (
26
+ [normalize_type(item) for item in step.arg_types]
27
+ if step.arg_types is not None
28
+ else None
29
+ )
30
+ try:
31
+ kwargs: dict[str, Any] = {"return_type": normalize_type(step.return_type)}
32
+ if parameters is not None:
33
+ kwargs["parameters"] = parameters
34
+ con.create_function(udf_name(index), function, **kwargs)
35
+ except duckdb.Error as e:
36
+ raise FatalError(
37
+ f"cannot register Python UDF {step.python!r}; "
38
+ "install de-gladius[python] and check the function signature: "
39
+ f"{e}"
40
+ ) from e
41
+
42
+
43
+ def _load_function(target: str) -> Callable[..., Any]:
44
+ module_name, function_name = target.split(":", 1)
45
+ try:
46
+ module = importlib.import_module(module_name)
47
+ function = getattr(module, function_name)
48
+ except (ImportError, AttributeError) as e:
49
+ raise FatalError(f"cannot load Python UDF {target!r}: {e}") from e
50
+ if not callable(function):
51
+ raise FatalError(f"Python UDF target is not callable: {target!r}")
52
+ return function