de-gladius 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_gladius-0.2.2.dist-info/METADATA +68 -0
- de_gladius-0.2.2.dist-info/RECORD +15 -0
- de_gladius-0.2.2.dist-info/WHEEL +4 -0
- de_gladius-0.2.2.dist-info/entry_points.txt +2 -0
- gladius/__init__.py +12 -0
- gladius/cli.py +78 -0
- gladius/compile/__init__.py +3 -0
- gladius/compile/ident.py +40 -0
- gladius/compile/transpiler.py +108 -0
- gladius/engine.py +182 -0
- gladius/incremental.py +106 -0
- gladius/loader.py +43 -0
- gladius/py.typed +0 -0
- gladius/spec.py +149 -0
- gladius/udf.py +52 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: de-gladius
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Gladius — 변환·쿼리. map/steps 선언을 SQL로 컴파일해 DuckDB에서 실행
|
|
5
|
+
Project-URL: Homepage, https://github.com/KooEric/de-arsenal
|
|
6
|
+
Project-URL: Repository, https://github.com/KooEric/de-arsenal
|
|
7
|
+
Project-URL: PyPI, https://pypi.org/project/de-gladius/
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: arsenal-core<0.3,>=0.2
|
|
18
|
+
Requires-Dist: duckdb>=1.0
|
|
19
|
+
Requires-Dist: pyarrow>=17
|
|
20
|
+
Requires-Dist: typer>=0.12
|
|
21
|
+
Provides-Extra: python
|
|
22
|
+
Requires-Dist: numpy>=1.26; extra == 'python'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# Gladius (`de-gladius`)
|
|
26
|
+
|
|
27
|
+
> 군단병 주력검. 가장 자주 쓰는 변환·쿼리 도구.
|
|
28
|
+
|
|
29
|
+
선언형 변환(`map`/`steps`)을 SQL로 컴파일해 DuckDB에서 실행하는 엔진. 필터·
|
|
30
|
+
리네임·캐스팅·정렬·중복 제거 같은 흔한 변환을 YAML 레시피로 표현하면, 손으로
|
|
31
|
+
쓴 SQL과 동일한 결과를 벡터화 엔진으로 빠르게 낸다. `gladius compile`로 생성된
|
|
32
|
+
SQL을 그대로 확인할 수 있어 마법이 없다. `gladius query`로 수집 결과 Parquet에
|
|
33
|
+
바로 SQL을 던지는 미니 DWH로도 쓴다.
|
|
34
|
+
|
|
35
|
+
## 설치
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install de-gladius
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
배포명은 `de-gladius`지만 Python import와 CLI는 기존처럼 `gladius`다.
|
|
42
|
+
|
|
43
|
+
## 사용 예
|
|
44
|
+
|
|
45
|
+
```yaml
|
|
46
|
+
# transform.yaml
|
|
47
|
+
name: clean-issues
|
|
48
|
+
input: ./data/issues
|
|
49
|
+
steps:
|
|
50
|
+
- filter: "state = 'open'"
|
|
51
|
+
- rename: { created_at: opened_at }
|
|
52
|
+
- cast: { number: bigint }
|
|
53
|
+
- select: [number, title, opened_at, user]
|
|
54
|
+
output: ./data/issues_clean
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
gladius compile transform.yaml # 생성될 SQL을 미리 확인
|
|
59
|
+
gladius run transform.yaml # Parquet in → DuckDB → Parquet out
|
|
60
|
+
gladius query "SELECT count(*) FROM './data/issues_clean/*.parquet'"
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## 문서
|
|
64
|
+
|
|
65
|
+
- [docs/01-scope.md](../../docs/01-scope.md) — Gladius가 다루는 범위(M3)
|
|
66
|
+
- [docs/08-limits.md](../../docs/08-limits.md) — 단일 노드 처리 envelope(벤치마크 실측)
|
|
67
|
+
- [benchmarks/RESULTS.md](../../benchmarks/RESULTS.md) — 변환 벤치마크 수치
|
|
68
|
+
- [저장소 루트 README](../../README.md)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
gladius/__init__.py,sha256=2fcA-JDV1MgDW_8xJz2vgJrVL5k8gxPvg50Gt5sG5cw,372
|
|
2
|
+
gladius/cli.py,sha256=ExWjSlfKrRnmBw8qv5jtv5Rr5U1bhztZV7KRcli-CKY,2447
|
|
3
|
+
gladius/engine.py,sha256=Bt6b2RhxsMhRZRHTyFG4yKoTiQNJxPWLo27bRn0RR40,6888
|
|
4
|
+
gladius/incremental.py,sha256=tViLkG8xWZC8cpL9YR1ns5owA4p_gAstf1t9P2NRBRI,3332
|
|
5
|
+
gladius/loader.py,sha256=erH-BTzPhyHB2ofNMofIKTQwmNR_14zobKSIxRpM6vo,1354
|
|
6
|
+
gladius/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
gladius/spec.py,sha256=RzEIO1RdccXmdvjt5NU2cFcTPVflr8CDUeClIqc3GVk,5006
|
|
8
|
+
gladius/udf.py,sha256=eEyPrrDJaeD4TVYB0wpuZA29H4OPxtrUz2dIYBuD4dE,1914
|
|
9
|
+
gladius/compile/__init__.py,sha256=_d4vxzVOi_e2R04-U7HH4VkHAZxrE0GGX5C_yj9IjVU,78
|
|
10
|
+
gladius/compile/ident.py,sha256=Gdb8ApQXWu3IhhJbwLHlNQTRwu_GbOotA2sRSlcSj8s,1408
|
|
11
|
+
gladius/compile/transpiler.py,sha256=0YwAhiiFbUoMCdoh5j1mZjEJlEwRzD-v7gRX-kAOcp4,4454
|
|
12
|
+
de_gladius-0.2.2.dist-info/METADATA,sha256=RPGm5kRHhxTd3AyN17B-lt8H8gjx8fqYfeJ_ol_Mg6g,2485
|
|
13
|
+
de_gladius-0.2.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
14
|
+
de_gladius-0.2.2.dist-info/entry_points.txt,sha256=P7moccpxo_Mx0BY2iuEjv7b5Q1Es_V_EE7CH_bUh6ms,44
|
|
15
|
+
de_gladius-0.2.2.dist-info/RECORD,,
|
gladius/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Gladius — 변환·쿼리 (핵심 처리).
|
|
2
|
+
|
|
3
|
+
map/steps 선언 → SQL 컴파일 → DuckDB 벡터화 실행. 단일 노드 수백 GB.
|
|
4
|
+
구조: spec.py compile/ engine.py cli.py (docs/02-architecture.md)
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
__version__ = version("de-gladius")
|
|
11
|
+
except PackageNotFoundError:
|
|
12
|
+
__version__ = "0.2.2"
|
gladius/cli.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Gladius CLI (M3 Task 3.6).
|
|
2
|
+
|
|
3
|
+
compile: 생성될 SQL을 그대로 출력 — 투명성 커맨드, 마법 없음.
|
|
4
|
+
run: 변환 실행.
|
|
5
|
+
query: 수집·변환 결과 Parquet에 즉석 SQL — 미니 DWH의 "아하 모먼트".
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import duckdb
|
|
13
|
+
import pyarrow.csv as pa_csv
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from arsenal_core.errors import ArsenalError
|
|
17
|
+
from gladius.compile import compile_sql
|
|
18
|
+
from gladius.engine import query as run_query
|
|
19
|
+
from gladius.engine import run_transform
|
|
20
|
+
from gladius.loader import load_transform
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(help="Gladius — 변환·쿼리. 선언이 SQL로 컴파일된다.")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.command(name="compile")
|
|
26
|
+
def compile_cmd(spec_path: Path) -> None:
|
|
27
|
+
"""변환 스펙이 생성할 SQL을 출력한다 (실행하지 않음)."""
|
|
28
|
+
try:
|
|
29
|
+
spec = load_transform(spec_path)
|
|
30
|
+
sql = compile_sql(spec)
|
|
31
|
+
except ArsenalError as e:
|
|
32
|
+
typer.echo(f"error: {e}", err=True)
|
|
33
|
+
raise typer.Exit(1) from e
|
|
34
|
+
typer.echo(sql)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@app.command(name="run")
|
|
38
|
+
def run_cmd(spec_path: Path) -> None:
|
|
39
|
+
"""변환 실행: Parquet in → DuckDB → Parquet out."""
|
|
40
|
+
try:
|
|
41
|
+
spec = load_transform(spec_path)
|
|
42
|
+
out = run_transform(spec)
|
|
43
|
+
except ArsenalError as e:
|
|
44
|
+
typer.echo(f"error: {e}", err=True)
|
|
45
|
+
raise typer.Exit(1) from e
|
|
46
|
+
typer.echo(f"done: {out}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.command(name="query")
|
|
50
|
+
def query_cmd(
|
|
51
|
+
sql: str,
|
|
52
|
+
fmt: str = typer.Option("table", "--format", help="table | csv | jsonl"),
|
|
53
|
+
limit: int = typer.Option(0, help="0 = 제한 없음"),
|
|
54
|
+
) -> None:
|
|
55
|
+
"""수집·변환 결과 Parquet에 즉석 SQL (미니 DWH)."""
|
|
56
|
+
effective_sql = sql if limit <= 0 else f"SELECT * FROM ({sql}) AS _gladius_query LIMIT {limit}"
|
|
57
|
+
try:
|
|
58
|
+
result = run_query(effective_sql)
|
|
59
|
+
except ArsenalError as e:
|
|
60
|
+
typer.echo(f"error: {e}", err=True)
|
|
61
|
+
raise typer.Exit(1) from e
|
|
62
|
+
|
|
63
|
+
if fmt == "table":
|
|
64
|
+
duckdb.sql("SELECT * FROM result").show() # pyright: ignore[reportUnknownMemberType]
|
|
65
|
+
elif fmt == "csv":
|
|
66
|
+
pa_csv.write_csv( # pyright: ignore[reportPrivateImportUsage, reportUnknownMemberType]
|
|
67
|
+
result, sys.stdout.buffer
|
|
68
|
+
)
|
|
69
|
+
elif fmt == "jsonl":
|
|
70
|
+
for row in result.to_pylist():
|
|
71
|
+
typer.echo(json.dumps(row))
|
|
72
|
+
else:
|
|
73
|
+
typer.echo(f"error: unsupported --format: {fmt!r} (table | csv | jsonl)", err=True)
|
|
74
|
+
raise typer.Exit(1)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
app()
|
gladius/compile/ident.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""식별자 안전 처리 — quoting + cast 타입 화이트리스트 (M3 Task 3.2).
|
|
2
|
+
|
|
3
|
+
경계: filter/derive/map의 expression은 사용자가 작성하는 SQL 조각이다 — 사용자
|
|
4
|
+
자신의 쿼리이므로 인젝션 방어 대상이 아니라 문서화 대상(dbt와 동일한 신뢰 모델).
|
|
5
|
+
방어 대상은 식별자·타입처럼 구조를 깨는 위치다.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from arsenal_core.errors import FatalError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def quote_ident(name: str) -> str:
|
|
12
|
+
return '"' + name.replace('"', '""') + '"'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def quote_str_literal(value: str) -> str:
|
|
16
|
+
"""SQL 문자열 리터럴로 안전하게 인용한다 (작은따옴표 이스케이프).
|
|
17
|
+
|
|
18
|
+
경로(input/output) 등 우리가 직접 SQL 리터럴에 보간하는 값에 사용한다 — 사용자가
|
|
19
|
+
작성한 filter/derive 표현식(위 모듈 docstring 참조)과는 다른 신뢰 경계다.
|
|
20
|
+
"""
|
|
21
|
+
return "'" + value.replace("'", "''") + "'"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_CAST_TYPES = { # cast step 타입 화이트리스트 — 임의 문자열을 타입 위치에 두지 않는다
|
|
25
|
+
"bigint": "BIGINT",
|
|
26
|
+
"integer": "INTEGER",
|
|
27
|
+
"double": "DOUBLE",
|
|
28
|
+
"varchar": "VARCHAR",
|
|
29
|
+
"boolean": "BOOLEAN",
|
|
30
|
+
"date": "DATE",
|
|
31
|
+
"timestamp": "TIMESTAMP",
|
|
32
|
+
"decimal": "DECIMAL(38, 9)",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def normalize_type(t: str) -> str:
|
|
37
|
+
try:
|
|
38
|
+
return _CAST_TYPES[t.lower()]
|
|
39
|
+
except KeyError:
|
|
40
|
+
raise FatalError(f"unsupported cast type: {t!r}") from None
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""steps → SQL 트랜스파일러 (M3 Task 3.3).
|
|
2
|
+
|
|
3
|
+
각 step이 CTE 하나로 컴파일된다:
|
|
4
|
+
WITH s0 AS (SELECT * FROM read_parquet('...')),
|
|
5
|
+
s1 AS (SELECT * FROM s0 WHERE ...), -- filter
|
|
6
|
+
s2 AS (SELECT * EXCLUDE (a), a AS b FROM s1) -- rename
|
|
7
|
+
SELECT ... FROM s2 -- select
|
|
8
|
+
|
|
9
|
+
원칙:
|
|
10
|
+
- 생성 SQL은 `gladius compile`로 그대로 노출 (투명성 — 마법 없음)
|
|
11
|
+
- 식별자는 화이트리스트 + 인용, 값은 바인딩 — 인젝션 안전 (hypothesis 퍼징)
|
|
12
|
+
- golden test: 손으로 쓴 SQL과 결과 동치 (docs/05-testing-plan.md)
|
|
13
|
+
|
|
14
|
+
select step은 CTE가 아니라 최종 SELECT로 처리한다. 마지막 위치가 아닌 select는
|
|
15
|
+
그 지점에서 실제 projection을 수행해(중간 select 허용) 이후 step이 좁혀진 컬럼
|
|
16
|
+
집합 위에서 동작하게 한다.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from collections.abc import Sequence
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from gladius.compile.ident import normalize_type, quote_ident, quote_str_literal
|
|
23
|
+
from gladius.spec import (
|
|
24
|
+
CastStep,
|
|
25
|
+
DedupStep,
|
|
26
|
+
DeriveStep,
|
|
27
|
+
FilterStep,
|
|
28
|
+
PythonUdfStep,
|
|
29
|
+
RenameStep,
|
|
30
|
+
SelectStep,
|
|
31
|
+
SqlStep,
|
|
32
|
+
Step,
|
|
33
|
+
TransformSpec,
|
|
34
|
+
)
|
|
35
|
+
from gladius.udf import udf_name
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def compile_sql(spec: TransformSpec, input_files: Sequence[Path] | None = None) -> str:
|
|
39
|
+
"""TransformSpec을 실행 가능한 DuckDB SQL 문자열로 컴파일한다."""
|
|
40
|
+
source = _input_source(spec, input_files)
|
|
41
|
+
ctes = [f"s0 AS (SELECT * FROM read_parquet({source}, union_by_name=true))"]
|
|
42
|
+
idx = 0
|
|
43
|
+
if spec.map:
|
|
44
|
+
idx += 1
|
|
45
|
+
cols = ", ".join(f"{expr} AS {quote_ident(new)}" for new, expr in spec.map.items())
|
|
46
|
+
ctes.append(f"s{idx} AS (SELECT {cols} FROM s{idx - 1})")
|
|
47
|
+
steps = spec.steps
|
|
48
|
+
for i, step in enumerate(steps):
|
|
49
|
+
is_last = i == len(steps) - 1
|
|
50
|
+
if isinstance(step, SelectStep) and is_last:
|
|
51
|
+
continue # 마지막 select는 CTE가 아니라 최종 SELECT에서 처리
|
|
52
|
+
idx += 1
|
|
53
|
+
ctes.append(f"s{idx} AS ({_compile_step(step, f's{idx - 1}', i)})")
|
|
54
|
+
final = _final_select(steps, f"s{idx}")
|
|
55
|
+
return "WITH " + ",\n".join(ctes) + "\n" + final
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _input_source(spec: TransformSpec, input_files: Sequence[Path] | None) -> str:
|
|
59
|
+
if input_files is None:
|
|
60
|
+
src = str(spec.input).rstrip("/")
|
|
61
|
+
return quote_str_literal(f"{src}/**/*.parquet")
|
|
62
|
+
if len(input_files) == 1:
|
|
63
|
+
return quote_str_literal(str(input_files[0]))
|
|
64
|
+
values = ", ".join(quote_str_literal(str(path)) for path in input_files)
|
|
65
|
+
return "[" + values + "]"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _compile_step(step: Step, prev: str, step_index: int) -> str:
|
|
69
|
+
match step:
|
|
70
|
+
case FilterStep(filter=cond):
|
|
71
|
+
return f"SELECT * FROM {prev} WHERE {cond}"
|
|
72
|
+
case RenameStep(rename=m):
|
|
73
|
+
ex = ", ".join(quote_ident(o) for o in m)
|
|
74
|
+
al = ", ".join(f"{quote_ident(o)} AS {quote_ident(n)}" for o, n in m.items())
|
|
75
|
+
return f"SELECT * EXCLUDE ({ex}), {al} FROM {prev}"
|
|
76
|
+
case CastStep(cast=m):
|
|
77
|
+
rep = ", ".join(
|
|
78
|
+
f"CAST({quote_ident(c)} AS {normalize_type(t)}) AS {quote_ident(c)}"
|
|
79
|
+
for c, t in m.items()
|
|
80
|
+
)
|
|
81
|
+
return f"SELECT * REPLACE ({rep}) FROM {prev}"
|
|
82
|
+
case DedupStep(dedup=keys):
|
|
83
|
+
part = ", ".join(quote_ident(k) for k in keys)
|
|
84
|
+
return (
|
|
85
|
+
f"SELECT * FROM {prev} QUALIFY row_number() "
|
|
86
|
+
f"OVER (PARTITION BY {part} ORDER BY {part}) = 1"
|
|
87
|
+
)
|
|
88
|
+
case DeriveStep(derive=m):
|
|
89
|
+
dv = ", ".join(f"{expr} AS {quote_ident(n)}" for n, expr in m.items())
|
|
90
|
+
return f"SELECT *, {dv} FROM {prev}"
|
|
91
|
+
case SelectStep(select=cols):
|
|
92
|
+
projected = ", ".join(quote_ident(c) for c in cols)
|
|
93
|
+
return f"SELECT {projected} FROM {prev}"
|
|
94
|
+
case SqlStep(sql=expression):
|
|
95
|
+
return expression.replace("{input}", prev)
|
|
96
|
+
case PythonUdfStep(args=args, output=output):
|
|
97
|
+
arguments = ", ".join(quote_ident(argument) for argument in args)
|
|
98
|
+
return (
|
|
99
|
+
f"SELECT *, {quote_ident(udf_name(step_index))}({arguments}) "
|
|
100
|
+
f"AS {quote_ident(output)} FROM {prev}"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _final_select(steps: list[Step], last: str) -> str:
|
|
105
|
+
if steps and isinstance(steps[-1], SelectStep):
|
|
106
|
+
cols = ", ".join(quote_ident(c) for c in steps[-1].select)
|
|
107
|
+
return f"SELECT {cols} FROM {last}"
|
|
108
|
+
return f"SELECT * FROM {last}"
|
gladius/engine.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""DuckDB 실행 엔진 (M3 Task 3.3~3.6). 엔진은 빌린다 — 우리는 컴파일과 신뢰성만.
|
|
2
|
+
|
|
3
|
+
Parquet in → (compile_sql 결과 실행) → Parquet out. Arrow 경유 zero-copy.
|
|
4
|
+
query()는 즉석 SQL(미니 DWH) — 원클릭 경험의 "아하 모먼트".
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
import typing as t
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import duckdb
|
|
14
|
+
import pyarrow as pa
|
|
15
|
+
|
|
16
|
+
from arsenal_core.errors import FatalError
|
|
17
|
+
from gladius.compile import compile_sql
|
|
18
|
+
from gladius.compile.ident import quote_str_literal
|
|
19
|
+
from gladius.incremental import (
|
|
20
|
+
IncrementalStore,
|
|
21
|
+
InputFile,
|
|
22
|
+
atomic_replace_directory,
|
|
23
|
+
discover_input_files,
|
|
24
|
+
reset_output,
|
|
25
|
+
state_path,
|
|
26
|
+
transform_hash,
|
|
27
|
+
)
|
|
28
|
+
from gladius.spec import TransformSpec
|
|
29
|
+
from gladius.udf import register_python_udfs
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run_transform(spec: TransformSpec) -> Path:
|
|
33
|
+
"""변환을 실행하고 output 경로를 반환한다.
|
|
34
|
+
|
|
35
|
+
전체 재계산 의미론(P0): 매 실행은 output을 통째로 다시 쓴다. 임시 디렉터리에
|
|
36
|
+
쓰고 os.replace로 원자 교체 — 실패한 실행이 이전 성공 출력을 훼손하지 않는다.
|
|
37
|
+
"""
|
|
38
|
+
if spec.incremental is not None:
|
|
39
|
+
return _run_incremental(spec)
|
|
40
|
+
sql = compile_sql(spec)
|
|
41
|
+
con = duckdb.connect() # in-memory, 상태 없음
|
|
42
|
+
register_python_udfs(con, spec)
|
|
43
|
+
con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
|
|
44
|
+
# spec.output은 str(URI 스킴 보존용) — 파일시스템 연산이 필요한 여기서만 Path로 감싼다.
|
|
45
|
+
output = Path(spec.output)
|
|
46
|
+
tmp = output.with_name(output.name + ".tmp")
|
|
47
|
+
tmp.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
try:
|
|
49
|
+
out_path = quote_str_literal(str(tmp / "part-0.parquet"))
|
|
50
|
+
con.execute(f"COPY ({sql}) TO {out_path} (FORMAT PARQUET, COMPRESSION ZSTD)")
|
|
51
|
+
except duckdb.Error as e:
|
|
52
|
+
raise FatalError(f"transform failed: {e}\n--- compiled SQL ---\n{sql}") from e
|
|
53
|
+
finally:
|
|
54
|
+
con.close()
|
|
55
|
+
if output.exists():
|
|
56
|
+
shutil.rmtree(output) # 변환 출력은 전체 재계산 의미론 (P0)
|
|
57
|
+
os.replace(tmp, output)
|
|
58
|
+
return output
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _run_incremental(spec: TransformSpec) -> Path:
|
|
62
|
+
incremental = spec.incremental
|
|
63
|
+
if incremental is None: # pragma: no cover - guarded by run_transform
|
|
64
|
+
raise FatalError("incremental execution requires an incremental spec")
|
|
65
|
+
files = discover_input_files(spec)
|
|
66
|
+
output = Path(spec.output)
|
|
67
|
+
store = IncrementalStore(state_path(spec))
|
|
68
|
+
try:
|
|
69
|
+
current_hash = transform_hash(spec)
|
|
70
|
+
known = store.signatures()
|
|
71
|
+
current = {item.key: item.signature for item in files}
|
|
72
|
+
changed_existing = any(
|
|
73
|
+
key in known and known[key] != signature for key, signature in current.items()
|
|
74
|
+
)
|
|
75
|
+
removed_input = any(key not in current for key in known)
|
|
76
|
+
if (
|
|
77
|
+
store.spec_hash() != current_hash
|
|
78
|
+
or (not output.exists() and known)
|
|
79
|
+
or changed_existing
|
|
80
|
+
or removed_input
|
|
81
|
+
):
|
|
82
|
+
reset_output(output)
|
|
83
|
+
store.reset(current_hash)
|
|
84
|
+
known = {}
|
|
85
|
+
pending = [item for item in files if known.get(item.key) != item.signature]
|
|
86
|
+
if not pending:
|
|
87
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
return output
|
|
89
|
+
if not files:
|
|
90
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
return output
|
|
92
|
+
|
|
93
|
+
sql = compile_sql(spec, [item.path for item in pending])
|
|
94
|
+
con = duckdb.connect()
|
|
95
|
+
register_python_udfs(con, spec)
|
|
96
|
+
con.execute(f"PRAGMA threads={os.cpu_count() or 1}")
|
|
97
|
+
tmp_input = output.with_name(output.name + ".input.tmp")
|
|
98
|
+
tmp_output = output.with_name(output.name + ".tmp")
|
|
99
|
+
try:
|
|
100
|
+
if tmp_input.exists():
|
|
101
|
+
shutil.rmtree(tmp_input)
|
|
102
|
+
tmp_input.mkdir(parents=True)
|
|
103
|
+
new_part = tmp_input / "new.parquet"
|
|
104
|
+
con.execute(
|
|
105
|
+
f"COPY ({sql}) TO {quote_str_literal(str(new_part))} "
|
|
106
|
+
"(FORMAT PARQUET, COMPRESSION ZSTD)"
|
|
107
|
+
)
|
|
108
|
+
if incremental.mode == "by_unit":
|
|
109
|
+
_append_incremental_part(output, new_part, pending)
|
|
110
|
+
else:
|
|
111
|
+
_merge_incremental_output(con, output, new_part, tmp_output, incremental.key or [])
|
|
112
|
+
store.mark_processed(pending)
|
|
113
|
+
except duckdb.Error as e:
|
|
114
|
+
raise FatalError(
|
|
115
|
+
f"incremental transform failed: {e}\n--- compiled SQL ---\n{sql}"
|
|
116
|
+
) from e
|
|
117
|
+
finally:
|
|
118
|
+
con.close()
|
|
119
|
+
if tmp_input.exists():
|
|
120
|
+
shutil.rmtree(tmp_input)
|
|
121
|
+
if tmp_output.exists():
|
|
122
|
+
shutil.rmtree(tmp_output)
|
|
123
|
+
return output
|
|
124
|
+
finally:
|
|
125
|
+
store.close()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _append_incremental_part(output: Path, new_part: Path, pending: list[InputFile]) -> None:
|
|
129
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
130
|
+
batch_id = hashlib.sha256(
|
|
131
|
+
"|".join(f"{item.key}:{item.signature}" for item in pending).encode()
|
|
132
|
+
).hexdigest()[:16]
|
|
133
|
+
target = output / f"part-{batch_id}.parquet"
|
|
134
|
+
os.replace(new_part, target)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _merge_incremental_output(
|
|
138
|
+
con: duckdb.DuckDBPyConnection,
|
|
139
|
+
output: Path,
|
|
140
|
+
new_part: Path,
|
|
141
|
+
tmp_output: Path,
|
|
142
|
+
keys: list[str],
|
|
143
|
+
) -> None:
|
|
144
|
+
new_relation = quote_str_literal(str(new_part))
|
|
145
|
+
if output.exists() and any(output.glob("**/*.parquet")):
|
|
146
|
+
old_relation = quote_str_literal(str(output / "**/*.parquet"))
|
|
147
|
+
partition = ", ".join('"' + key.replace('"', '""') + '"' for key in keys)
|
|
148
|
+
query = (
|
|
149
|
+
"SELECT * EXCLUDE (__gladius_precedence) FROM ("
|
|
150
|
+
f"SELECT *, 0 AS __gladius_precedence FROM "
|
|
151
|
+
f"read_parquet({new_relation}, union_by_name=true) "
|
|
152
|
+
"UNION ALL "
|
|
153
|
+
f"SELECT *, 1 AS __gladius_precedence FROM "
|
|
154
|
+
f"read_parquet({old_relation}, union_by_name=true)"
|
|
155
|
+
f") QUALIFY row_number() OVER (PARTITION BY {partition} "
|
|
156
|
+
"ORDER BY __gladius_precedence) = 1"
|
|
157
|
+
)
|
|
158
|
+
else:
|
|
159
|
+
query = f"SELECT * FROM read_parquet({new_relation}, union_by_name=true)"
|
|
160
|
+
tmp_output.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
con.execute(
|
|
162
|
+
f"COPY ({query}) TO {quote_str_literal(str(tmp_output / 'part-0.parquet'))} "
|
|
163
|
+
"(FORMAT PARQUET, COMPRESSION ZSTD)"
|
|
164
|
+
)
|
|
165
|
+
atomic_replace_directory(tmp_output, output)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def query(sql: str) -> pa.Table:
|
|
169
|
+
"""Parquet 레이크에 즉석 SQL — FROM './data/*.parquet' 경로 직접 참조.
|
|
170
|
+
|
|
171
|
+
in-memory DuckDB, 상태 없음. DuckDB 에러는 FatalError로 변환.
|
|
172
|
+
"""
|
|
173
|
+
con = duckdb.connect() # in-memory, 상태 없음
|
|
174
|
+
try:
|
|
175
|
+
arrow_table = con.sql( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
|
|
176
|
+
sql
|
|
177
|
+
).to_arrow_table()
|
|
178
|
+
return t.cast(pa.Table, arrow_table)
|
|
179
|
+
except duckdb.Error as e:
|
|
180
|
+
raise FatalError(str(e)) from e
|
|
181
|
+
finally:
|
|
182
|
+
con.close()
|
gladius/incremental.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Persistent bookkeeping and file discovery for incremental transforms."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import sqlite3
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from gladius.spec import TransformSpec
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class InputFile:
|
|
16
|
+
path: Path
|
|
17
|
+
key: str
|
|
18
|
+
signature: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class IncrementalStore:
|
|
22
|
+
"""Small SQLite journal making incremental output restart-safe."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, path: Path) -> None:
|
|
25
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
self._conn = sqlite3.connect(path)
|
|
27
|
+
self._conn.executescript(
|
|
28
|
+
"""
|
|
29
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
30
|
+
key TEXT PRIMARY KEY,
|
|
31
|
+
value TEXT NOT NULL
|
|
32
|
+
);
|
|
33
|
+
CREATE TABLE IF NOT EXISTS input_files (
|
|
34
|
+
path TEXT PRIMARY KEY,
|
|
35
|
+
signature TEXT NOT NULL,
|
|
36
|
+
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
37
|
+
);
|
|
38
|
+
"""
|
|
39
|
+
)
|
|
40
|
+
self._conn.commit()
|
|
41
|
+
|
|
42
|
+
def spec_hash(self) -> str | None:
|
|
43
|
+
row = self._conn.execute("SELECT value FROM metadata WHERE key='spec_hash'").fetchone()
|
|
44
|
+
return str(row[0]) if row else None
|
|
45
|
+
|
|
46
|
+
def reset(self, spec_hash: str) -> None:
|
|
47
|
+
self._conn.execute("DELETE FROM input_files")
|
|
48
|
+
self._conn.execute(
|
|
49
|
+
"INSERT INTO metadata(key, value) VALUES ('spec_hash', ?) "
|
|
50
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
51
|
+
(spec_hash,),
|
|
52
|
+
)
|
|
53
|
+
self._conn.commit()
|
|
54
|
+
|
|
55
|
+
def signatures(self) -> dict[str, str]:
|
|
56
|
+
rows = self._conn.execute("SELECT path, signature FROM input_files").fetchall()
|
|
57
|
+
return {str(path): str(signature) for path, signature in rows}
|
|
58
|
+
|
|
59
|
+
def mark_processed(self, files: list[InputFile]) -> None:
|
|
60
|
+
self._conn.executemany(
|
|
61
|
+
"INSERT INTO input_files(path, signature) VALUES (?, ?) "
|
|
62
|
+
"ON CONFLICT(path) DO UPDATE SET signature=excluded.signature, "
|
|
63
|
+
"updated_at=CURRENT_TIMESTAMP",
|
|
64
|
+
[(item.key, item.signature) for item in files],
|
|
65
|
+
)
|
|
66
|
+
self._conn.commit()
|
|
67
|
+
|
|
68
|
+
def close(self) -> None:
|
|
69
|
+
self._conn.close()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def transform_hash(spec: TransformSpec) -> str:
|
|
73
|
+
payload = spec.model_dump(mode="json")
|
|
74
|
+
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
|
75
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def discover_input_files(spec: TransformSpec) -> list[InputFile]:
|
|
79
|
+
root = Path(spec.input)
|
|
80
|
+
output = Path(spec.output).absolute()
|
|
81
|
+
candidates = [root] if root.is_file() else sorted(root.glob("**/*.parquet"))
|
|
82
|
+
result: list[InputFile] = []
|
|
83
|
+
for path in candidates:
|
|
84
|
+
resolved = path.absolute()
|
|
85
|
+
if resolved == output or output in resolved.parents:
|
|
86
|
+
continue
|
|
87
|
+
stat = path.stat()
|
|
88
|
+
key = str(resolved)
|
|
89
|
+
signature = f"{stat.st_size}:{stat.st_mtime_ns}"
|
|
90
|
+
result.append(InputFile(path, key, signature))
|
|
91
|
+
return result
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def state_path(spec: TransformSpec) -> Path:
|
|
95
|
+
return Path(spec.state_dir) / f"{spec.name}.db"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def reset_output(output: Path) -> None:
|
|
99
|
+
if output.exists():
|
|
100
|
+
shutil.rmtree(output)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def atomic_replace_directory(tmp: Path, output: Path) -> None:
|
|
104
|
+
if output.exists():
|
|
105
|
+
shutil.rmtree(output)
|
|
106
|
+
os.replace(tmp, output)
|
gladius/loader.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""YAML → 검증된 TransformSpec (M3 Task 3.5). arsenal-core 로더 패턴 재사용.
|
|
2
|
+
|
|
3
|
+
- ${VAR} 환경변수 치환 (비밀은 YAML에 두지 않는다)
|
|
4
|
+
- 검증 실패 시 필드 경로가 담긴 FatalError
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
from pydantic import ValidationError
|
|
13
|
+
|
|
14
|
+
from arsenal_core.errors import FatalError
|
|
15
|
+
from gladius.spec import TransformSpec
|
|
16
|
+
|
|
17
|
+
_ENV_PATTERN = re.compile(r"\$\{(\w+)\}")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _substitute_env(text: str) -> str:
|
|
21
|
+
def repl(m: re.Match[str]) -> str:
|
|
22
|
+
value = os.environ.get(m.group(1))
|
|
23
|
+
if value is None:
|
|
24
|
+
raise FatalError(f"environment variable not set: {m.group(1)}")
|
|
25
|
+
return value
|
|
26
|
+
|
|
27
|
+
return _ENV_PATTERN.sub(repl, text)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_transform(path: Path) -> TransformSpec:
|
|
31
|
+
try:
|
|
32
|
+
text = path.read_text(encoding="utf-8")
|
|
33
|
+
except FileNotFoundError as e:
|
|
34
|
+
raise FatalError(f"spec file not found: {path}") from e
|
|
35
|
+
except OSError as e:
|
|
36
|
+
raise FatalError(f"cannot read spec file {path}: {e}") from e
|
|
37
|
+
|
|
38
|
+
raw = yaml.safe_load(_substitute_env(text))
|
|
39
|
+
try:
|
|
40
|
+
return TransformSpec.model_validate(raw)
|
|
41
|
+
except ValidationError as e:
|
|
42
|
+
lines = [f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in e.errors()]
|
|
43
|
+
raise FatalError(f"invalid transform spec {path}:\n " + "\n ".join(lines)) from e
|
gladius/py.typed
ADDED
|
File without changes
|
gladius/spec.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""변환 YAML의 Pydantic 모델 — map/steps 선언 (M3 Task 3.1).
|
|
2
|
+
|
|
3
|
+
하강 경로 (설계 원칙 5): map → steps → SQL(`sql:` step) → Python(`python:` UDF).
|
|
4
|
+
각 step은 트랜스파일러에서 CTE 하나로 컴파일된다 (docs/02-architecture.md).
|
|
5
|
+
증분 변환(P1): `incremental: {mode: by_unit|by_key}`로 신규 입력만 재계산한다 (docs/07).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _Frozen(BaseModel):
|
|
15
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FilterStep(_Frozen):
|
|
19
|
+
filter: str # SQL WHERE 식 (예: "state = 'open'")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RenameStep(_Frozen):
|
|
23
|
+
rename: dict[str, str] # {old: new}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CastStep(_Frozen):
|
|
27
|
+
cast: dict[str, str] # {column: type} — DuckDB 타입명
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SelectStep(_Frozen):
|
|
31
|
+
select: list[str] # 최종 컬럼 목록
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DedupStep(_Frozen):
|
|
35
|
+
dedup: list[str] # 중복 판정 키 컬럼
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class DeriveStep(_Frozen):
|
|
39
|
+
derive: dict[str, str] # {new_column: SQL 식}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class SqlStep(_Frozen):
|
|
43
|
+
"""Raw SQL escape hatch; ``{input}`` is replaced with the previous CTE."""
|
|
44
|
+
|
|
45
|
+
sql: str
|
|
46
|
+
|
|
47
|
+
@field_validator("sql")
|
|
48
|
+
@classmethod
|
|
49
|
+
def _must_reference_input(cls, value: str) -> str:
|
|
50
|
+
if not value.strip():
|
|
51
|
+
raise ValueError("sql step must not be empty")
|
|
52
|
+
if value.count("{input}") != 1:
|
|
53
|
+
raise ValueError("sql step must contain exactly one {input} placeholder")
|
|
54
|
+
if ";" in value:
|
|
55
|
+
raise ValueError("sql step must contain one SELECT expression without ';'")
|
|
56
|
+
return value
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class PythonUdfStep(_Frozen):
|
|
60
|
+
"""Register a user function and append its result as a new column."""
|
|
61
|
+
|
|
62
|
+
python: str
|
|
63
|
+
args: list[str] = Field(min_length=1)
|
|
64
|
+
output: str
|
|
65
|
+
arg_types: list[str] | None = None
|
|
66
|
+
return_type: str = "VARCHAR"
|
|
67
|
+
|
|
68
|
+
@field_validator("python")
|
|
69
|
+
@classmethod
|
|
70
|
+
def _target_is_importable_shape(cls, value: str) -> str:
|
|
71
|
+
if value.count(":") != 1:
|
|
72
|
+
raise ValueError("python UDF target must use module:function form")
|
|
73
|
+
module, function = value.split(":")
|
|
74
|
+
if not module or not function:
|
|
75
|
+
raise ValueError("python UDF target must use module:function form")
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
@model_validator(mode="after")
|
|
79
|
+
def _validate_types(self) -> "PythonUdfStep":
|
|
80
|
+
if self.arg_types is not None and len(self.arg_types) != len(self.args):
|
|
81
|
+
raise ValueError("python UDF arg_types must match args length")
|
|
82
|
+
if not self.output:
|
|
83
|
+
raise ValueError("python UDF output must not be empty")
|
|
84
|
+
return self
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
Step = (
|
|
88
|
+
FilterStep
|
|
89
|
+
| RenameStep
|
|
90
|
+
| CastStep
|
|
91
|
+
| SelectStep
|
|
92
|
+
| DedupStep
|
|
93
|
+
| DeriveStep
|
|
94
|
+
| SqlStep
|
|
95
|
+
| PythonUdfStep
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class IncrementalSpec(_Frozen):
|
|
100
|
+
"""Incremental execution mode and optional upsert key."""
|
|
101
|
+
|
|
102
|
+
mode: Literal["by_unit", "by_key"]
|
|
103
|
+
key: list[str] | None = Field(
|
|
104
|
+
default=None,
|
|
105
|
+
description="Columns used to keep the newest transformed row in by_key mode.",
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
@model_validator(mode="after")
|
|
109
|
+
def _key_required_for_by_key(self) -> "IncrementalSpec":
|
|
110
|
+
if self.mode == "by_key" and not self.key:
|
|
111
|
+
raise ValueError("incremental.by_key requires a non-empty key")
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class TransformSpec(_Frozen):
|
|
116
|
+
name: str
|
|
117
|
+
# Parquet 디렉터리/파일 경로 (Pugio 산출물 — Arrow/Parquet 허브).
|
|
118
|
+
# str로 보관 — Path로 파싱하면 "./in" 같은 상대경로의 선행 "./"가 정규화로
|
|
119
|
+
# 사라져 컴파일된 SQL 리터럴이 원본 스펙과 달라진다(트랜스파일러 투명성 원칙 위반).
|
|
120
|
+
input: str
|
|
121
|
+
map: dict[str, str] | None = None # {new_column: source_expr} — steps보다 먼저 적용
|
|
122
|
+
steps: list[Step] = []
|
|
123
|
+
incremental: IncrementalSpec | None = None
|
|
124
|
+
state_dir: str = ".gladius"
|
|
125
|
+
# 출력 디렉터리. str로 보관 — Path로 파싱하면 "s3://bucket/x" 같은 URI 스킴이
|
|
126
|
+
# "s3:/bucket/x"로 정규화되어 훼손된다 (P1 httpfs/S3 싱크가 이 필드를 그대로
|
|
127
|
+
# 쓴다). 엔진이 mkdir/os.replace/rmtree 같은 파일시스템 연산이 필요한 지점에서
|
|
128
|
+
# Path(...)로 감싸 해석한다.
|
|
129
|
+
output: str
|
|
130
|
+
|
|
131
|
+
@field_validator("input", "output", "state_dir", mode="before")
|
|
132
|
+
@classmethod
|
|
133
|
+
def _coerce_path_fields_to_str(cls, v: object) -> object:
|
|
134
|
+
if isinstance(v, Path):
|
|
135
|
+
return str(v)
|
|
136
|
+
return v
|
|
137
|
+
|
|
138
|
+
@model_validator(mode="after")
|
|
139
|
+
def _require_map_or_steps(self) -> "TransformSpec":
|
|
140
|
+
if not self.map and not self.steps:
|
|
141
|
+
raise ValueError("transform spec requires at least one of 'map' or 'steps'")
|
|
142
|
+
return self
|
|
143
|
+
|
|
144
|
+
@field_validator("name")
|
|
145
|
+
@classmethod
|
|
146
|
+
def _name_is_path_safe(cls, value: str) -> str:
|
|
147
|
+
if "/" in value or "\\" in value or ".." in value:
|
|
148
|
+
raise ValueError("transform name must not contain '/', '\\\\', or '..'")
|
|
149
|
+
return value
|
gladius/udf.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Python UDF loading and DuckDB registration for Gladius."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import duckdb
|
|
8
|
+
|
|
9
|
+
from arsenal_core.errors import FatalError
|
|
10
|
+
from gladius.compile.ident import normalize_type
|
|
11
|
+
from gladius.spec import PythonUdfStep, TransformSpec
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def udf_name(index: int) -> str:
|
|
15
|
+
"""Return the deterministic, private SQL name for a Python UDF step."""
|
|
16
|
+
return f"__gladius_python_udf_{index}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def register_python_udfs(con: duckdb.DuckDBPyConnection, spec: TransformSpec) -> None:
|
|
20
|
+
"""Load and register all Python UDFs declared in ``spec``."""
|
|
21
|
+
for index, step in enumerate(spec.steps):
|
|
22
|
+
if not isinstance(step, PythonUdfStep):
|
|
23
|
+
continue
|
|
24
|
+
function = _load_function(step.python)
|
|
25
|
+
parameters = (
|
|
26
|
+
[normalize_type(item) for item in step.arg_types]
|
|
27
|
+
if step.arg_types is not None
|
|
28
|
+
else None
|
|
29
|
+
)
|
|
30
|
+
try:
|
|
31
|
+
kwargs: dict[str, Any] = {"return_type": normalize_type(step.return_type)}
|
|
32
|
+
if parameters is not None:
|
|
33
|
+
kwargs["parameters"] = parameters
|
|
34
|
+
con.create_function(udf_name(index), function, **kwargs)
|
|
35
|
+
except duckdb.Error as e:
|
|
36
|
+
raise FatalError(
|
|
37
|
+
f"cannot register Python UDF {step.python!r}; "
|
|
38
|
+
"install de-gladius[python] and check the function signature: "
|
|
39
|
+
f"{e}"
|
|
40
|
+
) from e
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _load_function(target: str) -> Callable[..., Any]:
|
|
44
|
+
module_name, function_name = target.split(":", 1)
|
|
45
|
+
try:
|
|
46
|
+
module = importlib.import_module(module_name)
|
|
47
|
+
function = getattr(module, function_name)
|
|
48
|
+
except (ImportError, AttributeError) as e:
|
|
49
|
+
raise FatalError(f"cannot load Python UDF {target!r}: {e}") from e
|
|
50
|
+
if not callable(function):
|
|
51
|
+
raise FatalError(f"Python UDF target is not callable: {target!r}")
|
|
52
|
+
return function
|