de-arsenal 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
arsenal/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ """Arsenal — 우산 CLI. 단일 진입점, 새 로직 없음 (pugio·gladius 위임).
2
+
3
+ 원클릭 경험의 실체: init(레시피 스캐폴드) → run(수집→변환 일괄) → query(즉석 확인).
4
+ 구현: docs/plans/2026-07-08-m4-integration-release.md Task 4.0~4.1
5
+ """
6
+
7
+ from importlib.metadata import PackageNotFoundError, version
8
+
9
+ try:
10
+ __version__ = version("de-arsenal")
11
+ except PackageNotFoundError:
12
+ __version__ = "0.2.2"
arsenal/cli.py ADDED
@@ -0,0 +1,183 @@
1
+ """Arsenal CLI — 단일 진입점. 위임만 하고 로직을 갖지 않는다.
2
+
3
+ 구현: docs/plans/2026-07-08-m4-integration-release.md Task 4.0
4
+ """
5
+
6
+ import contextlib
7
+ import importlib
8
+ import json
9
+ import os
10
+ import sys
11
+ from collections.abc import Generator
12
+ from importlib import resources
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ import duckdb
17
+ import pyarrow.csv as pa_csv
18
+ import typer
19
+
20
+ from arsenal.project import DbtProjectRef, load_project
21
+ from arsenal_core.errors import ArsenalError, FatalError
22
+ from arsenal_core.spec import load_pipeline
23
+ from gladius import engine as gladius_engine
24
+ from gladius.cli import app as gladius_app
25
+ from gladius.loader import load_transform
26
+ from pugio.cli import app as pugio_app
27
+ from pugio.runner import run_pipeline as _pugio_run_pipeline
28
+
29
+ app = typer.Typer(help="Arsenal — 원클릭 데이터 스택. init → run → query.")
30
+
31
+ _RECIPES_DIRNAME = "recipes"
32
+
33
+
34
+ def _fatal(e: ArsenalError) -> typer.Exit:
35
+ """ArsenalError → `error: {e}` 출력 + exit(1). 커맨드에서는 `raise _fatal(e) from e`로 사용."""
36
+ typer.echo(f"error: {e}", err=True)
37
+ return typer.Exit(1)
38
+
39
+
40
+ def _available_recipes() -> list[str]:
41
+ root = resources.files("arsenal") / _RECIPES_DIRNAME
42
+ return sorted(p.name for p in root.iterdir() if p.is_dir())
43
+
44
+
45
+ def _scaffold_recipe(recipe: str, dest: Path) -> None:
46
+ """레시피 패키지 데이터를 dest로 복사. 기존 파일이 하나라도 있으면 통째로 거부."""
47
+ available = _available_recipes()
48
+ if recipe not in available:
49
+ raise FatalError(f"unknown recipe: {recipe!r} (available: {', '.join(available)})")
50
+
51
+ src_root = resources.files("arsenal") / _RECIPES_DIRNAME / recipe
52
+ entries = [p for p in src_root.iterdir() if p.is_file()]
53
+
54
+ try:
55
+ dest.mkdir(parents=True, exist_ok=True)
56
+ for entry in entries:
57
+ target = dest / entry.name
58
+ if target.exists():
59
+ raise FatalError(f"refusing to overwrite existing file: {target}")
60
+ for entry in entries:
61
+ (dest / entry.name).write_bytes(entry.read_bytes())
62
+ except OSError as e:
63
+ raise FatalError(f"cannot scaffold recipe into {dest}: {e}") from e
64
+
65
+
66
+ @app.command()
67
+ def init(
68
+ recipe: str | None = typer.Argument(None, help="레시피 이름 (예: csv-cleanup)"),
69
+ dest: Path = Path("."),
70
+ list_recipes: bool = typer.Option(False, "--list", help="사용 가능한 레시피 목록 출력"),
71
+ ) -> None:
72
+ """레시피로 프로젝트 스캐폴드 (예: arsenal init csv-cleanup). --list 로 목록 확인."""
73
+ if list_recipes:
74
+ for name in _available_recipes():
75
+ typer.echo(name)
76
+ return
77
+ try:
78
+ if recipe is None:
79
+ raise FatalError("RECIPE argument required (use --list to see available recipes)")
80
+ _scaffold_recipe(recipe, dest)
81
+ except ArsenalError as e:
82
+ raise _fatal(e) from e
83
+ typer.echo(f"done: {recipe} scaffolded into {dest}")
84
+ typer.echo(f" 1. cd {dest}")
85
+ typer.echo(" 2. edit collect.yaml / transform.yaml as needed")
86
+ typer.echo(" 3. arsenal run")
87
+
88
+
89
+ @contextlib.contextmanager
90
+ def _chdir(path: Path) -> Generator[None, None, None]:
91
+ """상대 경로(sink `./data/...` 등)가 프로젝트 디렉터리 기준으로 풀리게 한다."""
92
+ prev = Path.cwd()
93
+ os.chdir(path)
94
+ try:
95
+ yield
96
+ finally:
97
+ os.chdir(prev)
98
+
99
+
100
+ def _run_pipeline(path: Path) -> None:
101
+ """pugio 파이프라인 하나 실행 — 위임 지점(테스트가 monkeypatch하는 seam)."""
102
+ spec = load_pipeline(path)
103
+ report = _pugio_run_pipeline(spec)
104
+ typer.echo(
105
+ f" collect {path.name}: "
106
+ f"fetched={report.fetched} written={report.written} skipped={report.skipped}"
107
+ )
108
+
109
+
110
+ def _run_transform(path: Path) -> None:
111
+ """gladius 변환 하나 실행 — 위임 지점(테스트가 monkeypatch하는 seam)."""
112
+ spec = load_transform(path)
113
+ out = gladius_engine.run_transform(spec)
114
+ typer.echo(f" transform {path.name}: {out}")
115
+
116
+
117
+ def _run_dbt(path: Path) -> None:
118
+ """Run a dbt-duckdb project through dbt's in-process CLI API."""
119
+ if not path.is_dir():
120
+ raise FatalError(f"dbt project not found: {path}")
121
+ try:
122
+ dbt_main: Any = importlib.import_module("dbt.cli.main")
123
+ dbt_runner: Any = dbt_main.dbtRunner
124
+ except (ImportError, AttributeError) as e:
125
+ raise FatalError("dbt stage requires the 'dbt' extra: pip install de-arsenal[dbt]") from e
126
+ try:
127
+ result: Any = dbt_runner().invoke(["run", "--project-dir", str(path)])
128
+ except Exception as e:
129
+ raise FatalError(f"dbt run failed for {path}: {e}") from e
130
+ if not result.success:
131
+ detail = str(getattr(result, "exception", "unknown dbt error"))
132
+ raise FatalError(f"dbt run failed for {path}: {detail}")
133
+ typer.echo(f" dbt {path}: run succeeded")
134
+
135
+
136
+ @app.command()
137
+ def run(project: Path = Path(".")) -> None:
138
+ """arsenal.yaml 순서대로 수집→변환 일괄 실행. 재실행 = 재개."""
139
+ try:
140
+ proj_dir = project if project.is_absolute() else project.absolute()
141
+ proj = load_project(proj_dir / "arsenal.yaml")
142
+ with _chdir(proj_dir):
143
+ for pipeline_path in proj.pipelines:
144
+ _run_pipeline(pipeline_path)
145
+ for transform in proj.transforms:
146
+ if isinstance(transform, DbtProjectRef):
147
+ _run_dbt(transform.dbt)
148
+ else:
149
+ _run_transform(transform)
150
+ except ArsenalError as e:
151
+ raise _fatal(e) from e
152
+ typer.echo(f"done: {len(proj.pipelines)} pipeline(s), {len(proj.transforms)} transform(s)")
153
+
154
+
155
+ @app.command()
156
+ def query(sql: str, fmt: str = typer.Option("table", "--format")) -> None:
157
+ """수집·변환 결과에 즉석 SQL (gladius.engine.query 위임)."""
158
+ try:
159
+ result = gladius_engine.query(sql)
160
+ except ArsenalError as e:
161
+ raise _fatal(e) from e
162
+
163
+ if fmt == "table":
164
+ duckdb.sql("SELECT * FROM result").show() # pyright: ignore[reportUnknownMemberType]
165
+ elif fmt == "csv":
166
+ pa_csv.write_csv( # pyright: ignore[reportPrivateImportUsage, reportUnknownMemberType]
167
+ result, sys.stdout.buffer
168
+ )
169
+ elif fmt == "jsonl":
170
+ for row in result.to_pylist():
171
+ typer.echo(json.dumps(row))
172
+ else:
173
+ typer.echo(f"error: unsupported --format: {fmt!r} (table | csv | jsonl)", err=True)
174
+ raise typer.Exit(1)
175
+
176
+
177
+ # 개별 도구는 서브커맨드로 그대로 노출 — 우산은 감싸되 가리지 않는다
178
+ app.add_typer(pugio_app, name="collect", help="수집 (pugio)")
179
+ app.add_typer(gladius_app, name="transform", help="변환·쿼리 (gladius)")
180
+
181
+
182
+ if __name__ == "__main__":
183
+ app()
arsenal/project.py ADDED
@@ -0,0 +1,68 @@
1
+ """arsenal.yaml — 프로젝트 매니페스트. `arsenal run`의 실행 순서 선언.
2
+
3
+ 구현: docs/plans/2026-07-08-m4-integration-release.md Task 4.0
4
+ 경로는 매니페스트 파일 기준 상대 경로로 해석한다.
5
+ """
6
+
7
+ from pathlib import Path
8
+
9
+ import yaml
10
+ from pydantic import BaseModel, ConfigDict, ValidationError
11
+
12
+ from arsenal_core.errors import FatalError
13
+
14
+
15
+ class DbtProjectRef(BaseModel):
16
+ model_config = ConfigDict(frozen=True, extra="forbid")
17
+
18
+ dbt: Path
19
+
20
+
21
+ class ArsenalProject(BaseModel):
22
+ model_config = ConfigDict(frozen=True, extra="forbid")
23
+
24
+ name: str
25
+ pipelines: list[Path] = [] # pugio 스펙들 — 순서대로 실행
26
+ transforms: list[Path | DbtProjectRef] = [] # gladius/dbt 단계 — 순서대로
27
+
28
+
29
+ def load_project(manifest_path: Path) -> ArsenalProject:
30
+ """arsenal.yaml 로드 + 상대 경로를 매니페스트 기준 절대 경로로 해석.
31
+
32
+ 참조된 파이프라인/변환 스펙 파일의 실존 여부는 검사하지 않는다 — 경로
33
+ 해석만 한다 (존재 검사는 각 스펙을 실제로 로드하는 시점의 책임).
34
+ """
35
+ try:
36
+ text = manifest_path.read_text(encoding="utf-8")
37
+ except FileNotFoundError as e:
38
+ raise FatalError(f"manifest not found: {manifest_path}") from e
39
+ except OSError as e:
40
+ raise FatalError(f"cannot read manifest {manifest_path}: {e}") from e
41
+
42
+ try:
43
+ raw = yaml.safe_load(text)
44
+ except yaml.YAMLError as e:
45
+ raise FatalError(f"invalid manifest {manifest_path}: {e}") from e
46
+
47
+ try:
48
+ parsed = ArsenalProject.model_validate(raw)
49
+ except ValidationError as e:
50
+ lines = [f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in e.errors()]
51
+ raise FatalError(f"invalid manifest {manifest_path}:\n " + "\n ".join(lines)) from e
52
+
53
+ base = manifest_path.parent
54
+ if not base.is_absolute():
55
+ base = base.absolute()
56
+
57
+ transforms: list[Path | DbtProjectRef] = []
58
+ for transform in parsed.transforms:
59
+ if isinstance(transform, Path):
60
+ transforms.append(base / transform)
61
+ else:
62
+ transforms.append(DbtProjectRef(dbt=base / transform.dbt))
63
+
64
+ return ArsenalProject(
65
+ name=parsed.name,
66
+ pipelines=[base / p for p in parsed.pipelines],
67
+ transforms=transforms,
68
+ )
arsenal/py.typed ADDED
File without changes
@@ -0,0 +1,11 @@
1
+ # api-to-postgres 레시피
2
+
3
+ REST API를 받아 검증 게이트를 통과시킨 뒤 Postgres에 멱등 upsert한다.
4
+
5
+ 0. 설치: 이 레시피는 postgres extra(`psycopg`)가 필요하다 —
6
+ `uv tool install "de-arsenal[postgres]"` (또는 `pip install "pugio[postgres]"`).
7
+ extra 없이 실행하면 postgres sink가 `FatalError`로 즉시 안내한다(모듈을 찾을 수
8
+ 없다는 원문 traceback이 아니다).
9
+ 1. 바꿀 것: `collect.yaml`의 API URL, 환경변수 `PG_DSN`(Postgres 접속 문자열), `merge_key`
10
+ 2. 실행: `arsenal run` — 실행 전 `PG_DSN`이 가리키는 Postgres에 접속 가능해야 한다
11
+ 3. 확인: 위반 레코드는 적재되지 않고 `.arsenal/dlq/api-to-postgres/`에 격리된다 (`pugio dlq list`)
@@ -0,0 +1,3 @@
1
+ name: api-to-postgres
2
+ pipelines:
3
+ - collect.yaml
@@ -0,0 +1,16 @@
1
+ name: api-to-postgres
2
+ source:
3
+ type: rest
4
+ url: https://api.example.com/records # <- 바꿀 것: 실제 API 엔드포인트
5
+ pagination: { mode: page, param: page, size_param: per_page, size: 100 }
6
+ validate:
7
+ rules:
8
+ - field: id
9
+ not_null: true
10
+ unique: true
11
+ on_violation: quarantine
12
+ sink:
13
+ type: postgres
14
+ dsn_env: PG_DSN # <- 바꿀 것: Postgres DSN을 담은 환경변수 이름
15
+ table: records
16
+ merge_key: [id]
@@ -0,0 +1,7 @@
1
+ # csv-cleanup 레시피
2
+
3
+ 로컬 CSV 파일들을 모아 중복 제거 + 타입 캐스팅된 테이블로 만든다.
4
+
5
+ 1. 바꿀 것: `input/` 디렉터리에 정리할 CSV를 넣고, `transform.yaml`의 dedup 키를 데이터에 맞게 조정
6
+ 2. 실행: `arsenal run`
7
+ 3. 확인: `arsenal query "SELECT * FROM './data/clean/*.parquet' LIMIT 20"`
@@ -0,0 +1,5 @@
1
+ name: csv-cleanup
2
+ pipelines:
3
+ - collect.yaml
4
+ transforms:
5
+ - transform.yaml
@@ -0,0 +1,8 @@
1
+ name: csv-cleanup
2
+ source:
3
+ type: file
4
+ path: ./input/*.csv # <- 정리할 CSV들을 이 디렉터리에 넣으세요
5
+ format: csv
6
+ sink:
7
+ type: parquet
8
+ path: ./data/raw
@@ -0,0 +1,6 @@
1
+ name: clean-rows
2
+ input: ./data/raw
3
+ steps:
4
+ - dedup: [id] # <- 중복 판정 키를 데이터에 맞게 바꾸세요
5
+ - cast: { id: bigint }
6
+ output: ./data/clean
@@ -0,0 +1,7 @@
1
+ # github-issues 레시피
2
+
3
+ GitHub 저장소의 이슈를 받아 정리된 테이블로 만든다.
4
+
5
+ 1. 바꿀 것: `collect.yaml`의 저장소 경로, 환경변수 `GITHUB_TOKEN`
6
+ 2. 실행: `arsenal run`
7
+ 3. 확인: `arsenal query "SELECT state, count(*) FROM './data/issues_clean/*.parquet' GROUP BY 1"`
@@ -0,0 +1,5 @@
1
+ name: github-issues
2
+ pipelines:
3
+ - collect.yaml
4
+ transforms:
5
+ - transform.yaml
@@ -0,0 +1,9 @@
1
+ name: github-issues
2
+ source:
3
+ type: rest
4
+ url: https://api.github.com/repos/duckdb/duckdb/issues # <- 저장소를 바꾸세요
5
+ headers: { Authorization: "Bearer ${GITHUB_TOKEN}" }
6
+ pagination: { mode: offset, param: page, size_param: per_page, size: 100 }
7
+ sink:
8
+ type: parquet
9
+ path: ./data/issues
@@ -0,0 +1,7 @@
1
+ name: clean-issues
2
+ input: ./data/issues
3
+ steps:
4
+ - filter: "state IS NOT NULL"
5
+ - cast: { number: bigint }
6
+ - select: [number, title, state, created_at]
7
+ output: ./data/issues_clean
@@ -0,0 +1,68 @@
1
+ Metadata-Version: 2.5
2
+ Name: de-arsenal
3
+ Version: 0.2.2
4
+ Summary: Arsenal — 원클릭 데이터 스택. 수집→검증→변환→쿼리를 명령 하나로 (pugio+gladius 우산 CLI)
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ License-Expression: Apache-2.0
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: License :: OSI Approved :: Apache Software License
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
+ Requires-Python: >=3.11
16
+ Requires-Dist: arsenal-core<0.3,>=0.2
17
+ Requires-Dist: de-gladius<0.3,>=0.2
18
+ Requires-Dist: pugio<0.3,>=0.2
19
+ Requires-Dist: typer>=0.12
20
+ Provides-Extra: dbt
21
+ Requires-Dist: dbt-duckdb<2,>=1.8; extra == 'dbt'
22
+ Provides-Extra: dlt
23
+ Requires-Dist: pugio[dlt]; extra == 'dlt'
24
+ Provides-Extra: postgres
25
+ Requires-Dist: pugio[postgres]; extra == 'postgres'
26
+ Description-Content-Type: text/markdown
27
+
28
+ # Arsenal (`de-arsenal`)
29
+
30
+ > 우산 CLI — Pugio(수집)와 Gladius(변환)를 한 진입점에서 부른다. 새 로직 없음,
31
+ > 위임만 한다(Unix 철학: 개별 도구는 독립적으로 완결됨).
32
+
33
+ `arsenal init <recipe>` → `arsenal run` → `arsenal query`, 세 명령으로
34
+ 수집→검증→변환→쿼리를 재현하는 원클릭 데이터 스택. 배포명은 `de-arsenal`,
35
+ 설치되는 명령어는 `arsenal`이다.
36
+
37
+ ## 설치
38
+
39
+ ```bash
40
+ uv tool install de-arsenal
41
+ # 또는
42
+ pipx install de-arsenal
43
+ ```
44
+
45
+ ## 사용 예
46
+
47
+ ```bash
48
+ arsenal init csv-cleanup # 레시피로 프로젝트 스캐폴드 (arsenal init --list로 목록)
49
+ arsenal run # arsenal.yaml 순서대로 수집→변환 일괄 실행. 재실행 = 재개
50
+ arsenal query "SELECT * FROM './data/clean/*.parquet' LIMIT 10"
51
+ ```
52
+
53
+ `arsenal.yaml` 매니페스트:
54
+
55
+ ```yaml
56
+ name: my-project
57
+ pipelines: # pugio로 순서대로 실행
58
+ - collect.yaml
59
+ transforms: # gladius로 순서대로 실행
60
+ - transform.yaml
61
+ ```
62
+
63
+ ## 문서
64
+
65
+ - [docs/01-scope.md](../../docs/01-scope.md) — M4 통합·릴리스 범위
66
+ - [docs/08-limits.md](../../docs/08-limits.md) — 정직한 한계선
67
+ - [examples/quickstart](../../examples/quickstart) — 10분 collect → transform → query 데모
68
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,19 @@
1
+ arsenal/__init__.py,sha256=kQD8ZKAf8TvsvZhLsYCOI8UCBpYPv2AR36c1YoD0uho,440
2
+ arsenal/cli.py,sha256=hVTKE4Rbsf6g9Qco8Ml6LgnQ_q7EAHdZvrJh6riHBWs,6722
3
+ arsenal/project.py,sha256=ccWBCbVeDBvVTcA28mY8sqVbOX4ChGpal4ot1Q70FPU,2330
4
+ arsenal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ arsenal/recipes/api-to-postgres/README.md,sha256=PAFjIOcrOz48Bg-Bhm2T4AjIF7TKmmrvZ3qqIFGgiD4,753
6
+ arsenal/recipes/api-to-postgres/arsenal.yaml,sha256=cSmAC_rhc-UkozeAvtH6qhYGVWooY-QCe-Uqv77BOPU,50
7
+ arsenal/recipes/api-to-postgres/collect.yaml,sha256=Ah6bDSnYPCwsCn8LBaEu6KIAqCrWbNZ8-wnEDsI3T6A,443
8
+ arsenal/recipes/csv-cleanup/README.md,sha256=7iMpcwkHjKeRgbQqUkWHAOphMdYCPB33f3W52rmBEGs,344
9
+ arsenal/recipes/csv-cleanup/arsenal.yaml,sha256=QhkQjsGVEDj2rGyV1_5mp0jmj-7sfKLMk4yi77B606M,77
10
+ arsenal/recipes/csv-cleanup/collect.yaml,sha256=UKDiXS6QkCGkrZ_npBmGibE0ibIUJmL43ahtGaUPTS0,176
11
+ arsenal/recipes/csv-cleanup/transform.yaml,sha256=4xE0bz7vTNkmPEcgS3u--GSZD6HP9KaKWTZm38UUn-o,171
12
+ arsenal/recipes/github-issues/README.md,sha256=R6mLG2aKsyFfu1cWX3AyHRbQvscpSnegviMmpfo834k,303
13
+ arsenal/recipes/github-issues/arsenal.yaml,sha256=wMJ0U_fjsLRvYCJwBPc4NlSz3p4hzNgTjFIaawAnrRk,79
14
+ arsenal/recipes/github-issues/collect.yaml,sha256=lM3gyaCiVAA8Fdjnv59lYnuDry0VXk0aIZlDxxiXzsc,307
15
+ arsenal/recipes/github-issues/transform.yaml,sha256=wybQxi7wjysELFhzX8gevphXM7SEWCCyasIy9J3iQeA,183
16
+ de_arsenal-0.2.2.dist-info/METADATA,sha256=xEnzAjMzMduyholouFE0Tr1j5Qth1NGqythStA79MKg,2444
17
+ de_arsenal-0.2.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
18
+ de_arsenal-0.2.2.dist-info/entry_points.txt,sha256=5a6_X5W3eUYokqtcyj6SJZJmCXOoy8jq2wJyCfkJ3AQ,44
19
+ de_arsenal-0.2.2.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ arsenal = arsenal.cli:app