arsenal-core 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arsenal_core-0.2.0/.gitignore +12 -0
- arsenal_core-0.2.0/PKG-INFO +53 -0
- arsenal_core-0.2.0/README.md +33 -0
- arsenal_core-0.2.0/pyproject.toml +28 -0
- arsenal_core-0.2.0/src/arsenal_core/__init__.py +15 -0
- arsenal_core-0.2.0/src/arsenal_core/errors.py +35 -0
- arsenal_core-0.2.0/src/arsenal_core/identity.py +18 -0
- arsenal_core-0.2.0/src/arsenal_core/io/__init__.py +8 -0
- arsenal_core-0.2.0/src/arsenal_core/py.typed +0 -0
- arsenal_core-0.2.0/src/arsenal_core/ratelimit.py +58 -0
- arsenal_core-0.2.0/src/arsenal_core/retry.py +46 -0
- arsenal_core-0.2.0/src/arsenal_core/schema.py +81 -0
- arsenal_core-0.2.0/src/arsenal_core/spec/__init__.py +43 -0
- arsenal_core-0.2.0/src/arsenal_core/spec/loader.py +67 -0
- arsenal_core-0.2.0/src/arsenal_core/spec/models.py +260 -0
- arsenal_core-0.2.0/src/arsenal_core/state/__init__.py +19 -0
- arsenal_core-0.2.0/src/arsenal_core/state/store.py +362 -0
- arsenal_core-0.2.0/tests/test_errors.py +35 -0
- arsenal_core-0.2.0/tests/test_gen_schema_docs.py +117 -0
- arsenal_core-0.2.0/tests/test_identity.py +23 -0
- arsenal_core-0.2.0/tests/test_import_core.py +19 -0
- arsenal_core-0.2.0/tests/test_ratelimit.py +67 -0
- arsenal_core-0.2.0/tests/test_retry.py +52 -0
- arsenal_core-0.2.0/tests/test_schema.py +31 -0
- arsenal_core-0.2.0/tests/test_schema_snapshot.py +64 -0
- arsenal_core-0.2.0/tests/test_spec.py +95 -0
- arsenal_core-0.2.0/tests/test_spec_m2.py +193 -0
- arsenal_core-0.2.0/tests/test_state_cursor.py +13 -0
- arsenal_core-0.2.0/tests/test_state_quarantine.py +47 -0
- arsenal_core-0.2.0/tests/test_state_store.py +129 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: arsenal-core
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Shared reliability core for DE Arsenal tools — state, identity, errors, spec, retry
|
|
5
|
+
Project-URL: Homepage, https://github.com/KooEric/de-arsenal
|
|
6
|
+
Project-URL: Repository, https://github.com/KooEric/de-arsenal
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Requires-Dist: pydantic>=2.7
|
|
17
|
+
Requires-Dist: pyyaml>=6
|
|
18
|
+
Requires-Dist: tenacity>=8.3
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# arsenal-core
|
|
22
|
+
|
|
23
|
+
DE Arsenal(Pugio·Gladius·Arsenal)이 공유하는 신뢰성 코어 라이브러리 —
|
|
24
|
+
파이프라인/변환 스펙 로더(Pydantic), 결정적 Unit ID, `StateStore`(SQLite
|
|
25
|
+
기반 unit 상태 기록), 에러 분류 체계(`RetryableError`/`AuthExpiredError`/
|
|
26
|
+
`FatalError`), backoff 재시도 래퍼. 그 자체로는 CLI가 없는 라이브러리 패키지다
|
|
27
|
+
— Pugio/Gladius가 이 위에서 "재개·멱등·에러 분류가 기본값"을 구현한다.
|
|
28
|
+
|
|
29
|
+
## 사용 예
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
from arsenal_core.state import UnitSpec, StateStore
|
|
35
|
+
from arsenal_core.errors import RetryableError, FatalError
|
|
36
|
+
|
|
37
|
+
# 결정적 unit_id: 같은 (pipeline, source, unit_key) → 항상 같은 id → 재실행 시 중복 방지
|
|
38
|
+
unit = UnitSpec.create(
|
|
39
|
+
pipeline="github-issues", source="issues", unit_key="page=1", payload={}
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
store = StateStore(Path("./state/github-issues.db"))
|
|
43
|
+
store.register(unit)
|
|
44
|
+
... # fetch → write
|
|
45
|
+
store.mark_done(unit.unit_id, row_count=100, byte_count=8192)
|
|
46
|
+
store.close()
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## 문서
|
|
50
|
+
|
|
51
|
+
- [docs/02-architecture.md](../../docs/02-architecture.md) — 공통 코어·상태 모델·멱등 전략
|
|
52
|
+
- [docs/08-limits.md](../../docs/08-limits.md) — 정직한 한계선
|
|
53
|
+
- [저장소 루트 README](../../README.md)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# arsenal-core
|
|
2
|
+
|
|
3
|
+
DE Arsenal(Pugio·Gladius·Arsenal)이 공유하는 신뢰성 코어 라이브러리 —
|
|
4
|
+
파이프라인/변환 스펙 로더(Pydantic), 결정적 Unit ID, `StateStore`(SQLite
|
|
5
|
+
기반 unit 상태 기록), 에러 분류 체계(`RetryableError`/`AuthExpiredError`/
|
|
6
|
+
`FatalError`), backoff 재시도 래퍼. 그 자체로는 CLI가 없는 라이브러리 패키지다
|
|
7
|
+
— Pugio/Gladius가 이 위에서 "재개·멱등·에러 분류가 기본값"을 구현한다.
|
|
8
|
+
|
|
9
|
+
## 사용 예
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from arsenal_core.state import UnitSpec, StateStore
|
|
15
|
+
from arsenal_core.errors import RetryableError, FatalError
|
|
16
|
+
|
|
17
|
+
# 결정적 unit_id: 같은 (pipeline, source, unit_key) → 항상 같은 id → 재실행 시 중복 방지
|
|
18
|
+
unit = UnitSpec.create(
|
|
19
|
+
pipeline="github-issues", source="issues", unit_key="page=1", payload={}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
store = StateStore(Path("./state/github-issues.db"))
|
|
23
|
+
store.register(unit)
|
|
24
|
+
... # fetch → write
|
|
25
|
+
store.mark_done(unit.unit_id, row_count=100, byte_count=8192)
|
|
26
|
+
store.close()
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## 문서
|
|
30
|
+
|
|
31
|
+
- [docs/02-architecture.md](../../docs/02-architecture.md) — 공통 코어·상태 모델·멱등 전략
|
|
32
|
+
- [docs/08-limits.md](../../docs/08-limits.md) — 정직한 한계선
|
|
33
|
+
- [저장소 루트 README](../../README.md)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "arsenal-core"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Shared reliability core for DE Arsenal tools — state, identity, errors, spec, retry"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "Apache-2.0"
|
|
8
|
+
dependencies = ["pydantic>=2.7", "pyyaml>=6", "tenacity>=8.3"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
"Programming Language :: Python :: 3.11",
|
|
12
|
+
"Programming Language :: Python :: 3.12",
|
|
13
|
+
"License :: OSI Approved :: Apache Software License",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Homepage = "https://github.com/KooEric/de-arsenal"
|
|
21
|
+
Repository = "https://github.com/KooEric/de-arsenal"
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["hatchling"]
|
|
25
|
+
build-backend = "hatchling.build"
|
|
26
|
+
|
|
27
|
+
[tool.hatch.build.targets.wheel]
|
|
28
|
+
packages = ["src/arsenal_core"]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""arsenal-core — DE Arsenal 공통 신뢰성 코어.
|
|
2
|
+
|
|
3
|
+
모든 도구(pugio, gladius, …)의 유일한 공통 의존성.
|
|
4
|
+
구조: docs/02-architecture.md 참조.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
__version__ = version("arsenal-core")
|
|
11
|
+
except PackageNotFoundError:
|
|
12
|
+
__version__ = "0.2.0"
|
|
13
|
+
from arsenal_core.schema import SchemaChange, SchemaDiff, compare_schema_snapshots
|
|
14
|
+
|
|
15
|
+
__all__ = ["SchemaChange", "SchemaDiff", "compare_schema_snapshots"]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""에러 분류 체계 — 커넥터가 분류하고, 러너는 타입만 보고 행동한다.
|
|
2
|
+
|
|
3
|
+
설계: docs/02-architecture.md "에러 분류 체계"
|
|
4
|
+
구현: docs/plans/2026-07-08-m1-core-foundation.md Task 1
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ArsenalError(Exception):
|
|
9
|
+
"""모든 Arsenal 예외의 루트."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FatalError(ArsenalError):
|
|
13
|
+
"""재시도 무의미: 설정 오류, 4xx(401/429 제외), 계약 위반. 즉시 중단."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RetryableError(ArsenalError):
|
|
17
|
+
"""일시적 실패: 네트워크, 5xx, 429, lock 충돌. backoff 재시도."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AuthExpiredError(RetryableError):
|
|
21
|
+
"""인증 만료. 에러가 아니라 갱신 트리거 (M2에서 refresh hook 연결)."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def classify_http_status(status: int) -> type[ArsenalError] | None:
|
|
25
|
+
"""HTTP 상태 코드를 예외 타입으로 분류. 2xx/3xx는 None.
|
|
26
|
+
|
|
27
|
+
401→AuthExpired, 429/5xx→Retryable, 그 외 4xx→Fatal.
|
|
28
|
+
"""
|
|
29
|
+
if status < 400:
|
|
30
|
+
return None
|
|
31
|
+
if status == 401:
|
|
32
|
+
return AuthExpiredError
|
|
33
|
+
if status == 429 or status >= 500:
|
|
34
|
+
return RetryableError
|
|
35
|
+
return FatalError
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""결정적 Unit ID — 멱등성의 근거. 같은 unit은 언제 실행해도 같은 ID.
|
|
2
|
+
|
|
3
|
+
설계: docs/02-architecture.md "결정적 ID"
|
|
4
|
+
구현: docs/plans/2026-07-08-m1-core-foundation.md Task 2
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
|
|
9
|
+
_ID_LEN = 16
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def unit_id(pipeline: str, source: str, unit_key: str) -> str:
|
|
13
|
+
"""sha256("{pipeline}\\x1f{source}\\x1f{unit_key}") 앞 16 hex.
|
|
14
|
+
|
|
15
|
+
\\x1f 구분자로 경계 모호성 제거. 결정성·충돌 저항은 hypothesis로 검증.
|
|
16
|
+
"""
|
|
17
|
+
raw = f"{pipeline}\x1f{source}\x1f{unit_key}"
|
|
18
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:_ID_LEN]
|
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""토큰 버킷 rate limiter — REST 소스가 rps 상한을 지키도록 acquire()로 스로틀한다.
|
|
2
|
+
|
|
3
|
+
설계: docs/02-architecture.md M2 rate_limit. penalize()는 429 Retry-After를
|
|
4
|
+
흡수해 다음 acquire()가 그만큼 더 기다리게 만든다 (with_retry의 재시도 루프와
|
|
5
|
+
맞물려 "실패 → 벌점 → 다음 시도 전 대기"를 구현한다).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import time
|
|
9
|
+
from typing import Protocol
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Clock(Protocol):
|
|
13
|
+
def monotonic(self) -> float: ...
|
|
14
|
+
def sleep(self, seconds: float) -> None: ...
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SystemClock:
|
|
18
|
+
"""실제 time.monotonic()/time.sleep()을 쓰는 Clock 구현체 — 다른 패키지(pugio.auth
|
|
19
|
+
등)도 재선언 없이 import해 쓸 수 있도록 공개 이름으로 export한다."""
|
|
20
|
+
|
|
21
|
+
def monotonic(self) -> float:
|
|
22
|
+
return time.monotonic()
|
|
23
|
+
|
|
24
|
+
def sleep(self, seconds: float) -> None:
|
|
25
|
+
time.sleep(seconds)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# 하위 호환: 기존 내부 전용 이름으로 참조하던 코드(및 다른 패키지의 재선언부)가
|
|
29
|
+
# 있을 수 있어 별칭을 유지한다.
|
|
30
|
+
_MonotonicClock = SystemClock
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class TokenBucket:
|
|
34
|
+
"""rps 상한 토큰 버킷. penalize()로 429 Retry-After를 흡수한다."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, rps: float, *, clock: Clock | None = None, burst: int = 1) -> None:
|
|
37
|
+
self._interval = 1.0 / rps
|
|
38
|
+
self._clock = clock or _MonotonicClock()
|
|
39
|
+
self._burst = float(burst)
|
|
40
|
+
self._tokens = float(burst)
|
|
41
|
+
self._last = self._clock.monotonic()
|
|
42
|
+
self._penalty_until = 0.0
|
|
43
|
+
|
|
44
|
+
def acquire(self) -> None:
|
|
45
|
+
now = self._clock.monotonic()
|
|
46
|
+
if now < self._penalty_until:
|
|
47
|
+
self._clock.sleep(self._penalty_until - now)
|
|
48
|
+
now = self._clock.monotonic()
|
|
49
|
+
self._tokens = min(self._burst, self._tokens + (now - self._last) / self._interval)
|
|
50
|
+
self._last = now
|
|
51
|
+
if self._tokens < 1.0:
|
|
52
|
+
self._clock.sleep((1.0 - self._tokens) * self._interval)
|
|
53
|
+
self._tokens = self._burst
|
|
54
|
+
self._last = self._clock.monotonic()
|
|
55
|
+
self._tokens -= 1.0
|
|
56
|
+
|
|
57
|
+
def penalize(self, seconds: float) -> None:
|
|
58
|
+
self._penalty_until = self._clock.monotonic() + seconds
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Retryable만 재시도하는 backoff 래퍼. 엔진은 빌린다 — tenacity 사용.
|
|
2
|
+
|
|
3
|
+
구현: docs/plans/2026-07-08-m1-core-foundation.md Task 5
|
|
4
|
+
FatalError는 즉시 전파, RetryableError만 exponential backoff + jitter.
|
|
5
|
+
|
|
6
|
+
M2-D: AuthExpiredError는 RetryableError의 서브클래스지만 여기서는 재시도 대상에서
|
|
7
|
+
제외한다. 401을 blind하게 backoff 재시도해봐야 provider.refresh()가 호출되지
|
|
8
|
+
않는 한 계속 401만 받는다 — 갱신 없는 재시도는 의미가 없다. AuthExpiredError는
|
|
9
|
+
즉시 전파시켜 runner.py가 "refresh() 후 한 번 더 시도"를 명시적으로 수행하게
|
|
10
|
+
한다 (러너 쪽 갱신 루프, docs/02-architecture.md M2-D).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from typing import TypeVar
|
|
15
|
+
|
|
16
|
+
from tenacity import (
|
|
17
|
+
retry,
|
|
18
|
+
retry_if_exception_type,
|
|
19
|
+
retry_if_not_exception_type,
|
|
20
|
+
stop_after_attempt,
|
|
21
|
+
wait_exponential_jitter,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
from arsenal_core.errors import AuthExpiredError, RetryableError
|
|
25
|
+
|
|
26
|
+
T = TypeVar("T")
|
|
27
|
+
|
|
28
|
+
DEFAULT_MAX_ATTEMPTS = 5
|
|
29
|
+
DEFAULT_BASE_WAIT = 1.0 # seconds
|
|
30
|
+
MAX_WAIT = 60.0 # seconds
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def with_retry(
|
|
34
|
+
fn: Callable[[], T],
|
|
35
|
+
*,
|
|
36
|
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
|
37
|
+
base_wait: float = DEFAULT_BASE_WAIT,
|
|
38
|
+
) -> T:
|
|
39
|
+
wrapped = retry(
|
|
40
|
+
retry=retry_if_exception_type(RetryableError)
|
|
41
|
+
& retry_if_not_exception_type(AuthExpiredError),
|
|
42
|
+
stop=stop_after_attempt(max_attempts),
|
|
43
|
+
wait=wait_exponential_jitter(initial=base_wait, max=MAX_WAIT),
|
|
44
|
+
reraise=True,
|
|
45
|
+
)(fn)
|
|
46
|
+
return wrapped()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Schema snapshot comparison primitives shared by collection components."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any, cast
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class SchemaChange:
|
|
10
|
+
"""One field-level schema change."""
|
|
11
|
+
|
|
12
|
+
field: str
|
|
13
|
+
before: dict[str, Any] | None
|
|
14
|
+
after: dict[str, Any] | None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class SchemaDiff:
|
|
19
|
+
"""Added, removed, and modified fields between two snapshots."""
|
|
20
|
+
|
|
21
|
+
added: tuple[SchemaChange, ...]
|
|
22
|
+
removed: tuple[SchemaChange, ...]
|
|
23
|
+
changed: tuple[SchemaChange, ...]
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def has_changes(self) -> bool:
|
|
27
|
+
return bool(self.added or self.removed or self.changed)
|
|
28
|
+
|
|
29
|
+
def summary(self) -> str:
|
|
30
|
+
"""Return a compact, stable message suitable for logs and state."""
|
|
31
|
+
parts: list[str] = []
|
|
32
|
+
if self.added:
|
|
33
|
+
parts.append("added=" + ",".join(change.field for change in self.added))
|
|
34
|
+
if self.removed:
|
|
35
|
+
parts.append("removed=" + ",".join(change.field for change in self.removed))
|
|
36
|
+
if self.changed:
|
|
37
|
+
parts.append("changed=" + ",".join(change.field for change in self.changed))
|
|
38
|
+
return "; ".join(parts)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def compare_schema_snapshots(previous: str, current: str) -> SchemaDiff:
|
|
42
|
+
"""Compare snapshots produced by the runner's deterministic JSON encoder.
|
|
43
|
+
|
|
44
|
+
Snapshot entries are intentionally treated as opaque dictionaries except for
|
|
45
|
+
their ``name`` key. This lets future snapshot metadata be added without
|
|
46
|
+
changing the comparison algorithm, while still detecting type/nullability
|
|
47
|
+
changes in today's format.
|
|
48
|
+
"""
|
|
49
|
+
previous_fields = _fields(previous)
|
|
50
|
+
current_fields = _fields(current)
|
|
51
|
+
added: list[SchemaChange] = []
|
|
52
|
+
removed: list[SchemaChange] = []
|
|
53
|
+
changed: list[SchemaChange] = []
|
|
54
|
+
|
|
55
|
+
for name in sorted(current_fields.keys() - previous_fields.keys()):
|
|
56
|
+
added.append(SchemaChange(name, None, current_fields[name]))
|
|
57
|
+
for name in sorted(previous_fields.keys() - current_fields.keys()):
|
|
58
|
+
removed.append(SchemaChange(name, previous_fields[name], None))
|
|
59
|
+
for name in sorted(current_fields.keys() & previous_fields.keys()):
|
|
60
|
+
before = previous_fields[name]
|
|
61
|
+
after = current_fields[name]
|
|
62
|
+
if before != after:
|
|
63
|
+
changed.append(SchemaChange(name, before, after))
|
|
64
|
+
|
|
65
|
+
return SchemaDiff(tuple(added), tuple(removed), tuple(changed))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _fields(snapshot: str) -> dict[str, dict[str, Any]]:
|
|
69
|
+
raw = cast(object, json.loads(snapshot))
|
|
70
|
+
if not isinstance(raw, list):
|
|
71
|
+
raise ValueError("schema snapshot must be a JSON list")
|
|
72
|
+
entries = cast(list[object], raw)
|
|
73
|
+
fields: dict[str, dict[str, Any]] = {}
|
|
74
|
+
for raw_entry in entries:
|
|
75
|
+
if not isinstance(raw_entry, dict):
|
|
76
|
+
raise ValueError("schema snapshot entries must contain a string name")
|
|
77
|
+
entry = cast(dict[str, Any], raw_entry)
|
|
78
|
+
if not isinstance(entry.get("name"), str):
|
|
79
|
+
raise ValueError("schema snapshot entries must contain a string name")
|
|
80
|
+
fields[entry["name"]] = entry
|
|
81
|
+
return fields
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from arsenal_core.spec.loader import load_pipeline
|
|
2
|
+
from arsenal_core.spec.models import (
|
|
3
|
+
AuthSpec,
|
|
4
|
+
ContractColumnSpec,
|
|
5
|
+
ContractSpec,
|
|
6
|
+
DatabaseSourceSpec,
|
|
7
|
+
DuckDBSinkSpec,
|
|
8
|
+
FileSourceSpec,
|
|
9
|
+
PaginationSpec,
|
|
10
|
+
ParquetSinkSpec,
|
|
11
|
+
PipelineSpec,
|
|
12
|
+
PostgresSinkSpec,
|
|
13
|
+
PythonSourceSpec,
|
|
14
|
+
RateLimitSpec,
|
|
15
|
+
RestSourceSpec,
|
|
16
|
+
SinkSpec,
|
|
17
|
+
SourceSpec,
|
|
18
|
+
SplitSpec,
|
|
19
|
+
ValidateRule,
|
|
20
|
+
ValidateSpec,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"AuthSpec",
|
|
25
|
+
"ContractColumnSpec",
|
|
26
|
+
"ContractSpec",
|
|
27
|
+
"DatabaseSourceSpec",
|
|
28
|
+
"DuckDBSinkSpec",
|
|
29
|
+
"FileSourceSpec",
|
|
30
|
+
"PaginationSpec",
|
|
31
|
+
"ParquetSinkSpec",
|
|
32
|
+
"PipelineSpec",
|
|
33
|
+
"PostgresSinkSpec",
|
|
34
|
+
"PythonSourceSpec",
|
|
35
|
+
"RateLimitSpec",
|
|
36
|
+
"RestSourceSpec",
|
|
37
|
+
"SinkSpec",
|
|
38
|
+
"SourceSpec",
|
|
39
|
+
"SplitSpec",
|
|
40
|
+
"ValidateRule",
|
|
41
|
+
"ValidateSpec",
|
|
42
|
+
"load_pipeline",
|
|
43
|
+
]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""YAML → 검증된 PipelineSpec. 에러 메시지 품질이 곧 UX.
|
|
2
|
+
|
|
3
|
+
구현: docs/plans/2026-07-08-m1-core-foundation.md Task 4
|
|
4
|
+
- ${VAR} 환경변수 치환 (비밀은 YAML에 두지 않는다)
|
|
5
|
+
- 검증 실패 시 필드 경로가 담긴 FatalError
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
from pydantic import ValidationError
|
|
14
|
+
|
|
15
|
+
from arsenal_core.errors import FatalError
|
|
16
|
+
from arsenal_core.spec.models import PipelineSpec
|
|
17
|
+
|
|
18
|
+
_ENV_PATTERN = re.compile(r"\$\{(\w+)\}")
|
|
19
|
+
|
|
20
|
+
# discriminated union 필드 → 허용된 태그 값. Pydantic은 이런 필드의 에러 loc에
|
|
21
|
+
# 태그를 끼워 넣는다(예: ("source","rest","url")) — 사용자에게는 YAML 그대로의
|
|
22
|
+
# 경로(source.url)가 더 읽기 쉽고, M1 에러 메시지와도 하위 호환된다.
|
|
23
|
+
_UNION_TAGS: dict[str, frozenset[str]] = {
|
|
24
|
+
"source": frozenset({"rest", "file", "database", "python", "dlt"}),
|
|
25
|
+
"sink": frozenset({"parquet", "duckdb", "postgres"}),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _substitute_env(text: str) -> str:
|
|
30
|
+
def repl(m: re.Match[str]) -> str:
|
|
31
|
+
value = os.environ.get(m.group(1))
|
|
32
|
+
if value is None:
|
|
33
|
+
raise FatalError(f"environment variable not set: {m.group(1)}")
|
|
34
|
+
return value
|
|
35
|
+
|
|
36
|
+
return _ENV_PATTERN.sub(repl, text)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _clean_loc(loc: tuple[int | str, ...]) -> tuple[int | str, ...]:
|
|
40
|
+
"""discriminated union의 태그 세그먼트를 제거한다 (source.rest.url → source.url)."""
|
|
41
|
+
cleaned: list[int | str] = []
|
|
42
|
+
for i, part in enumerate(loc):
|
|
43
|
+
prev = loc[i - 1] if i > 0 else None
|
|
44
|
+
allowed = _UNION_TAGS.get(str(prev)) if isinstance(prev, str) else None
|
|
45
|
+
if allowed is not None and isinstance(part, str) and part in allowed:
|
|
46
|
+
continue
|
|
47
|
+
cleaned.append(part)
|
|
48
|
+
return tuple(cleaned)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def load_pipeline(path: Path) -> PipelineSpec:
|
|
52
|
+
try:
|
|
53
|
+
text = path.read_text(encoding="utf-8")
|
|
54
|
+
except FileNotFoundError as e:
|
|
55
|
+
raise FatalError(f"spec file not found: {path}") from e
|
|
56
|
+
except OSError as e:
|
|
57
|
+
raise FatalError(f"cannot read spec file {path}: {e}") from e
|
|
58
|
+
|
|
59
|
+
raw = yaml.safe_load(_substitute_env(text))
|
|
60
|
+
try:
|
|
61
|
+
return PipelineSpec.model_validate(raw)
|
|
62
|
+
except ValidationError as e:
|
|
63
|
+
lines = [
|
|
64
|
+
f"{'.'.join(str(p) for p in _clean_loc(err['loc']))}: {err['msg']}"
|
|
65
|
+
for err in e.errors()
|
|
66
|
+
]
|
|
67
|
+
raise FatalError(f"invalid pipeline spec {path}:\n " + "\n ".join(lines)) from e
|