arsenal-core 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. arsenal_core-0.2.0/.gitignore +12 -0
  2. arsenal_core-0.2.0/PKG-INFO +53 -0
  3. arsenal_core-0.2.0/README.md +33 -0
  4. arsenal_core-0.2.0/pyproject.toml +28 -0
  5. arsenal_core-0.2.0/src/arsenal_core/__init__.py +15 -0
  6. arsenal_core-0.2.0/src/arsenal_core/errors.py +35 -0
  7. arsenal_core-0.2.0/src/arsenal_core/identity.py +18 -0
  8. arsenal_core-0.2.0/src/arsenal_core/io/__init__.py +8 -0
  9. arsenal_core-0.2.0/src/arsenal_core/py.typed +0 -0
  10. arsenal_core-0.2.0/src/arsenal_core/ratelimit.py +58 -0
  11. arsenal_core-0.2.0/src/arsenal_core/retry.py +46 -0
  12. arsenal_core-0.2.0/src/arsenal_core/schema.py +81 -0
  13. arsenal_core-0.2.0/src/arsenal_core/spec/__init__.py +43 -0
  14. arsenal_core-0.2.0/src/arsenal_core/spec/loader.py +67 -0
  15. arsenal_core-0.2.0/src/arsenal_core/spec/models.py +260 -0
  16. arsenal_core-0.2.0/src/arsenal_core/state/__init__.py +19 -0
  17. arsenal_core-0.2.0/src/arsenal_core/state/store.py +362 -0
  18. arsenal_core-0.2.0/tests/test_errors.py +35 -0
  19. arsenal_core-0.2.0/tests/test_gen_schema_docs.py +117 -0
  20. arsenal_core-0.2.0/tests/test_identity.py +23 -0
  21. arsenal_core-0.2.0/tests/test_import_core.py +19 -0
  22. arsenal_core-0.2.0/tests/test_ratelimit.py +67 -0
  23. arsenal_core-0.2.0/tests/test_retry.py +52 -0
  24. arsenal_core-0.2.0/tests/test_schema.py +31 -0
  25. arsenal_core-0.2.0/tests/test_schema_snapshot.py +64 -0
  26. arsenal_core-0.2.0/tests/test_spec.py +95 -0
  27. arsenal_core-0.2.0/tests/test_spec_m2.py +193 -0
  28. arsenal_core-0.2.0/tests/test_state_cursor.py +13 -0
  29. arsenal_core-0.2.0/tests/test_state_quarantine.py +47 -0
  30. arsenal_core-0.2.0/tests/test_state_store.py +129 -0
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ *.egg-info/
5
+ dist/
6
+ .coverage
7
+ coverage.xml
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .arsenal/
11
+ data/
12
+ .DS_Store
@@ -0,0 +1,53 @@
1
+ Metadata-Version: 2.5
2
+ Name: arsenal-core
3
+ Version: 0.2.0
4
+ Summary: Shared reliability core for DE Arsenal tools — state, identity, errors, spec, retry
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ License-Expression: Apache-2.0
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: License :: OSI Approved :: Apache Software License
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
+ Requires-Python: >=3.11
16
+ Requires-Dist: pydantic>=2.7
17
+ Requires-Dist: pyyaml>=6
18
+ Requires-Dist: tenacity>=8.3
19
+ Description-Content-Type: text/markdown
20
+
21
+ # arsenal-core
22
+
23
+ DE Arsenal(Pugio·Gladius·Arsenal)이 공유하는 신뢰성 코어 라이브러리 —
24
+ 파이프라인/변환 스펙 로더(Pydantic), 결정적 Unit ID, `StateStore`(SQLite
25
+ 기반 unit 상태 기록), 에러 분류 체계(`RetryableError`/`AuthExpiredError`/
26
+ `FatalError`), backoff 재시도 래퍼. 그 자체로는 CLI가 없는 라이브러리 패키지다
27
+ — Pugio/Gladius가 이 위에서 "재개·멱등·에러 분류가 기본값"을 구현한다.
28
+
29
+ ## 사용 예
30
+
31
+ ```python
32
+ from pathlib import Path
33
+
34
+ from arsenal_core.state import UnitSpec, StateStore
35
+ from arsenal_core.errors import RetryableError, FatalError
36
+
37
+ # 결정적 unit_id: 같은 (pipeline, source, unit_key) → 항상 같은 id → 재실행 시 중복 방지
38
+ unit = UnitSpec.create(
39
+ pipeline="github-issues", source="issues", unit_key="page=1", payload={}
40
+ )
41
+
42
+ store = StateStore(Path("./state/github-issues.db"))
43
+ store.register(unit)
44
+ ... # fetch → write
45
+ store.mark_done(unit.unit_id, row_count=100, byte_count=8192)
46
+ store.close()
47
+ ```
48
+
49
+ ## 문서
50
+
51
+ - [docs/02-architecture.md](../../docs/02-architecture.md) — 공통 코어·상태 모델·멱등 전략
52
+ - [docs/08-limits.md](../../docs/08-limits.md) — 정직한 한계선
53
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,33 @@
1
+ # arsenal-core
2
+
3
+ DE Arsenal(Pugio·Gladius·Arsenal)이 공유하는 신뢰성 코어 라이브러리 —
4
+ 파이프라인/변환 스펙 로더(Pydantic), 결정적 Unit ID, `StateStore`(SQLite
5
+ 기반 unit 상태 기록), 에러 분류 체계(`RetryableError`/`AuthExpiredError`/
6
+ `FatalError`), backoff 재시도 래퍼. 그 자체로는 CLI가 없는 라이브러리 패키지다
7
+ — Pugio/Gladius가 이 위에서 "재개·멱등·에러 분류가 기본값"을 구현한다.
8
+
9
+ ## 사용 예
10
+
11
+ ```python
12
+ from pathlib import Path
13
+
14
+ from arsenal_core.state import UnitSpec, StateStore
15
+ from arsenal_core.errors import RetryableError, FatalError
16
+
17
+ # 결정적 unit_id: 같은 (pipeline, source, unit_key) → 항상 같은 id → 재실행 시 중복 방지
18
+ unit = UnitSpec.create(
19
+ pipeline="github-issues", source="issues", unit_key="page=1", payload={}
20
+ )
21
+
22
+ store = StateStore(Path("./state/github-issues.db"))
23
+ store.register(unit)
24
+ ... # fetch → write
25
+ store.mark_done(unit.unit_id, row_count=100, byte_count=8192)
26
+ store.close()
27
+ ```
28
+
29
+ ## 문서
30
+
31
+ - [docs/02-architecture.md](../../docs/02-architecture.md) — 공통 코어·상태 모델·멱등 전략
32
+ - [docs/08-limits.md](../../docs/08-limits.md) — 정직한 한계선
33
+ - [저장소 루트 README](../../README.md)
@@ -0,0 +1,28 @@
1
+ [project]
2
+ name = "arsenal-core"
3
+ version = "0.2.0"
4
+ description = "Shared reliability core for DE Arsenal tools — state, identity, errors, spec, retry"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ license = "Apache-2.0"
8
+ dependencies = ["pydantic>=2.7", "pyyaml>=6", "tenacity>=8.3"]
9
+ classifiers = [
10
+ "Programming Language :: Python :: 3",
11
+ "Programming Language :: Python :: 3.11",
12
+ "Programming Language :: Python :: 3.12",
13
+ "License :: OSI Approved :: Apache Software License",
14
+ "Operating System :: OS Independent",
15
+ "Intended Audience :: Developers",
16
+ "Topic :: Software Development :: Libraries :: Python Modules",
17
+ ]
18
+
19
+ [project.urls]
20
+ Homepage = "https://github.com/KooEric/de-arsenal"
21
+ Repository = "https://github.com/KooEric/de-arsenal"
22
+
23
+ [build-system]
24
+ requires = ["hatchling"]
25
+ build-backend = "hatchling.build"
26
+
27
+ [tool.hatch.build.targets.wheel]
28
+ packages = ["src/arsenal_core"]
@@ -0,0 +1,15 @@
1
+ """arsenal-core — DE Arsenal 공통 신뢰성 코어.
2
+
3
+ 모든 도구(pugio, gladius, …)의 유일한 공통 의존성.
4
+ 구조: docs/02-architecture.md 참조.
5
+ """
6
+
7
+ from importlib.metadata import PackageNotFoundError, version
8
+
9
+ try:
10
+ __version__ = version("arsenal-core")
11
+ except PackageNotFoundError:
12
+ __version__ = "0.2.0"
13
+ from arsenal_core.schema import SchemaChange, SchemaDiff, compare_schema_snapshots
14
+
15
+ __all__ = ["SchemaChange", "SchemaDiff", "compare_schema_snapshots"]
@@ -0,0 +1,35 @@
1
+ """에러 분류 체계 — 커넥터가 분류하고, 러너는 타입만 보고 행동한다.
2
+
3
+ 설계: docs/02-architecture.md "에러 분류 체계"
4
+ 구현: docs/plans/2026-07-08-m1-core-foundation.md Task 1
5
+ """
6
+
7
+
8
+ class ArsenalError(Exception):
9
+ """모든 Arsenal 예외의 루트."""
10
+
11
+
12
+ class FatalError(ArsenalError):
13
+ """재시도 무의미: 설정 오류, 4xx(401/429 제외), 계약 위반. 즉시 중단."""
14
+
15
+
16
+ class RetryableError(ArsenalError):
17
+ """일시적 실패: 네트워크, 5xx, 429, lock 충돌. backoff 재시도."""
18
+
19
+
20
+ class AuthExpiredError(RetryableError):
21
+ """인증 만료. 에러가 아니라 갱신 트리거 (M2에서 refresh hook 연결)."""
22
+
23
+
24
+ def classify_http_status(status: int) -> type[ArsenalError] | None:
25
+ """HTTP 상태 코드를 예외 타입으로 분류. 2xx/3xx는 None.
26
+
27
+ 401→AuthExpired, 429/5xx→Retryable, 그 외 4xx→Fatal.
28
+ """
29
+ if status < 400:
30
+ return None
31
+ if status == 401:
32
+ return AuthExpiredError
33
+ if status == 429 or status >= 500:
34
+ return RetryableError
35
+ return FatalError
@@ -0,0 +1,18 @@
1
+ """결정적 Unit ID — 멱등성의 근거. 같은 unit은 언제 실행해도 같은 ID.
2
+
3
+ 설계: docs/02-architecture.md "결정적 ID"
4
+ 구현: docs/plans/2026-07-08-m1-core-foundation.md Task 2
5
+ """
6
+
7
+ import hashlib
8
+
9
+ _ID_LEN = 16
10
+
11
+
12
+ def unit_id(pipeline: str, source: str, unit_key: str) -> str:
13
+ """sha256("{pipeline}\\x1f{source}\\x1f{unit_key}") 앞 16 hex.
14
+
15
+ \\x1f 구분자로 경계 모호성 제거. 결정성·충돌 저항은 hypothesis로 검증.
16
+ """
17
+ raw = f"{pipeline}\x1f{source}\x1f{unit_key}"
18
+ return hashlib.sha256(raw.encode()).hexdigest()[:_ID_LEN]
@@ -0,0 +1,8 @@
1
+ """Arrow/Parquet 허브 유틸 (예약 모듈).
2
+
3
+ 도구 간 교환 포맷은 Arrow RecordBatch, 저장 포맷은 Parquet — 모든 도구가
4
+ 같은 파일을 읽는다 (단일 스토리지 복사본, docs/07-cost-efficiency.md).
5
+
6
+ M1에서는 pyarrow 직접 사용으로 충분해 비워둔다. 공통 헬퍼(스키마 스냅샷
7
+ 직렬화, 증분 판별 등)가 두 도구 이상에서 반복되는 시점에 채운다 (YAGNI).
8
+ """
File without changes
@@ -0,0 +1,58 @@
1
+ """토큰 버킷 rate limiter — REST 소스가 rps 상한을 지키도록 acquire()로 스로틀한다.
2
+
3
+ 설계: docs/02-architecture.md M2 rate_limit. penalize()는 429 Retry-After를
4
+ 흡수해 다음 acquire()가 그만큼 더 기다리게 만든다 (with_retry의 재시도 루프와
5
+ 맞물려 "실패 → 벌점 → 다음 시도 전 대기"를 구현한다).
6
+ """
7
+
8
+ import time
9
+ from typing import Protocol
10
+
11
+
12
+ class Clock(Protocol):
13
+ def monotonic(self) -> float: ...
14
+ def sleep(self, seconds: float) -> None: ...
15
+
16
+
17
+ class SystemClock:
18
+ """실제 time.monotonic()/time.sleep()을 쓰는 Clock 구현체 — 다른 패키지(pugio.auth
19
+ 등)도 재선언 없이 import해 쓸 수 있도록 공개 이름으로 export한다."""
20
+
21
+ def monotonic(self) -> float:
22
+ return time.monotonic()
23
+
24
+ def sleep(self, seconds: float) -> None:
25
+ time.sleep(seconds)
26
+
27
+
28
+ # 하위 호환: 기존 내부 전용 이름으로 참조하던 코드(및 다른 패키지의 재선언부)가
29
+ # 있을 수 있어 별칭을 유지한다.
30
+ _MonotonicClock = SystemClock
31
+
32
+
33
+ class TokenBucket:
34
+ """rps 상한 토큰 버킷. penalize()로 429 Retry-After를 흡수한다."""
35
+
36
+ def __init__(self, rps: float, *, clock: Clock | None = None, burst: int = 1) -> None:
37
+ self._interval = 1.0 / rps
38
+ self._clock = clock or _MonotonicClock()
39
+ self._burst = float(burst)
40
+ self._tokens = float(burst)
41
+ self._last = self._clock.monotonic()
42
+ self._penalty_until = 0.0
43
+
44
+ def acquire(self) -> None:
45
+ now = self._clock.monotonic()
46
+ if now < self._penalty_until:
47
+ self._clock.sleep(self._penalty_until - now)
48
+ now = self._clock.monotonic()
49
+ self._tokens = min(self._burst, self._tokens + (now - self._last) / self._interval)
50
+ self._last = now
51
+ if self._tokens < 1.0:
52
+ self._clock.sleep((1.0 - self._tokens) * self._interval)
53
+ self._tokens = self._burst
54
+ self._last = self._clock.monotonic()
55
+ self._tokens -= 1.0
56
+
57
+ def penalize(self, seconds: float) -> None:
58
+ self._penalty_until = self._clock.monotonic() + seconds
@@ -0,0 +1,46 @@
1
+ """Retryable만 재시도하는 backoff 래퍼. 엔진은 빌린다 — tenacity 사용.
2
+
3
+ 구현: docs/plans/2026-07-08-m1-core-foundation.md Task 5
4
+ FatalError는 즉시 전파, RetryableError만 exponential backoff + jitter.
5
+
6
+ M2-D: AuthExpiredError는 RetryableError의 서브클래스지만 여기서는 재시도 대상에서
7
+ 제외한다. 401을 blind하게 backoff 재시도해봐야 provider.refresh()가 호출되지
8
+ 않는 한 계속 401만 받는다 — 갱신 없는 재시도는 의미가 없다. AuthExpiredError는
9
+ 즉시 전파시켜 runner.py가 "refresh() 후 한 번 더 시도"를 명시적으로 수행하게
10
+ 한다 (러너 쪽 갱신 루프, docs/02-architecture.md M2-D).
11
+ """
12
+
13
+ from collections.abc import Callable
14
+ from typing import TypeVar
15
+
16
+ from tenacity import (
17
+ retry,
18
+ retry_if_exception_type,
19
+ retry_if_not_exception_type,
20
+ stop_after_attempt,
21
+ wait_exponential_jitter,
22
+ )
23
+
24
+ from arsenal_core.errors import AuthExpiredError, RetryableError
25
+
26
+ T = TypeVar("T")
27
+
28
+ DEFAULT_MAX_ATTEMPTS = 5
29
+ DEFAULT_BASE_WAIT = 1.0 # seconds
30
+ MAX_WAIT = 60.0 # seconds
31
+
32
+
33
+ def with_retry(
34
+ fn: Callable[[], T],
35
+ *,
36
+ max_attempts: int = DEFAULT_MAX_ATTEMPTS,
37
+ base_wait: float = DEFAULT_BASE_WAIT,
38
+ ) -> T:
39
+ wrapped = retry(
40
+ retry=retry_if_exception_type(RetryableError)
41
+ & retry_if_not_exception_type(AuthExpiredError),
42
+ stop=stop_after_attempt(max_attempts),
43
+ wait=wait_exponential_jitter(initial=base_wait, max=MAX_WAIT),
44
+ reraise=True,
45
+ )(fn)
46
+ return wrapped()
@@ -0,0 +1,81 @@
1
+ """Schema snapshot comparison primitives shared by collection components."""
2
+
3
+ import json
4
+ from dataclasses import dataclass
5
+ from typing import Any, cast
6
+
7
+
8
+ @dataclass(frozen=True)
9
+ class SchemaChange:
10
+ """One field-level schema change."""
11
+
12
+ field: str
13
+ before: dict[str, Any] | None
14
+ after: dict[str, Any] | None
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class SchemaDiff:
19
+ """Added, removed, and modified fields between two snapshots."""
20
+
21
+ added: tuple[SchemaChange, ...]
22
+ removed: tuple[SchemaChange, ...]
23
+ changed: tuple[SchemaChange, ...]
24
+
25
+ @property
26
+ def has_changes(self) -> bool:
27
+ return bool(self.added or self.removed or self.changed)
28
+
29
+ def summary(self) -> str:
30
+ """Return a compact, stable message suitable for logs and state."""
31
+ parts: list[str] = []
32
+ if self.added:
33
+ parts.append("added=" + ",".join(change.field for change in self.added))
34
+ if self.removed:
35
+ parts.append("removed=" + ",".join(change.field for change in self.removed))
36
+ if self.changed:
37
+ parts.append("changed=" + ",".join(change.field for change in self.changed))
38
+ return "; ".join(parts)
39
+
40
+
41
+ def compare_schema_snapshots(previous: str, current: str) -> SchemaDiff:
42
+ """Compare snapshots produced by the runner's deterministic JSON encoder.
43
+
44
+ Snapshot entries are intentionally treated as opaque dictionaries except for
45
+ their ``name`` key. This lets future snapshot metadata be added without
46
+ changing the comparison algorithm, while still detecting type/nullability
47
+ changes in today's format.
48
+ """
49
+ previous_fields = _fields(previous)
50
+ current_fields = _fields(current)
51
+ added: list[SchemaChange] = []
52
+ removed: list[SchemaChange] = []
53
+ changed: list[SchemaChange] = []
54
+
55
+ for name in sorted(current_fields.keys() - previous_fields.keys()):
56
+ added.append(SchemaChange(name, None, current_fields[name]))
57
+ for name in sorted(previous_fields.keys() - current_fields.keys()):
58
+ removed.append(SchemaChange(name, previous_fields[name], None))
59
+ for name in sorted(current_fields.keys() & previous_fields.keys()):
60
+ before = previous_fields[name]
61
+ after = current_fields[name]
62
+ if before != after:
63
+ changed.append(SchemaChange(name, before, after))
64
+
65
+ return SchemaDiff(tuple(added), tuple(removed), tuple(changed))
66
+
67
+
68
+ def _fields(snapshot: str) -> dict[str, dict[str, Any]]:
69
+ raw = cast(object, json.loads(snapshot))
70
+ if not isinstance(raw, list):
71
+ raise ValueError("schema snapshot must be a JSON list")
72
+ entries = cast(list[object], raw)
73
+ fields: dict[str, dict[str, Any]] = {}
74
+ for raw_entry in entries:
75
+ if not isinstance(raw_entry, dict):
76
+ raise ValueError("schema snapshot entries must contain a string name")
77
+ entry = cast(dict[str, Any], raw_entry)
78
+ if not isinstance(entry.get("name"), str):
79
+ raise ValueError("schema snapshot entries must contain a string name")
80
+ fields[entry["name"]] = entry
81
+ return fields
@@ -0,0 +1,43 @@
1
+ from arsenal_core.spec.loader import load_pipeline
2
+ from arsenal_core.spec.models import (
3
+ AuthSpec,
4
+ ContractColumnSpec,
5
+ ContractSpec,
6
+ DatabaseSourceSpec,
7
+ DuckDBSinkSpec,
8
+ FileSourceSpec,
9
+ PaginationSpec,
10
+ ParquetSinkSpec,
11
+ PipelineSpec,
12
+ PostgresSinkSpec,
13
+ PythonSourceSpec,
14
+ RateLimitSpec,
15
+ RestSourceSpec,
16
+ SinkSpec,
17
+ SourceSpec,
18
+ SplitSpec,
19
+ ValidateRule,
20
+ ValidateSpec,
21
+ )
22
+
23
+ __all__ = [
24
+ "AuthSpec",
25
+ "ContractColumnSpec",
26
+ "ContractSpec",
27
+ "DatabaseSourceSpec",
28
+ "DuckDBSinkSpec",
29
+ "FileSourceSpec",
30
+ "PaginationSpec",
31
+ "ParquetSinkSpec",
32
+ "PipelineSpec",
33
+ "PostgresSinkSpec",
34
+ "PythonSourceSpec",
35
+ "RateLimitSpec",
36
+ "RestSourceSpec",
37
+ "SinkSpec",
38
+ "SourceSpec",
39
+ "SplitSpec",
40
+ "ValidateRule",
41
+ "ValidateSpec",
42
+ "load_pipeline",
43
+ ]
@@ -0,0 +1,67 @@
1
+ """YAML → 검증된 PipelineSpec. 에러 메시지 품질이 곧 UX.
2
+
3
+ 구현: docs/plans/2026-07-08-m1-core-foundation.md Task 4
4
+ - ${VAR} 환경변수 치환 (비밀은 YAML에 두지 않는다)
5
+ - 검증 실패 시 필드 경로가 담긴 FatalError
6
+ """
7
+
8
+ import os
9
+ import re
10
+ from pathlib import Path
11
+
12
+ import yaml
13
+ from pydantic import ValidationError
14
+
15
+ from arsenal_core.errors import FatalError
16
+ from arsenal_core.spec.models import PipelineSpec
17
+
18
+ _ENV_PATTERN = re.compile(r"\$\{(\w+)\}")
19
+
20
+ # discriminated union 필드 → 허용된 태그 값. Pydantic은 이런 필드의 에러 loc에
21
+ # 태그를 끼워 넣는다(예: ("source","rest","url")) — 사용자에게는 YAML 그대로의
22
+ # 경로(source.url)가 더 읽기 쉽고, M1 에러 메시지와도 하위 호환된다.
23
+ _UNION_TAGS: dict[str, frozenset[str]] = {
24
+ "source": frozenset({"rest", "file", "database", "python", "dlt"}),
25
+ "sink": frozenset({"parquet", "duckdb", "postgres"}),
26
+ }
27
+
28
+
29
+ def _substitute_env(text: str) -> str:
30
+ def repl(m: re.Match[str]) -> str:
31
+ value = os.environ.get(m.group(1))
32
+ if value is None:
33
+ raise FatalError(f"environment variable not set: {m.group(1)}")
34
+ return value
35
+
36
+ return _ENV_PATTERN.sub(repl, text)
37
+
38
+
39
+ def _clean_loc(loc: tuple[int | str, ...]) -> tuple[int | str, ...]:
40
+ """discriminated union의 태그 세그먼트를 제거한다 (source.rest.url → source.url)."""
41
+ cleaned: list[int | str] = []
42
+ for i, part in enumerate(loc):
43
+ prev = loc[i - 1] if i > 0 else None
44
+ allowed = _UNION_TAGS.get(str(prev)) if isinstance(prev, str) else None
45
+ if allowed is not None and isinstance(part, str) and part in allowed:
46
+ continue
47
+ cleaned.append(part)
48
+ return tuple(cleaned)
49
+
50
+
51
+ def load_pipeline(path: Path) -> PipelineSpec:
52
+ try:
53
+ text = path.read_text(encoding="utf-8")
54
+ except FileNotFoundError as e:
55
+ raise FatalError(f"spec file not found: {path}") from e
56
+ except OSError as e:
57
+ raise FatalError(f"cannot read spec file {path}: {e}") from e
58
+
59
+ raw = yaml.safe_load(_substitute_env(text))
60
+ try:
61
+ return PipelineSpec.model_validate(raw)
62
+ except ValidationError as e:
63
+ lines = [
64
+ f"{'.'.join(str(p) for p in _clean_loc(err['loc']))}: {err['msg']}"
65
+ for err in e.errors()
66
+ ]
67
+ raise FatalError(f"invalid pipeline spec {path}:\n " + "\n ".join(lines)) from e