de-scutum 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_scutum-0.2.2/.gitignore +12 -0
- de_scutum-0.2.2/PKG-INFO +32 -0
- de_scutum-0.2.2/README.md +14 -0
- de_scutum-0.2.2/pyproject.toml +26 -0
- de_scutum-0.2.2/src/scutum/__init__.py +20 -0
- de_scutum-0.2.2/src/scutum/contract.py +63 -0
- de_scutum-0.2.2/src/scutum/lock.py +86 -0
- de_scutum-0.2.2/src/scutum/py.typed +0 -0
- de_scutum-0.2.2/tests/test_contract.py +37 -0
- de_scutum-0.2.2/tests/test_lock.py +32 -0
de_scutum-0.2.2/PKG-INFO
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: de-scutum
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Scutum — Arrow data contracts and validation reports
|
|
5
|
+
Project-URL: Homepage, https://github.com/KooEric/de-arsenal
|
|
6
|
+
Project-URL: Repository, https://github.com/KooEric/de-arsenal
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Requires-Dist: arsenal-core<0.3,>=0.2
|
|
15
|
+
Requires-Dist: pyarrow>=17
|
|
16
|
+
Requires-Dist: pydantic>=2.7
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# Scutum
|
|
20
|
+
|
|
21
|
+
Arrow data contract와 파일 lock retry/backoff를 제공한다. Pugio의
|
|
22
|
+
`PipelineSpec.contract`가 계약 검증과 DLQ 정책에 연결된다.
|
|
23
|
+
|
|
24
|
+
PyPI 배포명은 `de-scutum`이며 Python import 경로는 `scutum`을 유지한다.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install de-scutum
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from scutum import DataContract
|
|
32
|
+
```
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Scutum
|
|
2
|
+
|
|
3
|
+
Arrow data contract와 파일 lock retry/backoff를 제공한다. Pugio의
|
|
4
|
+
`PipelineSpec.contract`가 계약 검증과 DLQ 정책에 연결된다.
|
|
5
|
+
|
|
6
|
+
PyPI 배포명은 `de-scutum`이며 Python import 경로는 `scutum`을 유지한다.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install de-scutum
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from scutum import DataContract
|
|
14
|
+
```
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "de-scutum"
|
|
3
|
+
version = "0.2.2"
|
|
4
|
+
description = "Scutum — Arrow data contracts and validation reports"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "Apache-2.0"
|
|
8
|
+
dependencies = ["arsenal-core>=0.2,<0.3", "pyarrow>=17", "pydantic>=2.7"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
"Programming Language :: Python :: 3.11",
|
|
12
|
+
"Programming Language :: Python :: 3.12",
|
|
13
|
+
"License :: OSI Approved :: Apache Software License",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.urls]
|
|
18
|
+
Homepage = "https://github.com/KooEric/de-arsenal"
|
|
19
|
+
Repository = "https://github.com/KooEric/de-arsenal"
|
|
20
|
+
|
|
21
|
+
[build-system]
|
|
22
|
+
requires = ["hatchling"]
|
|
23
|
+
build-backend = "hatchling.build"
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.wheel]
|
|
26
|
+
packages = ["src/scutum"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Scutum — explicit data contracts for Arrow batches."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("de-scutum")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "0.2.2"
|
|
9
|
+
|
|
10
|
+
from scutum.contract import ContractColumn, ContractReport, DataContract
|
|
11
|
+
from scutum.lock import FileLock, LockConflict, file_lock
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"ContractColumn",
|
|
15
|
+
"ContractReport",
|
|
16
|
+
"DataContract",
|
|
17
|
+
"FileLock",
|
|
18
|
+
"LockConflict",
|
|
19
|
+
"file_lock",
|
|
20
|
+
]
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Data contract models and vectorized Arrow validation."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import pyarrow as pa
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ContractColumn(BaseModel):
|
|
10
|
+
"""Expected field type and nullability."""
|
|
11
|
+
|
|
12
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
13
|
+
|
|
14
|
+
name: str
|
|
15
|
+
type: str = Field(description="Arrow type string, for example int64 or string")
|
|
16
|
+
nullable: bool = True
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ContractViolation:
|
|
21
|
+
field: str
|
|
22
|
+
message: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class ContractReport:
|
|
27
|
+
contract: str
|
|
28
|
+
ok: bool
|
|
29
|
+
violations: tuple[ContractViolation, ...]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DataContract(BaseModel):
|
|
33
|
+
"""A strict or additive schema contract for a RecordBatch."""
|
|
34
|
+
|
|
35
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
36
|
+
|
|
37
|
+
name: str
|
|
38
|
+
columns: list[ContractColumn] = Field(min_length=1)
|
|
39
|
+
allow_extra: bool = False
|
|
40
|
+
|
|
41
|
+
def check(self, batch: pa.RecordBatch) -> ContractReport:
|
|
42
|
+
actual = {field.name: field for field in batch.schema}
|
|
43
|
+
expected = {column.name: column for column in self.columns}
|
|
44
|
+
violations: list[ContractViolation] = []
|
|
45
|
+
for column in self.columns:
|
|
46
|
+
field = actual.get(column.name)
|
|
47
|
+
if field is None:
|
|
48
|
+
violations.append(ContractViolation(column.name, "missing column"))
|
|
49
|
+
continue
|
|
50
|
+
if str(field.type) != column.type:
|
|
51
|
+
violations.append(
|
|
52
|
+
ContractViolation(
|
|
53
|
+
column.name,
|
|
54
|
+
f"type mismatch: expected {column.type}, got {field.type}",
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
if not column.nullable and batch.column(column.name).null_count > 0:
|
|
58
|
+
violations.append(ContractViolation(column.name, "contains null values"))
|
|
59
|
+
if not self.allow_extra:
|
|
60
|
+
for name in actual:
|
|
61
|
+
if name not in expected:
|
|
62
|
+
violations.append(ContractViolation(name, "unexpected column"))
|
|
63
|
+
return ContractReport(self.name, not violations, tuple(violations))
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Small cross-process file lock with bounded retry/backoff."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Callable, Generator
|
|
8
|
+
from contextlib import contextmanager, suppress
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from arsenal_core.errors import RetryableError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LockConflict(RetryableError):
|
|
15
|
+
"""The lock remained held after the configured retry budget."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FileLock:
|
|
19
|
+
"""Exclusive lock file using O_EXCL and deterministic retry policy."""
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
path: Path,
|
|
24
|
+
*,
|
|
25
|
+
attempts: int = 5,
|
|
26
|
+
backoff_seconds: float = 0.1,
|
|
27
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
28
|
+
) -> None:
|
|
29
|
+
if attempts < 1:
|
|
30
|
+
raise ValueError("attempts must be at least 1")
|
|
31
|
+
if backoff_seconds < 0:
|
|
32
|
+
raise ValueError("backoff_seconds must not be negative")
|
|
33
|
+
self.path = path
|
|
34
|
+
self.attempts = attempts
|
|
35
|
+
self.backoff_seconds = backoff_seconds
|
|
36
|
+
self._sleep = sleep
|
|
37
|
+
self._held = False
|
|
38
|
+
|
|
39
|
+
def acquire(self) -> None:
|
|
40
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
for attempt in range(self.attempts):
|
|
42
|
+
try:
|
|
43
|
+
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
44
|
+
except FileExistsError:
|
|
45
|
+
if attempt == self.attempts - 1:
|
|
46
|
+
raise LockConflict(
|
|
47
|
+
f"lock busy after {self.attempts} attempts: {self.path}"
|
|
48
|
+
) from None
|
|
49
|
+
self._sleep(self.backoff_seconds * (2**attempt))
|
|
50
|
+
else:
|
|
51
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
52
|
+
handle.write(f"pid={os.getpid()}\n")
|
|
53
|
+
self._held = True
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
def release(self) -> None:
|
|
57
|
+
if self._held:
|
|
58
|
+
with suppress(FileNotFoundError):
|
|
59
|
+
self.path.unlink()
|
|
60
|
+
self._held = False
|
|
61
|
+
|
|
62
|
+
def __enter__(self) -> FileLock:
|
|
63
|
+
self.acquire()
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
|
|
67
|
+
self.release()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@contextmanager
|
|
71
|
+
def file_lock(
|
|
72
|
+
path: Path,
|
|
73
|
+
*,
|
|
74
|
+
attempts: int = 5,
|
|
75
|
+
backoff_seconds: float = 0.1,
|
|
76
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
77
|
+
) -> Generator[FileLock, None, None]:
|
|
78
|
+
"""Context-manager convenience wrapper around :class:`FileLock`."""
|
|
79
|
+
lock = FileLock(
|
|
80
|
+
path,
|
|
81
|
+
attempts=attempts,
|
|
82
|
+
backoff_seconds=backoff_seconds,
|
|
83
|
+
sleep=sleep,
|
|
84
|
+
)
|
|
85
|
+
with lock:
|
|
86
|
+
yield lock
|
|
File without changes
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import pyarrow as pa
|
|
2
|
+
import pytest
|
|
3
|
+
from pydantic import ValidationError
|
|
4
|
+
from scutum import DataContract
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def contract(**kwargs: object) -> DataContract:
|
|
8
|
+
return DataContract.model_validate(
|
|
9
|
+
{
|
|
10
|
+
"name": "orders",
|
|
11
|
+
"columns": [{"name": "id", "type": "int64", "nullable": False}],
|
|
12
|
+
**kwargs,
|
|
13
|
+
}
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_contract_accepts_matching_batch() -> None:
|
|
18
|
+
batch = pa.RecordBatch.from_arrays(
|
|
19
|
+
[pa.array([1, 2], type=pa.int64())],
|
|
20
|
+
schema=pa.schema([pa.field("id", pa.int64(), nullable=False)]),
|
|
21
|
+
)
|
|
22
|
+
report = contract().check(batch)
|
|
23
|
+
assert report.ok is True
|
|
24
|
+
assert report.violations == ()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_contract_reports_missing_type_null_and_extra() -> None:
|
|
28
|
+
report = contract().check(pa.RecordBatch.from_pylist([{"id": None, "extra": "x"}]))
|
|
29
|
+
messages = {(violation.field, violation.message) for violation in report.violations}
|
|
30
|
+
assert report.ok is False
|
|
31
|
+
assert ("id", "contains null values") in messages
|
|
32
|
+
assert ("extra", "unexpected column") in messages
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_contract_rejects_empty_columns() -> None:
|
|
36
|
+
with pytest.raises(ValidationError):
|
|
37
|
+
DataContract.model_validate({"name": "empty", "columns": []})
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from scutum import FileLock, LockConflict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_file_lock_retries_then_acquires(tmp_path: Path) -> None:
|
|
8
|
+
path = tmp_path / "dataset.lock"
|
|
9
|
+
path.write_text("holder", encoding="utf-8")
|
|
10
|
+
slept: list[float] = []
|
|
11
|
+
|
|
12
|
+
lock = FileLock(path, attempts=3, backoff_seconds=0.25, sleep=slept.append)
|
|
13
|
+
|
|
14
|
+
def release_holder(seconds: float) -> None:
|
|
15
|
+
slept.append(seconds)
|
|
16
|
+
if len(slept) == 2:
|
|
17
|
+
path.unlink()
|
|
18
|
+
|
|
19
|
+
lock = FileLock(path, attempts=3, backoff_seconds=0.25, sleep=release_holder)
|
|
20
|
+
lock.acquire()
|
|
21
|
+
try:
|
|
22
|
+
assert path.exists()
|
|
23
|
+
assert slept == [0.25, 0.5]
|
|
24
|
+
finally:
|
|
25
|
+
lock.release()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_file_lock_raises_retryable_conflict(tmp_path: Path) -> None:
|
|
29
|
+
path = tmp_path / "dataset.lock"
|
|
30
|
+
path.write_text("holder", encoding="utf-8")
|
|
31
|
+
with pytest.raises(LockConflict, match="lock busy"):
|
|
32
|
+
FileLock(path, attempts=2, backoff_seconds=0, sleep=lambda _: None).acquire()
|