de-scutum 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ *.egg-info/
5
+ dist/
6
+ .coverage
7
+ coverage.xml
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .arsenal/
11
+ data/
12
+ .DS_Store
@@ -0,0 +1,32 @@
1
+ Metadata-Version: 2.5
2
+ Name: de-scutum
3
+ Version: 0.2.2
4
+ Summary: Scutum — Arrow data contracts and validation reports
5
+ Project-URL: Homepage, https://github.com/KooEric/de-arsenal
6
+ Project-URL: Repository, https://github.com/KooEric/de-arsenal
7
+ License-Expression: Apache-2.0
8
+ Classifier: License :: OSI Approved :: Apache Software License
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Requires-Python: >=3.11
14
+ Requires-Dist: arsenal-core<0.3,>=0.2
15
+ Requires-Dist: pyarrow>=17
16
+ Requires-Dist: pydantic>=2.7
17
+ Description-Content-Type: text/markdown
18
+
19
+ # Scutum
20
+
21
+ Arrow data contract와 파일 lock retry/backoff를 제공한다. Pugio의
22
+ `PipelineSpec.contract`가 계약 검증과 DLQ 정책에 연결된다.
23
+
24
+ PyPI 배포명은 `de-scutum`이며 Python import 경로는 `scutum`을 유지한다.
25
+
26
+ ```bash
27
+ pip install de-scutum
28
+ ```
29
+
30
+ ```python
31
+ from scutum import DataContract
32
+ ```
@@ -0,0 +1,14 @@
1
+ # Scutum
2
+
3
+ Arrow data contract와 파일 lock retry/backoff를 제공한다. Pugio의
4
+ `PipelineSpec.contract`가 계약 검증과 DLQ 정책에 연결된다.
5
+
6
+ PyPI 배포명은 `de-scutum`이며 Python import 경로는 `scutum`을 유지한다.
7
+
8
+ ```bash
9
+ pip install de-scutum
10
+ ```
11
+
12
+ ```python
13
+ from scutum import DataContract
14
+ ```
@@ -0,0 +1,26 @@
1
+ [project]
2
+ name = "de-scutum"
3
+ version = "0.2.2"
4
+ description = "Scutum — Arrow data contracts and validation reports"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ license = "Apache-2.0"
8
+ dependencies = ["arsenal-core>=0.2,<0.3", "pyarrow>=17", "pydantic>=2.7"]
9
+ classifiers = [
10
+ "Programming Language :: Python :: 3",
11
+ "Programming Language :: Python :: 3.11",
12
+ "Programming Language :: Python :: 3.12",
13
+ "License :: OSI Approved :: Apache Software License",
14
+ "Operating System :: OS Independent",
15
+ ]
16
+
17
+ [project.urls]
18
+ Homepage = "https://github.com/KooEric/de-arsenal"
19
+ Repository = "https://github.com/KooEric/de-arsenal"
20
+
21
+ [build-system]
22
+ requires = ["hatchling"]
23
+ build-backend = "hatchling.build"
24
+
25
+ [tool.hatch.build.targets.wheel]
26
+ packages = ["src/scutum"]
@@ -0,0 +1,20 @@
1
+ """Scutum — explicit data contracts for Arrow batches."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("de-scutum")
7
+ except PackageNotFoundError:
8
+ __version__ = "0.2.2"
9
+
10
+ from scutum.contract import ContractColumn, ContractReport, DataContract
11
+ from scutum.lock import FileLock, LockConflict, file_lock
12
+
13
+ __all__ = [
14
+ "ContractColumn",
15
+ "ContractReport",
16
+ "DataContract",
17
+ "FileLock",
18
+ "LockConflict",
19
+ "file_lock",
20
+ ]
@@ -0,0 +1,63 @@
1
+ """Data contract models and vectorized Arrow validation."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+ import pyarrow as pa
6
+ from pydantic import BaseModel, ConfigDict, Field
7
+
8
+
9
+ class ContractColumn(BaseModel):
10
+ """Expected field type and nullability."""
11
+
12
+ model_config = ConfigDict(frozen=True, extra="forbid")
13
+
14
+ name: str
15
+ type: str = Field(description="Arrow type string, for example int64 or string")
16
+ nullable: bool = True
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class ContractViolation:
21
+ field: str
22
+ message: str
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class ContractReport:
27
+ contract: str
28
+ ok: bool
29
+ violations: tuple[ContractViolation, ...]
30
+
31
+
32
+ class DataContract(BaseModel):
33
+ """A strict or additive schema contract for a RecordBatch."""
34
+
35
+ model_config = ConfigDict(frozen=True, extra="forbid")
36
+
37
+ name: str
38
+ columns: list[ContractColumn] = Field(min_length=1)
39
+ allow_extra: bool = False
40
+
41
+ def check(self, batch: pa.RecordBatch) -> ContractReport:
42
+ actual = {field.name: field for field in batch.schema}
43
+ expected = {column.name: column for column in self.columns}
44
+ violations: list[ContractViolation] = []
45
+ for column in self.columns:
46
+ field = actual.get(column.name)
47
+ if field is None:
48
+ violations.append(ContractViolation(column.name, "missing column"))
49
+ continue
50
+ if str(field.type) != column.type:
51
+ violations.append(
52
+ ContractViolation(
53
+ column.name,
54
+ f"type mismatch: expected {column.type}, got {field.type}",
55
+ )
56
+ )
57
+ if not column.nullable and batch.column(column.name).null_count > 0:
58
+ violations.append(ContractViolation(column.name, "contains null values"))
59
+ if not self.allow_extra:
60
+ for name in actual:
61
+ if name not in expected:
62
+ violations.append(ContractViolation(name, "unexpected column"))
63
+ return ContractReport(self.name, not violations, tuple(violations))
@@ -0,0 +1,86 @@
1
+ """Small cross-process file lock with bounded retry/backoff."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import time
7
+ from collections.abc import Callable, Generator
8
+ from contextlib import contextmanager, suppress
9
+ from pathlib import Path
10
+
11
+ from arsenal_core.errors import RetryableError
12
+
13
+
14
+ class LockConflict(RetryableError):
15
+ """The lock remained held after the configured retry budget."""
16
+
17
+
18
+ class FileLock:
19
+ """Exclusive lock file using O_EXCL and deterministic retry policy."""
20
+
21
+ def __init__(
22
+ self,
23
+ path: Path,
24
+ *,
25
+ attempts: int = 5,
26
+ backoff_seconds: float = 0.1,
27
+ sleep: Callable[[float], None] = time.sleep,
28
+ ) -> None:
29
+ if attempts < 1:
30
+ raise ValueError("attempts must be at least 1")
31
+ if backoff_seconds < 0:
32
+ raise ValueError("backoff_seconds must not be negative")
33
+ self.path = path
34
+ self.attempts = attempts
35
+ self.backoff_seconds = backoff_seconds
36
+ self._sleep = sleep
37
+ self._held = False
38
+
39
+ def acquire(self) -> None:
40
+ self.path.parent.mkdir(parents=True, exist_ok=True)
41
+ for attempt in range(self.attempts):
42
+ try:
43
+ fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
44
+ except FileExistsError:
45
+ if attempt == self.attempts - 1:
46
+ raise LockConflict(
47
+ f"lock busy after {self.attempts} attempts: {self.path}"
48
+ ) from None
49
+ self._sleep(self.backoff_seconds * (2**attempt))
50
+ else:
51
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
52
+ handle.write(f"pid={os.getpid()}\n")
53
+ self._held = True
54
+ return
55
+
56
+ def release(self) -> None:
57
+ if self._held:
58
+ with suppress(FileNotFoundError):
59
+ self.path.unlink()
60
+ self._held = False
61
+
62
+ def __enter__(self) -> FileLock:
63
+ self.acquire()
64
+ return self
65
+
66
+ def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
67
+ self.release()
68
+
69
+
70
+ @contextmanager
71
+ def file_lock(
72
+ path: Path,
73
+ *,
74
+ attempts: int = 5,
75
+ backoff_seconds: float = 0.1,
76
+ sleep: Callable[[float], None] = time.sleep,
77
+ ) -> Generator[FileLock, None, None]:
78
+ """Context-manager convenience wrapper around :class:`FileLock`."""
79
+ lock = FileLock(
80
+ path,
81
+ attempts=attempts,
82
+ backoff_seconds=backoff_seconds,
83
+ sleep=sleep,
84
+ )
85
+ with lock:
86
+ yield lock
File without changes
@@ -0,0 +1,37 @@
1
+ import pyarrow as pa
2
+ import pytest
3
+ from pydantic import ValidationError
4
+ from scutum import DataContract
5
+
6
+
7
+ def contract(**kwargs: object) -> DataContract:
8
+ return DataContract.model_validate(
9
+ {
10
+ "name": "orders",
11
+ "columns": [{"name": "id", "type": "int64", "nullable": False}],
12
+ **kwargs,
13
+ }
14
+ )
15
+
16
+
17
+ def test_contract_accepts_matching_batch() -> None:
18
+ batch = pa.RecordBatch.from_arrays(
19
+ [pa.array([1, 2], type=pa.int64())],
20
+ schema=pa.schema([pa.field("id", pa.int64(), nullable=False)]),
21
+ )
22
+ report = contract().check(batch)
23
+ assert report.ok is True
24
+ assert report.violations == ()
25
+
26
+
27
+ def test_contract_reports_missing_type_null_and_extra() -> None:
28
+ report = contract().check(pa.RecordBatch.from_pylist([{"id": None, "extra": "x"}]))
29
+ messages = {(violation.field, violation.message) for violation in report.violations}
30
+ assert report.ok is False
31
+ assert ("id", "contains null values") in messages
32
+ assert ("extra", "unexpected column") in messages
33
+
34
+
35
+ def test_contract_rejects_empty_columns() -> None:
36
+ with pytest.raises(ValidationError):
37
+ DataContract.model_validate({"name": "empty", "columns": []})
@@ -0,0 +1,32 @@
1
+ from pathlib import Path
2
+
3
+ import pytest
4
+ from scutum import FileLock, LockConflict
5
+
6
+
7
+ def test_file_lock_retries_then_acquires(tmp_path: Path) -> None:
8
+ path = tmp_path / "dataset.lock"
9
+ path.write_text("holder", encoding="utf-8")
10
+ slept: list[float] = []
11
+
12
+ lock = FileLock(path, attempts=3, backoff_seconds=0.25, sleep=slept.append)
13
+
14
+ def release_holder(seconds: float) -> None:
15
+ slept.append(seconds)
16
+ if len(slept) == 2:
17
+ path.unlink()
18
+
19
+ lock = FileLock(path, attempts=3, backoff_seconds=0.25, sleep=release_holder)
20
+ lock.acquire()
21
+ try:
22
+ assert path.exists()
23
+ assert slept == [0.25, 0.5]
24
+ finally:
25
+ lock.release()
26
+
27
+
28
+ def test_file_lock_raises_retryable_conflict(tmp_path: Path) -> None:
29
+ path = tmp_path / "dataset.lock"
30
+ path.write_text("holder", encoding="utf-8")
31
+ with pytest.raises(LockConflict, match="lock busy"):
32
+ FileLock(path, attempts=2, backoff_seconds=0, sleep=lambda _: None).acquire()