ohmydata 0.0.1__py3-none-any.whl

ohmydata/__init__.py

@@ -0,0 +1,5 @@
+"""Provisional offline-only Oh My Data package scaffold."""
+
+__version__ = "0.0.0"
+
+__all__ = ["__version__"]

ohmydata/core/__init__.py

@@ -0,0 +1,52 @@
+from .errors import (
+    AuthenticationError,
+    CoverageError,
+    EmptyResponseError,
+    OhMyDataError,
+    PaginationError,
+    PermanentProviderError,
+    PermissionDeniedError,
+    ProviderError,
+    RateLimitError,
+    RetryExhaustedError,
+    SchemaMismatchError,
+    SnapshotConflictError,
+    SnapshotIntegrityError,
+    TransientProviderError,
+)
+from .policy import AttemptRecord, RetryPolicy, RetryResult, execute_with_retry
+from .provenance import EmptyDisposition, FetchProvenance
+from .rate_limit import RateLimitDecision, RateLimiter, RateLimitPolicy
+from .snapshot import SnapshotMode, SnapshotRef, SnapshotReplay, SnapshotStore
+from .specs import RequestSpec
+
+__all__ = [
+    "AttemptRecord",
+    "AuthenticationError",
+    "CoverageError",
+    "EmptyDisposition",
+    "EmptyResponseError",
+    "FetchProvenance",
+    "OhMyDataError",
+    "PaginationError",
+    "PermanentProviderError",
+    "PermissionDeniedError",
+    "ProviderError",
+    "RateLimitDecision",
+    "RateLimitError",
+    "RateLimitPolicy",
+    "RateLimiter",
+    "RequestSpec",
+    "RetryExhaustedError",
+    "RetryPolicy",
+    "RetryResult",
+    "SchemaMismatchError",
+    "SnapshotConflictError",
+    "SnapshotIntegrityError",
+    "SnapshotMode",
+    "SnapshotRef",
+    "SnapshotReplay",
+    "SnapshotStore",
+    "TransientProviderError",
+    "execute_with_retry",
+]

ohmydata/core/errors.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass
+
+
+class OhMyDataError(Exception):
+    pass
+
+
+class ProviderError(OhMyDataError):
+    pass
+
+
+class PermanentProviderError(ProviderError):
+    pass
+
+
+class AuthenticationError(PermanentProviderError):
+    pass
+
+
+class PermissionDeniedError(PermanentProviderError):
+    pass
+
+
+class EmptyResponseError(PermanentProviderError):
+    pass
+
+
+class SchemaMismatchError(PermanentProviderError):
+    pass
+
+
+class PaginationError(PermanentProviderError):
+    pass
+
+
+class TransientProviderError(ProviderError):
+    pass
+
+
+class RateLimitError(TransientProviderError):
+    pass
+
+
+class SnapshotIntegrityError(OhMyDataError):
+    pass
+
+
+class SnapshotConflictError(SnapshotIntegrityError):
+    pass
+
+
+class CoverageError(OhMyDataError):
+    pass
+
+
+@dataclass(frozen=True)
+class AttemptRecord:
+    attempt: int
+    exception_type: str | None
+    retry_delay_seconds: float | None
+
+
+class RetryExhaustedError(TransientProviderError):
+    def __init__(self, attempts: tuple[AttemptRecord, ...]):
+        self.attempts = attempts
+        super().__init__(f"retry exhausted after {len(attempts)} attempts")

ohmydata/core/policy.py

@@ -0,0 +1,101 @@
+import math
+import random
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+from .errors import AttemptRecord, RetryExhaustedError, TransientProviderError
+
+__all__ = ["AttemptRecord", "RetryPolicy", "RetryResult", "execute_with_retry"]
+
+T = TypeVar("T")
+
+
+def _finite_number(value: object) -> bool:
+    return (
+        not isinstance(value, bool)
+        and isinstance(value, (int, float))
+        and math.isfinite(float(value))
+    )
+
+
+def _valid_attempts(value: object) -> bool:
+    return type(value) is int and value >= 1
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    max_attempts: int = 3
+    base_delay_seconds: float = 1.0
+    backoff_multiplier: float = 2.0
+    max_delay_seconds: float = 60.0
+    jitter_ratio: float = 0.0
+
+    def __post_init__(self):
+        if (
+            not _valid_attempts(self.max_attempts)
+            or any(
+                not _finite_number(x)
+                for x in (
+                    self.base_delay_seconds,
+                    self.backoff_multiplier,
+                    self.max_delay_seconds,
+                    self.jitter_ratio,
+                )
+            )
+            or self.max_attempts < 1
+            or self.base_delay_seconds < 0
+            or self.backoff_multiplier < 1
+            or self.max_delay_seconds < 0
+            or not 0 <= self.jitter_ratio <= 1
+        ):
+            raise ValueError("invalid retry policy")
+
+
+@dataclass(frozen=True)
+class RetryResult(Generic[T]):
+    value: T
+    attempts: tuple[AttemptRecord, ...]
+
+
+def execute_with_retry(
+    fn: Callable[[], T],
+    policy: RetryPolicy | None = None,
+    *,
+    sleep: Callable[[float], None] = time.sleep,
+    random_value: Callable[[], float] = random.random,
+    classifier: Callable[[Exception], bool] = lambda e: isinstance(e, TransientProviderError),
+) -> RetryResult[T]:
+    policy = policy or RetryPolicy()
+    records: list[AttemptRecord] = []
+    for i in range(policy.max_attempts):
+        try:
+            value = fn()
+            records.append(AttemptRecord(i + 1, None, None))
+            return RetryResult(value, tuple(records))
+        except Exception as exc:
+            if not isinstance(exc, TransientProviderError) or not classifier(exc):
+                raise
+            delay = None
+            if i + 1 < policy.max_attempts:
+                r = random_value()
+                if not 0 <= r <= 1:
+                    raise ValueError("random value out of range")
+                bounded = min(
+                    policy.max_delay_seconds,
+                    policy.base_delay_seconds * policy.backoff_multiplier**i,
+                )
+                delay = bounded * (1 + policy.jitter_ratio * (2 * r - 1))
+                sleep(delay)
+            records.append(AttemptRecord(i + 1, type(exc).__name__, delay))
+            if i + 1 == policy.max_attempts:
+                err = RetryExhaustedError(
+                    tuple(
+                        AttemptRecord(x.attempt, x.exception_type, x.retry_delay_seconds)
+                        for x in records
+                    )
+                )
+                err.__cause__ = exc
+                raise err from exc
+    raise AssertionError

ohmydata/core/provenance.py

@@ -0,0 +1,93 @@
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import Enum
+from types import MappingProxyType
+from typing import Any, cast
+
+from .policy import AttemptRecord
+from .specs import RequestSpec
+
+
+class EmptyDisposition(str, Enum):
+    NOT_EMPTY = "NOT_EMPTY"
+    ALLOWED_EMPTY = "ALLOWED_EMPTY"
+
+
+@dataclass(frozen=True)
+class FetchProvenance:
+    provider: str
+    endpoint: str
+    request_identity: str
+    effective_parameters: dict[str, Any]
+    requested_fields: tuple[str, ...]
+    retrieved_at: datetime
+    attempts: tuple[AttemptRecord, ...]
+    row_count: int
+    columns: tuple[str, ...]
+    warnings: tuple[str, ...]
+    snapshot_identities: tuple[str, ...]
+    empty_disposition: EmptyDisposition
+
+    @classmethod
+    def from_request(cls, spec: RequestSpec, **kwargs: Any) -> "FetchProvenance":
+        return cls(
+            spec.provider,
+            spec.endpoint,
+            spec.request_identity,
+            spec.effective_parameters,
+            tuple(spec.fields),
+            **kwargs,
+        )
+
+    def __post_init__(self):
+        if (
+            self.retrieved_at.tzinfo is None
+            or self.row_count < 0
+            or len(set(self.columns)) != len(self.columns)
+        ):
+            raise ValueError("invalid provenance")
+        if (self.row_count == 0) != (self.empty_disposition == EmptyDisposition.ALLOWED_EMPTY):
+            raise ValueError("empty disposition mismatch")
+        object.__setattr__(self, "retrieved_at", self.retrieved_at.astimezone(UTC))
+        for name in ("attempts", "requested_fields", "columns", "warnings", "snapshot_identities"):
+            object.__setattr__(self, name, tuple(getattr(self, name)))
+        object.__setattr__(self, "effective_parameters", self._freeze(self.effective_parameters))
+
+    @staticmethod
+    def _freeze(value: Any) -> Any:
+        if isinstance(value, dict):
+            source = cast(dict[str, Any], value)
+            return MappingProxyType({k: FetchProvenance._freeze(v) for k, v in source.items()})
+        if isinstance(value, list):
+            return tuple(FetchProvenance._freeze(v) for v in cast(list[Any], value))
+        return value
+
+    @property
+    def attempt_count(self) -> int:
+        return len(self.attempts)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "provider": self.provider,
+            "endpoint": self.endpoint,
+            "request_identity": self.request_identity,
+            "effective_parameters": self._thaw(self.effective_parameters),
+            "requested_fields": list(self.requested_fields),
+            "retrieved_at": self.retrieved_at.astimezone(UTC).isoformat(),
+            "attempt_count": self.attempt_count,
+            "attempts": [a.__dict__ for a in self.attempts],
+            "row_count": self.row_count,
+            "columns": list(self.columns),
+            "warnings": list(self.warnings),
+            "snapshot_identities": list(self.snapshot_identities),
+            "empty_disposition": self.empty_disposition.value,
+        }
+
+    @staticmethod
+    def _thaw(value: Any) -> Any:
+        if isinstance(value, MappingProxyType):
+            source = cast(dict[str, Any], value)
+            return {k: FetchProvenance._thaw(v) for k, v in source.items()}
+        if isinstance(value, tuple):
+            return [FetchProvenance._thaw(v) for v in cast(tuple[Any, ...], value)]
+        return value

ohmydata/core/rate_limit.py

@@ -0,0 +1,53 @@
+import math
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RateLimitPolicy:
+    min_interval_seconds: float
+
+    def __post_init__(self):
+        if (
+            isinstance(self.min_interval_seconds, bool)
+            or not math.isfinite(float(self.min_interval_seconds))
+            or self.min_interval_seconds < 0
+        ):
+            raise ValueError("negative interval")
+
+
+@dataclass(frozen=True)
+class RateLimitDecision:
+    waited_seconds: float
+    acquired_at: float
+
+
+class RateLimiter:
+    def __init__(
+        self,
+        policy: RateLimitPolicy,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], None] = time.sleep,
+    ):
+        self.policy, self.clock, self.sleep, self._last = policy, clock, sleep, None
+        self._lock = threading.Lock()
+
+    def acquire(self) -> RateLimitDecision:
+        with self._lock:
+            now = self.clock()
+            if not math.isfinite(now):
+                raise ValueError("clock must return finite value")
+            wait = (
+                0
+                if self._last is None
+                else max(0, self.policy.min_interval_seconds - (now - self._last))
+            )
+            if wait:
+                self.sleep(wait)
+                now = self.clock()
+                if not math.isfinite(now):
+                    raise ValueError("clock must return finite value")
+            self._last = now
+            return RateLimitDecision(wait, now)

ohmydata/core/snapshot.py

@@ -0,0 +1,292 @@
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+from copy import deepcopy
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import Enum
+from pathlib import Path
+from typing import Any, cast
+
+from .errors import SnapshotConflictError, SnapshotIntegrityError
+from .specs import RequestSpec
+
+_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
+_HEX64 = re.compile(r"^[0-9a-f]{64}$")
+_MANIFEST_KEYS = frozenset(
+    {
+        "manifest_schema_version",
+        "provider",
+        "endpoint",
+        "canonical_request",
+        "request_identity",
+        "response_sha256",
+        "response_byte_size",
+        "serialization_identifier",
+        "retrieved_at",
+        "snapshot_identity",
+        "mode",
+    }
+)
+
+
+class SnapshotMode(str, Enum):
+    APPEND = "append"
+    FROZEN = "frozen"
+
+
+@dataclass(frozen=True)
+class SnapshotRef:
+    path: Path
+    snapshot_identity: str
+    mode: SnapshotMode
+    provider: str
+    endpoint: str
+    request_identity: str
+    response_sha256: str
+    serialization_identifier: str
+
+
+@dataclass(frozen=True)
+class SnapshotReplay:
+    payload: bytes
+    manifest: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class _ValidatedSnapshot:
+    ref: SnapshotRef
+    replay: SnapshotReplay
+
+
+def _safe(value: Any) -> bool:
+    return (
+        isinstance(value, str)
+        and value not in {".", ".."}
+        and _IDENTIFIER.fullmatch(value) is not None
+    )
+
+
+def _identity(
+    request_identity: str, response_sha256: str, serialization: str, mode: SnapshotMode
+) -> str:
+    return hashlib.sha256(
+        (request_identity + response_sha256 + serialization + mode.value).encode()
+    ).hexdigest()
+
+
+class SnapshotStore:
+    def __init__(self, root: Path):
+        self.root = Path(os.path.abspath(root))
+
+    def _final_path(
+        self,
+        provider: str,
+        endpoint: str,
+        request_identity: str,
+        mode: SnapshotMode,
+        serialization: str,
+        response_sha256: str,
+    ) -> Path:
+        base = self.root / provider / endpoint / request_identity / mode.value
+        if mode is SnapshotMode.APPEND:
+            return (
+                base / hashlib.sha256(serialization.encode("utf-8")).hexdigest() / response_sha256
+            )
+        return base
+
+    def _read_validated(self, path: Path) -> _ValidatedSnapshot:
+        path = Path(os.path.abspath(path))
+        try:
+            manifest = json.loads(
+                (path / "manifest.json").read_text(encoding="utf-8"),
+                parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)),
+            )
+            payload = (path / "response.bin").read_bytes()
+        except Exception as exc:
+            raise SnapshotIntegrityError("invalid snapshot files") from exc
+        if not isinstance(manifest, dict):
+            raise SnapshotIntegrityError("manifest fields mismatch")
+        manifest = cast(dict[str, Any], manifest)
+        if frozenset(manifest) != _MANIFEST_KEYS:
+            raise SnapshotIntegrityError("manifest fields mismatch")
+        if (
+            type(manifest["manifest_schema_version"]) is not int
+            or manifest["manifest_schema_version"] != 1
+        ):
+            raise SnapshotIntegrityError("manifest schema mismatch")
+        provider, endpoint, serialization = (
+            manifest["provider"],
+            manifest["endpoint"],
+            manifest["serialization_identifier"],
+        )
+        if not _safe(provider) or not _safe(endpoint) or not _safe(serialization):
+            raise SnapshotIntegrityError("unsafe identifier")
+        mode_raw = manifest["mode"]
+        if not isinstance(mode_raw, str) or mode_raw not in {m.value for m in SnapshotMode}:
+            raise SnapshotIntegrityError("mode mismatch")
+        mode = SnapshotMode(mode_raw)
+        request_identity = manifest["request_identity"]
+        response_sha = manifest["response_sha256"]
+        snapshot_identity = manifest["snapshot_identity"]
+        if not all(
+            isinstance(x, str) and _HEX64.fullmatch(x)
+            for x in (request_identity, response_sha, snapshot_identity)
+        ):
+            raise SnapshotIntegrityError("identity mismatch")
+        canonical = manifest["canonical_request"]
+        if not isinstance(canonical, dict):
+            raise SnapshotIntegrityError("canonical request mismatch")
+        canonical = cast(dict[str, Any], canonical)
+        if set(canonical) != {
+            "provider",
+            "endpoint",
+            "parameters",
+            "fields",
+        }:
+            raise SnapshotIntegrityError("canonical request mismatch")
+        if canonical["provider"] != provider or canonical["endpoint"] != endpoint:
+            raise SnapshotIntegrityError("canonical request mismatch")
+        if not isinstance(canonical["parameters"], dict) or not isinstance(
+            canonical["fields"], list
+        ):
+            raise SnapshotIntegrityError("canonical request mismatch")
+        try:
+            canonical_bytes = json.dumps(
+                canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+            ).encode("utf-8")
+        except Exception as exc:
+            raise SnapshotIntegrityError("canonical request mismatch") from exc
+        if hashlib.sha256(canonical_bytes).hexdigest() != request_identity:
+            raise SnapshotIntegrityError("canonical request mismatch")
+        if type(manifest["response_byte_size"]) is not int or manifest["response_byte_size"] != len(
+            payload
+        ):
+            raise SnapshotIntegrityError("response size mismatch")
+        if hashlib.sha256(payload).hexdigest() != response_sha:
+            raise SnapshotIntegrityError("response hash mismatch")
+        if not isinstance(manifest["retrieved_at"], str):
+            raise SnapshotIntegrityError("timestamp mismatch")
+        try:
+            timestamp = datetime.fromisoformat(manifest["retrieved_at"])
+            if timestamp.tzinfo is None or timestamp.utcoffset() != UTC.utcoffset(None):
+                raise ValueError
+        except Exception as exc:
+            raise SnapshotIntegrityError("timestamp mismatch") from exc
+        if _identity(request_identity, response_sha, serialization, mode) != snapshot_identity:
+            raise SnapshotIntegrityError("snapshot identity mismatch")
+        expected_path = self._final_path(
+            provider, endpoint, request_identity, mode, serialization, response_sha
+        )
+        if path != expected_path:
+            raise SnapshotIntegrityError("snapshot path mismatch")
+        ref = SnapshotRef(
+            path,
+            snapshot_identity,
+            mode,
+            provider,
+            endpoint,
+            request_identity,
+            response_sha,
+            serialization,
+        )
+        return _ValidatedSnapshot(ref, SnapshotReplay(payload, dict(manifest)))
+
+    def _resolve_existing(self, final: Path, desired: SnapshotRef) -> SnapshotRef:
+        winner = self._read_validated(final).ref
+        same_request = (winner.provider, winner.endpoint, winner.request_identity, winner.mode) == (
+            desired.provider,
+            desired.endpoint,
+            desired.request_identity,
+            desired.mode,
+        )
+        same_content = (
+            winner.response_sha256 == desired.response_sha256
+            and winner.serialization_identifier == desired.serialization_identifier
+        )
+        if same_request and same_content:
+            return winner
+        if desired.mode is SnapshotMode.FROZEN:
+            raise SnapshotConflictError("frozen snapshot conflict")
+        raise SnapshotIntegrityError("append snapshot collision")
+
+    def write(
+        self,
+        spec: RequestSpec,
+        payload: bytes,
+        retrieved_at: datetime,
+        serialization: str,
+        mode: SnapshotMode = SnapshotMode.APPEND,
+    ) -> SnapshotRef:
+        if (
+            type(payload) is not bytes
+            or type(retrieved_at) is not datetime
+            or retrieved_at.tzinfo is None
+            or retrieved_at.utcoffset() is None
+        ):
+            raise TypeError("invalid snapshot input")
+        if not _safe(serialization):
+            raise ValueError("invalid snapshot input")
+        if type(mode) is not SnapshotMode:
+            raise TypeError("invalid snapshot mode")
+        response_sha = hashlib.sha256(payload).hexdigest()
+        final = self._final_path(
+            spec.provider, spec.endpoint, spec.request_identity, mode, serialization, response_sha
+        )
+        desired = SnapshotRef(
+            final,
+            _identity(spec.request_identity, response_sha, serialization, mode),
+            mode,
+            spec.provider,
+            spec.endpoint,
+            spec.request_identity,
+            response_sha,
+            serialization,
+        )
+        if final.exists():
+            return self._resolve_existing(final, desired)
+        manifest = {
+            "manifest_schema_version": 1,
+            "provider": spec.provider,
+            "endpoint": spec.endpoint,
+            "canonical_request": spec.canonical_payload,
+            "request_identity": spec.request_identity,
+            "response_sha256": response_sha,
+            "response_byte_size": len(payload),
+            "serialization_identifier": serialization,
+            "retrieved_at": retrieved_at.astimezone(UTC).isoformat().replace("+00:00", "Z"),
+            "snapshot_identity": desired.snapshot_identity,
+            "mode": mode.value,
+        }
+        final.parent.mkdir(parents=True, exist_ok=True)
+        tmp = Path(tempfile.mkdtemp(prefix=".tmp-", dir=final.parent))
+        try:
+            (tmp / "response.bin").write_bytes(payload)
+            (tmp / "manifest.json").write_text(
+                json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")),
+                encoding="utf-8",
+            )
+            try:
+                os.rename(tmp, final)
+            except OSError:
+                if final.exists():
+                    return self._resolve_existing(final, desired)
+                raise
+            return self._read_validated(final).ref
+        finally:
+            if tmp.exists():
+                shutil.rmtree(tmp, ignore_errors=True)
+
+    def replay(self, ref: SnapshotRef, expected: RequestSpec | None = None) -> SnapshotReplay:
+        validated = self._read_validated(ref.path)
+        if validated.ref != ref:
+            raise SnapshotIntegrityError("snapshot reference mismatch")
+        if expected is not None and (
+            validated.ref.request_identity != expected.request_identity
+            or validated.replay.manifest["canonical_request"] != expected.canonical_payload
+        ):
+            raise SnapshotIntegrityError("request mismatch")
+        return SnapshotReplay(validated.replay.payload, deepcopy(validated.replay.manifest))