smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/_operator_deadline.py

@@ -0,0 +1,50 @@
+"""Wall-clock watchdog for the dedicated POSIX operator process, independent of driver progress."""
+
+from __future__ import annotations
+
+import signal
+import threading
+from types import FrameType
+from typing import Any
+
+from .errors import MigrationRefused
+
+
+class DeadlineInterrupt(BaseException):
+    """Interrupt the operator without being swallowed by adapter exception translation."""
+
+
+class OperatorDeadline:
+    """Own SIGALRM only in a dedicated main thread with no pre-existing alarm."""
+
+    def __init__(self) -> None:
+        self.previous: Any = None
+        self.installed = False
+
+    def __enter__(self) -> OperatorDeadline:
+        if threading.current_thread() is not threading.main_thread():
+            raise MigrationRefused("local cutover runs in a dedicated process main thread")
+        if signal.getitimer(signal.ITIMER_REAL) != (0.0, 0.0):
+            raise MigrationRefused("local cutover cannot share an existing process alarm")
+        self.previous = signal.getsignal(signal.SIGALRM)
+        signal.signal(signal.SIGALRM, self._interrupt)
+        self.installed = True
+        return self
+
+    @staticmethod
+    def _interrupt(_signum: int, _frame: FrameType | None) -> None:
+        raise DeadlineInterrupt("local operator wall-clock deadline reached")
+
+    def arm(self, milliseconds: int) -> None:
+        if type(milliseconds) is not int or milliseconds < 1:
+            raise MigrationRefused("operator deadline must be a positive integer in milliseconds")
+        signal.setitimer(signal.ITIMER_REAL, milliseconds / 1000)
+
+    def cancel(self) -> None:
+        if self.installed:
+            signal.setitimer(signal.ITIMER_REAL, 0)
+
+    def __exit__(self, *_args: Any) -> None:
+        self.cancel()
+        signal.signal(signal.SIGALRM, self.previous)
+        self.installed = False

sde/_usage.py

@@ -0,0 +1,314 @@
+"""Local operation and transaction ownership; no database or process-global client state."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+from functools import wraps
+from threading import Lock
+from typing import Concatenate, ParamSpec, Protocol, TypeVar
+
+from .errors import ResourceBusy, ResourceClosed
+
+_session_owner: ContextVar[object | None] = ContextVar("sde_session_owner", default=None)
+_operation: ContextVar[object | None] = ContextVar("sde_operation", default=None)
+_transaction: ContextVar[TransactionScope | None] = ContextVar("sde_transaction", default=None)
+
+
+@dataclass(eq=False)
+class TransactionScope:
+    gate: UsageGate
+    parent: TransactionScope | None
+    session: object | None
+    active: bool = True
+
+
+def check_scope() -> None:
+    scope = _transaction.get()
+    while scope is not None:
+        if not scope.active:
+            raise ResourceClosed("the inherited transaction scope has already ended")
+        scope = scope.parent
+
+
+@contextmanager
+def session_owner(owner: object) -> Iterator[None]:
+    check_scope()
+    logical = _session_scope.get()
+    if logical is not None and logical.active and logical.usage.owner is not owner:
+        raise ResourceBusy("another session owns the current transaction scope")
+    scope = _transaction.get()
+    if scope is not None and scope.session is not owner:
+        raise ResourceBusy("another session owns the current transaction scope")
+    marker = _session_owner.set(owner)
+    try:
+        yield
+    finally:
+        _session_owner.reset(marker)
+
+
+class UsageGate:
+    """Synchronize admission with transaction claims; reject foreign use before native I/O."""
+
+    def __init__(self) -> None:
+        self.lock = Lock()
+        self.pid = os.getpid()
+        self.operation_owner: object | None = None
+        self.operation_depth = 0
+        self.transaction_owner: TransactionScope | None = None
+
+    def _process(self) -> None:
+        if self.pid != os.getpid():
+            raise ResourceClosed(
+                "create a fresh adapter after fork; the inherited connection is not usable"
+            )
+
+    def _check_transaction(self) -> None:
+        check_scope()
+        owned = self.transaction_owner
+        if owned is None:
+            return
+        scope = _transaction.get()
+        while scope is not None and scope.gate is not self:
+            scope = scope.parent
+        if scope is not owned or owned.session is not _session_owner.get():
+            raise ResourceBusy("the connection is owned by another active transaction scope")
+
+    @contextmanager
+    def operation(self) -> Iterator[None]:
+        self._process()
+        owner = _operation.get()
+        marker = None
+        if owner is None:
+            owner = object()
+            marker = _operation.set(owner)
+        admitted = False
+        failed = False
+        try:
+            with self.lock:
+                self._check_transaction()
+                if self.operation_owner is not None and self.operation_owner is not owner:
+                    raise ResourceBusy("the connection already has an operation in progress")
+                self.operation_owner = owner
+                self.operation_depth += 1
+                admitted = True
+            try:
+                yield
+            except BaseException:
+                failed = True
+                raise
+            finally:
+                if admitted:
+                    with self.lock:
+                        self.operation_depth -= 1
+                        if self.operation_depth == 0:
+                            self.operation_owner = None
+                if not failed:
+                    check_scope()
+        finally:
+            if marker is not None:
+                _operation.reset(marker)
+
+    @contextmanager
+    def transaction(self) -> Iterator[TransactionScope]:
+        self._process()
+        with self.lock:
+            self._check_transaction()
+            if self.operation_owner is not None:
+                raise ResourceBusy("a transaction cannot begin while an operation is in progress")
+            previous = self.transaction_owner
+            scope = TransactionScope(self, _transaction.get(), _session_owner.get())
+            self.transaction_owner = scope
+        marker = _transaction.set(scope)
+        try:
+            yield scope
+        finally:
+            with self.lock:
+                scope.active = False
+                self.transaction_owner = previous
+            _transaction.reset(marker)
+
+    def idle(self) -> None:
+        with self.lock:
+            if self.operation_owner is not None:
+                if self.transaction_owner is not None:
+                    self.transaction_owner.active = False
+                raise ResourceBusy("finish all operations before leaving the transaction")
+
+
+class Guarded(Protocol):
+    @property
+    def _usage(self) -> UsageGate: ...
+
+
+G = TypeVar("G", bound=Guarded)
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def guarded(method: Callable[Concatenate[G, P], R]) -> Callable[Concatenate[G, P], R]:
+    @wraps(method)
+    def call(self: G, /, *args: P.args, **kwargs: P.kwargs) -> R:
+        with self._usage.operation():
+            return method(self, *args, **kwargs)
+
+    return call
+
+
+@dataclass(eq=False)
+class SessionScope:
+    usage: SessionUsage
+    group: str
+    active: bool = True
+
+
+_session_scope: ContextVar[SessionScope | None] = ContextVar("sde_session_scope", default=None)
+
+
+class SessionUsage:
+    def __init__(self, owner: object) -> None:
+        self.owner = owner
+        self.pid = os.getpid()
+        self.lock = Lock()
+        self.closed = False
+        self.busy = False
+        self.transaction_scope: SessionScope | None = None
+
+    def check(self) -> None:
+        if self.pid != os.getpid():
+            raise ResourceClosed("create a fresh session and adapters after fork")
+        check_scope()
+        context = _session_scope.get()
+        if self.closed or (context is not None and not context.active):
+            raise ResourceClosed("the session or inherited transaction scope has ended")
+        if self.transaction_scope is not None and context is not self.transaction_scope:
+            raise ResourceBusy("the session belongs to another active transaction context")
+
+    def group(self, name: str) -> None:
+        from .errors import ModelPlanningError
+
+        self.check()
+        if self.transaction_scope is not None and self.transaction_scope.group != name:
+            raise ModelPlanningError(
+                "the operation is outside the active transaction's colocation group"
+            )
+
+    @contextmanager
+    def operation(self) -> Iterator[None]:
+        self.check()  # Refuse inherited locks before acquiring one in a child.
+        with self.lock:
+            self.check()
+            if self.busy:
+                raise ResourceBusy("the session already has an operation in progress")
+            self.busy = True
+        failed = False
+        try:
+            with session_owner(self.owner):
+                try:
+                    yield
+                except BaseException:
+                    failed = True
+                    raise
+                finally:
+                    if not failed:
+                        self.check()
+        finally:
+            with self.lock:
+                self.busy = False
+
+    @contextmanager
+    def transaction(self, group: str) -> Iterator[None]:
+        self.check()
+        with self.lock:
+            self.group(group)
+            if self.busy:
+                raise ResourceBusy("finish the session operation before beginning a transaction")
+            previous = self.transaction_scope
+            scope = SessionScope(self, group)
+            self.transaction_scope = scope
+        marker = _session_scope.set(scope)
+        try:
+            with session_owner(self.owner):
+                yield
+        finally:
+            with self.lock:
+                scope.active = False
+                self.transaction_scope = previous
+            _session_scope.reset(marker)
+
+    def idle(self) -> None:
+        with self.lock:
+            if self.busy:
+                if self.transaction_scope is not None:
+                    self.transaction_scope.active = False
+                raise ResourceBusy("finish all session operations before leaving the transaction")
+
+    def seal(self) -> None:
+        with self.lock:
+            if self.transaction_scope is not None:
+                self.transaction_scope.active = False
+
+    def close(self) -> None:
+        if self.pid != os.getpid():
+            raise ResourceClosed(
+                "create a fresh session after fork; do not close inherited adapters"
+            )
+        with self.lock:
+            if self.busy or self.transaction_scope is not None:
+                raise ResourceBusy(
+                    "cannot close a session while its operation or transaction is active"
+                )
+            self.closed = True
+
+
+class SessionGuarded(Protocol):
+    @property
+    def _session_usage(self) -> SessionUsage: ...
+
+
+S = TypeVar("S", bound=SessionGuarded)
+
+
+def session_call(method: Callable[Concatenate[S, P], R]) -> Callable[Concatenate[S, P], R]:
+    @wraps(method)
+    def call(self: S, /, *args: P.args, **kwargs: P.kwargs) -> R:
+        with self._session_usage.operation():
+            return method(self, *args, **kwargs)
+
+    return call
+
+
+def _after_fork() -> None:
+    _session_owner.set(None)
+    _operation.set(None)
+    _transaction.set(None)
+    _session_scope.set(None)
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_after_fork)
+
+
+Reader = TypeVar("Reader")
+
+
+def session_function(
+    method: Callable[Concatenate[Reader, P], R],
+) -> Callable[Concatenate[Reader, P], R]:
+    """Keep free migration helpers inside a real Session lease; inspection contexts are separate."""
+
+    @wraps(method)
+    def call(reader: Reader, /, *args: P.args, **kwargs: P.kwargs) -> R:
+        usage = getattr(reader, "_session_usage", None)
+        if isinstance(usage, SessionUsage):
+            usage.check()
+            if usage.transaction_scope is not None:
+                raise ResourceBusy("run migration helpers outside application transactions")
+            with usage.operation():
+                return method(reader, *args, **kwargs)
+        return method(reader, *args, **kwargs)
+
+    return call

sde/bulk.py

@@ -0,0 +1,79 @@
+"""Bounded application batches, independent of migration's idempotent copy protocol."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any, Final, Protocol, cast
+from uuid import UUID
+
+from .errors import BulkWriteRefused
+
+MAX_BATCH_ROWS: Final = 1000
+MAX_BATCH_VALUES: Final = 60_000
+
+
+class BulkWritable(Protocol):
+    """One native application insert, with no conflict suppression, splitting or retry."""
+
+    def insert_many(self, table: str, rows: Sequence[Mapping[str, Any]]) -> None: ...
+
+
+def bulk_writer(engine: object) -> BulkWritable:
+    if not callable(getattr(engine, "insert_many", None)):
+        raise BulkWriteRefused("this adapter does not support bulk writes (insert_many)")
+    return cast(BulkWritable, engine)
+
+
+def batch_columns(rows: object, *, extra_columns: int = 0) -> list[str]:
+    """Check the entire batch before an adapter can start work. Values are never in errors."""
+    if not isinstance(rows, Sequence) or isinstance(rows, (str, bytes, bytearray)):
+        raise BulkWriteRefused("a batch must be a sequence of row mappings")
+    if len(rows) > MAX_BATCH_ROWS:
+        raise BulkWriteRefused(f"a batch may contain at most {MAX_BATCH_ROWS} rows")
+    if not rows:
+        return []
+    columns: list[str] | None = None
+    for row in rows:
+        if not isinstance(row, Mapping) or not row or any(not isinstance(c, str) for c in row):
+            raise BulkWriteRefused("each batch row must be a nonempty mapping with string fields")
+        here = sorted(row)
+        if columns is None:
+            columns = here
+            if len(rows) * (len(columns) + extra_columns) > MAX_BATCH_VALUES:
+                raise BulkWriteRefused(
+                    f"a batch may contain at most {MAX_BATCH_VALUES} values, including generation"
+                )
+        elif here != columns:
+            raise BulkWriteRefused("all batch rows must have the same fields")
+    assert columns is not None
+    return columns
+
+
+def _snapshot(value: Any, active: set[int]) -> Any:
+    if value is None or isinstance(
+        value, (str, bool, int, float, Decimal, UUID, date, datetime, bytes)
+    ):
+        return value
+    if isinstance(value, (bytearray, memoryview)):
+        return bytes(value)
+    identity = id(value)
+    if identity in active:
+        raise BulkWriteRefused("batch values must not contain cycles")
+    active.add(identity)
+    try:
+        if isinstance(value, Mapping) and all(isinstance(key, str) for key in value):
+            return {key: _snapshot(item, active) for key, item in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [_snapshot(item, active) for item in value]
+        raise BulkWriteRefused("batch values must use the SDK scalar types or JSON containers")
+    finally:
+        active.remove(identity)
+
+
+def snapshot_rows(rows: Sequence[Mapping[str, Any]]) -> tuple[dict[str, Any], ...]:
+    try:
+        return tuple({key: _snapshot(value, set()) for key, value in row.items()} for row in rows)
+    except RecursionError:
+        raise BulkWriteRefused("batch values are nested too deeply") from None

sde/canonical.py

@@ -0,0 +1,141 @@
+"""Canonical encoding: the one place where the cross-language contract lives.
+
+Every SDE library has to produce byte-identical output from this. A definition amounting to
+"whatever the Python implementation does" is not a definition, so the rules are spelled out here and
+in ``docs/format-contract.md``, and the conformance vectors pin them.
+
+The rules, in the order they are applied:
+
+1. UTF-8, no byte order mark.
+2. Object keys are NFC-normalised, then sorted by Unicode code point. Normalise first: sorting first
+   would order ``é`` (U+00E9) and ``e`` + U+0301 differently while both normalise to the same key.
+3. No insignificant whitespace. ``{"a":1,"b":[2,3]}`` and nothing else.
+4. Every string is NFC-normalised.
+5. Escaping is minimal and exhaustive: only ``"``, ``\\`` and C0 control characters are escaped,
+   controls using the short forms where JSON defines them and ``\\u00XX`` otherwise. Everything else
+   is emitted as raw UTF-8. This matters because JSON writers disagree: some escape all non-ASCII,
+   some escape U+2028, some escape forward slashes. Any of those would change the hash.
+6. Floating point values are rejected outright. Their textual form differs across languages and the
+   difference is unfixable after the fact. Note the distinction that cost a spec revision: a *field*
+   may have type ``float64``; the IR records the *name* of that type, which is a string. There is no
+   float literal anywhere in the encoding.
+7. Integers are emitted as their shortest decimal form, no leading ``+``, no exponent.
+
+Ordering of arrays is not this module's business. Where order carries no meaning the caller sorts;
+where it does, the caller records an explicit index in each element rather than relying on position.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import unicodedata
+from typing import Final
+
+__all__ = ["CanonicalError", "canonical_bytes", "canonical_str", "digest16"]
+
+# Short escape forms JSON defines. Everything else below 0x20 gets \u00XX.
+_SHORT_ESCAPES: Final[dict[int, str]] = {
+    0x08: "\\b",
+    0x09: "\\t",
+    0x0A: "\\n",
+    0x0C: "\\f",
+    0x0D: "\\r",
+    0x22: '\\"',
+    0x5C: "\\\\",
+}
+
+
+class CanonicalError(ValueError):
+    """A value cannot be encoded canonically.
+
+    Raised rather than coerced on purpose: silently accepting a float, a NaN or a non-string key
+    would produce bytes that another implementation cannot reproduce, and the failure would surface
+    much later as two versions of the same model.
+    """
+
+
+def _escape(text: str) -> str:
+    normalised = unicodedata.normalize("NFC", text)
+    out: list[str] = ['"']
+    for char in normalised:
+        code = ord(char)
+        short = _SHORT_ESCAPES.get(code)
+        if short is not None:
+            out.append(short)
+        elif code < 0x20:
+            out.append(f"\\u{code:04x}")
+        else:
+            out.append(char)
+    out.append('"')
+    return "".join(out)
+
+
+def _encode(value: object, path: str) -> str:
+    # bool before int: bool is a subclass of int in Python and would otherwise encode as 1/0.
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    if value is None:
+        return "null"
+    if isinstance(value, str):
+        return _escape(value)
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        raise CanonicalError(
+            f"float at {path}: floating point is not representable in canonical form, "
+            "because its textual form differs between languages. Use an integer, or a decimal "
+            "string, or the name of a float type if you meant to describe a type."
+        )
+    if isinstance(value, (list, tuple)):
+        items = [_encode(item, f"{path}[{i}]") for i, item in enumerate(value)]
+        return "[" + ",".join(items) + "]"
+    if isinstance(value, dict):
+        pairs: list[tuple[str, object]] = []
+        for key, item in value.items():
+            if not isinstance(key, str):
+                raise CanonicalError(
+                    f"non-string key {key!r} at {path}: object keys must be strings, because "
+                    "key ordering is defined over Unicode code points"
+                )
+            pairs.append((unicodedata.normalize("NFC", key), item))
+        # Sort after normalising, on the normalised key.
+        pairs.sort(key=lambda pair: pair[0])
+        seen: set[str] = set()
+        parts: list[str] = []
+        for key, item in pairs:
+            if key in seen:
+                raise CanonicalError(
+                    f"duplicate key {key!r} at {path} after NFC normalisation: two keys that "
+                    "differ "
+                    "only in Unicode composition are the same key here"
+                )
+            seen.add(key)
+            parts.append(_escape(key) + ":" + _encode(item, f"{path}.{key}"))
+        return "{" + ",".join(parts) + "}"
+    raise CanonicalError(
+        f"{type(value).__name__} at {path} has no canonical form. The canonical encoding accepts "
+        "only null, bool, int, str, list and dict; anything richer has to be reduced to those by "
+        "the caller, so that the reduction is visible and testable."
+    )
+
+
+def canonical_str(value: object) -> str:
+    """Canonical form as text. Prefer :func:`canonical_bytes` for hashing."""
+    return _encode(value, "$")
+
+
+def canonical_bytes(value: object) -> bytes:
+    """Canonical form as UTF-8 bytes. This is what gets hashed and what vectors compare."""
+    return canonical_str(value).encode("utf-8")
+
+
+def digest16(value: object) -> str:
+    """The identifier form used for ``model_version`` and ``shape.id``.
+
+    Lowercase hex, first 8 bytes of SHA-256 over the canonical bytes. Sixteen characters is short
+    enough to appear in logs and error messages without wrapping, and 64 bits of collision
+    resistance is ample for the number of models and shapes one application declares.
+    """
+    return hashlib.sha256(canonical_bytes(value)).hexdigest()[:16]

sde/capabilities.py

@@ -0,0 +1,62 @@
+"""Asking an engine adapter whether it will answer a call, which is not the same as its type.
+
+Two optional protocols decide whether an engine takes part in something: :class:`
+sde.watermark.WatermarkStore` for the forward-only map check, and :class:`sde.migration.Migratable`
+for a migration. Both were originally asked with ``isinstance(engine, Protocol)``, which is the
+obvious spelling and the wrong question.
+
+A ``runtime_checkable`` protocol resolves its members with ``hasattr`` up to Python 3.11 and with
+:func:`inspect.getattr_static` from 3.12 onwards - and the second deliberately ignores
+``__getattr__``. So an object that forwards to a wrapped adapter answers ``hasattr`` for every
+member of the protocol, passes ``isinstance`` on 3.11, and fails it on 3.12. This library supports
+and tests all three, which makes that **the same client, the same wrapper, and a different answer
+per interpreter**.
+
+Wrapping an engine adapter is an ordinary thing for a client to do - metrics, logging, a retry, a
+connection pool - and the consequence was two wrong diagnoses shipped as helpful messages: "this
+engine has nowhere to keep the bookkeeping, so you have no rollback protection", and "this engine
+cannot take part in a migration". Both named the client's *engine* for a property of their own
+wrapper, and the second refuses a migration outright.
+
+Found from a test, not from review: a two-line proxy over the real ClickHouse adapter, written to
+make one write fail, was refused as an engine with no row-level operations. The version dependence
+came from CI on 3.11, which is the part no single machine would have shown.
+
+So the question asked here is the one that matters - **will this object respond to these calls** -
+and it is asked with ordinary attribute access, which honours every way Python has of providing an
+attribute.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = ["members_of", "satisfies"]
+
+
+def members_of(protocol: type) -> tuple[str, ...]:
+    """The public members a protocol names: its methods and its annotated data attributes.
+
+    Both halves are needed and neither is enough. Methods have class attributes, so ``dir`` finds
+    them; an annotated attribute with no value - ``dialect: str`` - exists only in
+    ``__annotations__``. Assembled from two public sources rather than from
+    ``__protocol_attrs__``, which is an implementation detail that did not exist in every version
+    this library supports.
+    """
+    named = {name for name in dir(protocol) if not name.startswith("_")}
+    named.update(
+        name
+        for name in getattr(protocol, "__annotations__", {})
+        if not name.startswith("_")
+    )
+    return tuple(sorted(named))
+
+
+def satisfies(obj: Any, protocol: type) -> bool:
+    """Whether every member the protocol names can be reached on this object.
+
+    Presence rather than callability, deliberately. A member that exists and is not callable fails
+    at the call with a message naming it, which is a better failure than a capability check that
+    quietly answers "no" and sends the reader to look at their engine.
+    """
+    return all(hasattr(obj, name) for name in members_of(protocol))