smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/__init__.py

@@ -0,0 +1,318 @@
+"""Smart Data Engine - client library.
+
+You declare entities and relations. We decide which database engine each colocation group lives in,
+what its physical layout is there, and when it should move - and your code never names a table or an
+engine, which is exactly what lets us change both without touching it.
+
+    from datetime import datetime from decimal import Decimal from typing import Annotated from uuid
+    import UUID import sde
+
+    @sde.entity
+    class User:
+        id: UUID email: str
+
+        class Meta:
+            pii = ["email"]
+
+    @sde.entity
+    class Order:
+        id: UUID user: sde.Ref[User] total: Annotated[Decimal, sde.precision(12, 2)] created_at:
+        datetime
+
+        class Meta:
+            atomic_with = ["Payment"] residency = "EU"
+
+Everything about storage is our decision. The four things you declare - atomicity, residency,
+personal data and a cost ceiling - are the ones that cannot be read from traffic no matter how long
+we watch it.
+
+This library is Apache-2.0 and it works without an account: hand it a placement map you wrote
+yourself and it will route, create schema and run, with no key and no network. That is a supported
+mode, not a loophole.
+"""
+
+from __future__ import annotations
+
+from .bulk import MAX_BATCH_ROWS, MAX_BATCH_VALUES, BulkWritable
+from .canonical import CanonicalError, canonical_bytes, canonical_str, digest16
+from .capabilities import members_of, satisfies
+from .cutover import CUTOVER_PROTOCOL, CUTOVER_RELAYOUT_PROTOCOL, CutoverPlan, load_cutover_plan
+from .entity import Ref, clear_registry, entity, registry
+from .errors import (
+    BulkWriteRefused,
+    DeclarationError,
+    EngineError,
+    MapError,
+    MapRolledBack,
+    MigrationRefused,
+    ModelPlanningError,
+    ResourceBusy,
+    ResourceClosed,
+    SdeError,
+)
+from .explain import Cost, Explains, PlanFinding, QueryPlan, QueryPlanRefused, explain
+from .frozen_verification import FrozenVerifyReport, verify_frozen
+from .groups import Group, colocation_groups, group_of
+from .hashing import NameMap, hash_identifiers, load_or_create_salt
+from .index_build import (
+    INDEX_CHANGE_PROTOCOL,
+    INDEX_PROTOCOL,
+    IndexPlan,
+    IndexReceipt,
+    index_build_name,
+    load_index_plan,
+)
+from .infer import InferredModel, Note, infer_model, infer_models
+from .inspection import InspectionContext
+from .internal import internal_failures, reset_internal_failures
+from .layout import (
+    DIALECTS,
+    FIXED_SCHEMA,
+    ORDERBOOK_KEY,
+    ORDERBOOK_SHAPE,
+    ORDERBOOK_TABLE,
+    DerivedLayout,
+    can_store,
+    default_layout,
+    denormalized_layout,
+    fixed_schema_mismatch,
+    group_columns,
+    snake_case,
+    stored_types,
+)
+from .local_cutover import CutoverReceipt, CutoverRecoveryRequired, LocalCutover, load_local_map
+from .migration import (
+    BACKFILL_TABLE,
+    CHUNK_ROWS,
+    DIALECT_PRECISION,
+    PRECISION_INDEPENDENT,
+    BackfillProgress,
+    Difference,
+    EntityProgress,
+    Migratable,
+    VerifyReport,
+    backfill,
+    precision_refusal,
+    verify,
+)
+from .model import CONTRACT, LogicalModel, build_model, neutral_declaration
+from .physical import PHYSICAL_DESIGN_SINCE, PhysicalFinding
+from .placement import (
+    ALSO_WRITE_SINCE,
+    MAP_CONTRACT,
+    MAP_CONTRACT_FLOOR,
+    RESERVED_TABLES,
+    GroupPlacement,
+    Materialization,
+    PhysicalLayout,
+    PlacementMap,
+    load_map,
+)
+from .provisioning import prepare_schema
+from .query import (
+    MAX_PAGE_ROWS,
+    NumericSummary,
+    Queryable,
+    QueryRefused,
+    Range,
+    ScanPage,
+    Summarizable,
+)
+from .routing import Router, resolve
+from .schema import CompatibilityViews, compatibility_views, schema_is_fixed, schema_statements
+from .session import Engine, ManagedEngine, Session
+from .shapes import SHAPE_KINDS, WRITE_KINDS, OperationShape, enumerate_shapes
+from .staging import (
+    STAGING_PROTOCOL,
+    STAGING_RELAYOUT_PROTOCOL,
+    StagingPlan,
+    StagingReceipt,
+    load_staging_plan,
+    staging_table_name,
+)
+from .telemetry import (
+    MEASURED_FIELDS,
+    CopyFreshness,
+    FanOutStats,
+    GroupFeatures,
+    Histogram,
+    Recorder,
+    ShapeStats,
+    StorageMeasurement,
+    StorageSample,
+    StorageSize,
+    Window,
+    has_time_dimension,
+    time_fields,
+)
+from .types import Float32, Int32, Json, Timestamp, precision
+from .watermark import (
+    WATERMARK_TABLE,
+    Protection,
+    WatermarkCheck,
+    WatermarkStore,
+    enforce_forward_only,
+)
+from .write_fence import EPOCH_COLUMN as WRITE_EPOCH_COLUMN
+from .write_fence import FenceState, WriteFence
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ALSO_WRITE_SINCE",
+    "BACKFILL_TABLE",
+    "CHUNK_ROWS",
+    "CONTRACT",
+    "CUTOVER_PROTOCOL",
+    "CUTOVER_RELAYOUT_PROTOCOL",
+    "DIALECTS",
+    "DIALECT_PRECISION",
+    "FIXED_SCHEMA",
+    "INDEX_CHANGE_PROTOCOL",
+    "INDEX_PROTOCOL",
+    "MAP_CONTRACT",
+    "MAP_CONTRACT_FLOOR",
+    "MAX_BATCH_ROWS",
+    "MAX_BATCH_VALUES",
+    "MAX_PAGE_ROWS",
+    "MEASURED_FIELDS",
+    "ORDERBOOK_KEY",
+    "ORDERBOOK_SHAPE",
+    "ORDERBOOK_TABLE",
+    "PHYSICAL_DESIGN_SINCE",
+    "PRECISION_INDEPENDENT",
+    "RESERVED_TABLES",
+    "SHAPE_KINDS",
+    "STAGING_PROTOCOL",
+    "STAGING_RELAYOUT_PROTOCOL",
+    "WATERMARK_TABLE",
+    "WRITE_EPOCH_COLUMN",
+    "WRITE_KINDS",
+    "BackfillProgress",
+    "BulkWritable",
+    "BulkWriteRefused",
+    "CanonicalError",
+    "CompatibilityViews",
+    "CopyFreshness",
+    "Cost",
+    "CutoverPlan",
+    "CutoverReceipt",
+    "CutoverRecoveryRequired",
+    "DeclarationError",
+    "DerivedLayout",
+    "Difference",
+    "Engine",
+    "EngineError",
+    "EntityProgress",
+    "Explains",
+    "FanOutStats",
+    "FenceState",
+    "Float32",
+    "FrozenVerifyReport",
+    "Group",
+    "GroupFeatures",
+    "GroupPlacement",
+    "Histogram",
+    "IndexPlan",
+    "IndexReceipt",
+    "InferredModel",
+    "InspectionContext",
+    "Int32",
+    "Json",
+    "LocalCutover",
+    "LogicalModel",
+    "ManagedEngine",
+    "MapError",
+    "MapRolledBack",
+    "Materialization",
+    "Migratable",
+    "MigrationRefused",
+    "ModelPlanningError",
+    "NameMap",
+    "Note",
+    "NumericSummary",
+    "OperationShape",
+    "PhysicalFinding",
+    "PhysicalLayout",
+    "PlacementMap",
+    "PlanFinding",
+    "Protection",
+    "QueryPlan",
+    "QueryPlanRefused",
+    "QueryRefused",
+    "Queryable",
+    "Range",
+    "Recorder",
+    "Ref",
+    "ResourceBusy",
+    "ResourceClosed",
+    "Router",
+    "ScanPage",
+    "SdeError",
+    "Session",
+    "ShapeStats",
+    "StagingPlan",
+    "StagingReceipt",
+    "StorageMeasurement",
+    "StorageSample",
+    "StorageSize",
+    "Summarizable",
+    "Timestamp",
+    "VerificationRequest",
+    "VerifyReport",
+    "WatermarkCheck",
+    "WatermarkStore",
+    "Window",
+    "WriteFence",
+    "__version__",
+    "backfill",
+    "build_model",
+    "can_store",
+    "canonical_bytes",
+    "canonical_str",
+    "clear_registry",
+    "colocation_groups",
+    "compatibility_views",
+    "default_layout",
+    "denormalized_layout",
+    "digest16",
+    "enforce_forward_only",
+    "entity",
+    "enumerate_shapes",
+    "explain",
+    "fixed_schema_mismatch",
+    "group_columns",
+    "group_of",
+    "has_time_dimension",
+    "hash_identifiers",
+    "index_build_name",
+    "infer_model",
+    "infer_models",
+    "internal_failures",
+    "load_cutover_plan",
+    "load_index_plan",
+    "load_local_map",
+    "load_map",
+    "load_or_create_salt",
+    "load_staging_plan",
+    "members_of",
+    "neutral_declaration",
+    "precision",
+    "precision_refusal",
+    "prepare_schema",
+    "registry",
+    "reset_internal_failures",
+    "resolve",
+    "satisfies",
+    "schema_is_fixed",
+    "schema_statements",
+    "snake_case",
+    "staging_table_name",
+    "stored_types",
+    "time_fields",
+    "verification_request",
+    "verify",
+    "verify_frozen",
+]
+
+from .verification import VerificationRequest, verification_request

sde/_cutover_project.py

@@ -0,0 +1,179 @@
+"""Durable project state for a client-side cutover; no rows or credentials are stored here."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from . import _local_state
+from .errors import MigrationRefused
+
+
+def encode(value: Any) -> bytes:
+    # Native principal/namespace names are exact strings, not canonical map identifiers.
+    # Keep their spelling; the state checksum is over this explicitly versioned raw-JSON form.
+    return json.dumps(
+        value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False
+    ).encode("utf-8")
+
+
+def _abandoned_stages(payload: Any) -> bool:
+    stages = payload.get("stages") if isinstance(payload, dict) else None
+    if not isinstance(stages, dict):
+        return False
+    return any(
+        isinstance(record, dict)
+        and isinstance(record.get("receipt"), dict)
+        and record["receipt"].get("outcome") == "abandoned"
+        for record in stages.values()
+    )
+
+
+def _index_changes(payload: Any) -> bool:
+    """Whether an in-place index record - executing or kept - also removes indexes (protocol 2)."""
+    if not isinstance(payload, dict):
+        return False
+    records: list[Any] = []
+    history = payload.get("indexes")
+    if isinstance(history, dict):
+        records.extend(history.values())
+    execution = payload.get("execution")
+    if isinstance(execution, dict) and execution.get("kind") == "index":
+        records.append(execution)
+    return any(
+        isinstance(record, dict)
+        and isinstance(record.get("plan"), dict)
+        and record["plan"].get("protocol") == 2
+        for record in records
+    )
+
+
+class ProjectState:
+    """One POSIX directory and lock, shared by all local executors for this project."""
+
+    def __init__(self, root: Path, project_id: str, model_version: str) -> None:
+        self.root = root.resolve()
+        self.project_id = project_id
+        self.model_version = model_version
+        self.path = self.root / "project.json"
+        self.map_path = self.root / "active-map.json"
+
+    def lock(self) -> Any:
+        return _local_state.transaction(self.root)
+
+    def read(self) -> dict[str, Any]:
+        try:
+            envelope = json.loads(self.path.read_bytes())
+            if not isinstance(envelope, dict) or set(envelope) != {
+                "storage_contract",
+                "payload",
+                "sha256",
+            }:
+                raise ValueError("invalid state envelope")
+            if type(envelope["storage_contract"]) is not int or envelope[
+                "storage_contract"
+            ] not in (1, 2, 3, 4, 5):
+                raise ValueError("unsupported state storage contract")
+            payload = envelope["payload"]
+            if (
+                not isinstance(payload, dict)
+                or hashlib.sha256(encode(payload)).hexdigest() != envelope["sha256"]
+            ):
+                raise ValueError("state checksum mismatch")
+            if (
+                payload.get("project_id") != self.project_id
+                or payload.get("model_version") != self.model_version
+            ):
+                raise ValueError("state belongs to another local project/model")
+            fields = {
+                "project_id",
+                "model_version",
+                "active_map",
+                "execution",
+                "completed",
+                "retired_names",
+            }
+            if envelope["storage_contract"] >= 2:
+                fields.add("stages")
+                if not isinstance(payload.get("stages"), dict):
+                    raise ValueError("staging history must be an object")
+            if envelope["storage_contract"] >= 3:
+                # In-place index builds: a reader of contracts 1 and 2 refuses this state rather
+                # than ignoring a history it would not know to keep.
+                fields.add("indexes")
+                if not isinstance(payload.get("indexes"), dict):
+                    raise ValueError("index build history must be an object")
+            if envelope["storage_contract"] < 4 and _abandoned_stages(payload):
+                # Contract 4 is what makes an older operator refuse an abandoned staging's record
+                # instead of reading a receipt whose tables may name no identity.
+                raise ValueError("an abandoned staging needs state storage contract 4")
+            if envelope["storage_contract"] < 5 and _index_changes(payload):
+                # Contract 5 is what makes an older operator refuse a build that also removes
+                # indexes, instead of resuming it with no idea that removals follow the decision.
+                raise ValueError("an index change needs state storage contract 5")
+            if set(payload) != fields:
+                raise ValueError("unknown or missing project state fields")
+            return payload
+        except (OSError, ValueError, TypeError) as exc:
+            raise MigrationRefused(
+                "local cutover state is missing or corrupt; "
+                "restore verified state before proceeding"
+            ) from exc
+
+    def confirm(self) -> None:
+        # Call under the project lock. Neither reading matching bytes nor an in-memory receipt
+        # confirms the directory fsync that may have failed after an earlier publication.
+        _local_state.confirm_file(self.map_path)
+        _local_state.confirm_file(self.path)
+
+    def write(self, payload: dict[str, Any]) -> None:
+        body = encode(payload)
+        envelope = {
+            "storage_contract": 5
+            if _index_changes(payload)
+            else 4
+            if _abandoned_stages(payload)
+            else 3
+            if "indexes" in payload
+            else 2
+            if "stages" in payload
+            else 1,
+            "payload": payload,
+            "sha256": hashlib.sha256(body).hexdigest(),
+        }
+        _local_state.write_bytes(self.path, encode(envelope))
+
+    def enroll(self, document: dict[str, Any], map_payload: bytes) -> None:
+        with self.lock():
+            if self.map_path.exists() and self.map_path.read_bytes() != map_payload:
+                raise MigrationRefused("an existing active map differs from the enrolled state")
+            if self.path.exists():
+                state = self.read()
+                if (
+                    state["active_map"] != document
+                    or state["execution"] is not None
+                    or state["completed"]
+                ):
+                    raise MigrationRefused(
+                        "local project is already enrolled; use its current state"
+                    )
+            else:
+                state = {
+                    "project_id": self.project_id,
+                    "model_version": self.model_version,
+                    "active_map": document,
+                    "execution": None,
+                    "completed": {},
+                    "retired_names": [],
+                }
+                self.write(state)
+            if not self.map_path.exists():
+                _local_state.write_bytes(self.map_path, map_payload, replace=False)
+            # Matching visible files may come from a publication whose directory fsync failed.
+            # An identical retry must confirm their durability before enrollment succeeds.
+            self.confirm()
+
+    def publish(self, payload: bytes) -> None:
+        _local_state.write_bytes(self.map_path, payload)

sde/_local_state.py

@@ -0,0 +1,188 @@
+"""Private implementation of durable local executor files on a POSIX filesystem.
+
+A local executor project has one writer at a time. The lock covers the read/decide/write
+operation, not just the final syscall; a per-process reentrant mutex and flock cover threads
+and independent processes.
+All state files are published only after their complete contents have been fsynced. Immutable
+files use link, so even a writer outside the lock cannot overwrite an existing artifact.
+
+Readers opening a file see an old or new complete inode. Multi-file readers must hold transaction(),
+as the local executor does for the entire operation. This is not a cross-host/NFS locking protocol.
+"""
+
+from __future__ import annotations
+
+import errno
+import fcntl
+import os
+import tempfile
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from functools import wraps
+from pathlib import Path
+from threading import Lock, RLock
+from typing import Concatenate, ParamSpec, Protocol, TypeVar
+
+
+class DurabilityUncertain(OSError):
+    """Publication happened, but its durability could not be confirmed. Inspect before retrying."""
+
+
+class _LockState:
+    def __init__(self) -> None:
+        self.mutex = RLock()
+        self.fd: int | None = None
+
+
+_locks: dict[Path, _LockState] = {}
+_guard = Lock()
+
+
+def _after_fork() -> None:
+    # Close inherited descriptions, without LOCK_UN (which would unlock the parent's flock).
+    # A child must neither inherit a reentrancy decision nor keep a dead parent's lock alive.
+    global _guard
+    for _, state in sorted(_locks.items()):
+        if state.fd is not None:
+            os.close(state.fd)
+    _locks.clear()
+    _guard = Lock()
+
+
+def _before_fork() -> None:
+    # Fork must not fall between opening/closing a descriptor and updating its registration.
+    _guard.acquire()
+
+
+def _after_parent_fork() -> None:
+    _guard.release()
+
+
+os.register_at_fork(
+    before=_before_fork, after_in_parent=_after_parent_fork, after_in_child=_after_fork
+)
+
+
+def sync_directory(path: Path) -> None:
+    fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def confirm_file(path: Path) -> None:
+    """Confirm an already visible result before treating an uncertain publication as complete."""
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+    sync_directory(path.parent)
+
+
+def ensure_directory(path: Path) -> None:
+    """Create parents and persist the directory entries, not only the files placed in them."""
+    if path.is_dir():
+        return
+    ensure_directory(path.parent)
+    try:
+        path.mkdir(mode=0o700)
+    except FileExistsError:
+        if not path.is_dir():
+            raise
+    sync_directory(path.parent)
+
+
+@contextmanager
+def transaction(directory: Path) -> Iterator[None]:
+    root = directory.resolve()
+    ensure_directory(root)
+    with _guard:
+        state = _locks.setdefault(root, _LockState())
+    with state.mutex:
+        if state.fd is not None:
+            yield
+            return
+        with _guard:
+            fd = os.open(root / ".state.lock", os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600)
+            state.fd = fd
+        owner_pid = os.getpid()
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            # Closing releases flock even after an exception. The lock file is never unlinked:
+            # replacing it could give a new process a different inode from an existing waiter.
+            if os.getpid() == owner_pid:
+                with _guard:
+                    state.fd = None
+                    os.close(fd)
+
+
+class _Store(Protocol):
+    @property
+    def root(self) -> Path: ...
+
+
+S = TypeVar("S", bound=_Store)
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def serialized(method: Callable[Concatenate[S, P], R]) -> Callable[Concatenate[S, P], R]:
+    @wraps(method)
+    def wrapped(self: S, /, *args: P.args, **kwargs: P.kwargs) -> R:
+        with transaction(self.root):
+            return method(self, *args, **kwargs)
+
+    return wrapped
+
+
+def _write_all(fd: int, payload: bytes) -> None:
+    remaining = memoryview(payload)
+    while remaining:
+        written = os.write(fd, remaining)
+        if written == 0:
+            raise OSError(errno.EIO, "state write made no progress")
+        remaining = remaining[written:]
+
+
+def write_bytes(path: Path, payload: bytes, *, replace: bool = True) -> None:
+    """Publish a complete, durable file, with mode 0600 before any bytes are written.
+
+    Before publication, failure leaves the prior file untouched (or the name absent). After
+    publication an fsync failure is an explicitly uncertain result, never a claimed rollback.
+    """
+    with transaction(path.parent):
+        ensure_directory(path.parent)
+        fd, temporary_name = tempfile.mkstemp(
+            prefix=f".{path.name}.", suffix=".tmp", dir=path.parent
+        )
+        temporary = Path(temporary_name)
+        published = False
+        try:
+            try:
+                _write_all(fd, payload)
+                os.fsync(fd)
+            finally:
+                os.close(fd)
+            if replace:
+                os.replace(temporary, path)
+            else:
+                os.link(temporary, path)
+            published = True
+            sync_directory(path.parent)
+        except OSError as exc:
+            if published:
+                raise DurabilityUncertain(
+                    f"{path.name} was published, but directory durability could not be confirmed. "
+                    "Inspect the stored state before retrying; the previous value was not restored."
+                ) from exc
+            raise
+        finally:
+            temporary.unlink(missing_ok=True)
+
+
+def write_text(path: Path, text: str, *, replace: bool = True) -> None:
+    write_bytes(path, text.encode("utf-8"), replace=replace)