smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/write_fence.py

@@ -0,0 +1,283 @@
+"""Engine-enforced write generations and named barriers, entirely on the client's side.
+
+These are primitives for a cutover executor, not permission to switch a placement map. The caller
+must still close the source and target, compare their stable contents and durably commit its choice.
+A generation travels with each write; checking it before an INSERT would leave a paused process
+able to commit against a stale placement. Native CHECK constraints make that test part of INSERT.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, Literal, Protocol
+
+from .errors import MigrationRefused
+from .generation import DRAIN_TABLE as DRAIN_TABLE
+from .generation import EPOCH_COLUMN as EPOCH_COLUMN
+from .generation import MAX_EPOCH as MAX_EPOCH
+from .generation import check_epoch as check_epoch
+from .placement import BACKFILL_TABLE, WATERMARK_TABLE
+
+FENCE_PREFIX = "__sde_f_"
+SETUP = FENCE_PREFIX + "setup"
+ColumnState = Literal["absent", "valid", "conflict"]
+
+
+@dataclass(frozen=True)
+class FenceMetadata:
+    identity: str
+    column: ColumnState
+    constraints: Mapping[str, str]
+
+
+class FenceBackend(Protocol):
+    """DDL capability held by provisioning/execution roles, never required by a runtime role."""
+
+    def metadata(self, table: str) -> FenceMetadata: ...
+    def add_column(self, table: str) -> None: ...
+    def add_constraint(self, table: str, name: str, expression: str) -> None: ...
+    def drop_constraint(self, table: str, name: str) -> None: ...
+    def drain(self, table: str, *, project_id: str, hold: str) -> None: ...
+    def restore(self, table: str, *, project_id: str, hold: str) -> None: ...
+
+
+def _identity(value: str, label: str) -> None:
+    if not isinstance(value, str) or re.fullmatch("[0-9a-f]{32}", value) is None:
+        raise MigrationRefused(f"write fence {label} must be 32 lowercase hexadecimal digits")
+
+
+def _predicate(value: str) -> str:
+    """Normalize only our predicate grammar, preserving quoted identifiers and their spaces."""
+    value = value.strip()
+    if value.startswith("CHECK"):
+        value = value[5:].strip()
+    value = re.sub(r"\s+NOT\s+VALID$", "", value)
+    while value.startswith("(") and value.endswith(")"):
+        value = value[1:-1].strip()
+    if value in ("true", "1"):
+        return "1"
+    if value in ("false", "0"):
+        return "0"
+    matched = re.fullmatch(
+        r'\(*\s*(?:"__sde_write_epoch"|`__sde_write_epoch`|__sde_write_epoch)'
+        r"\s*\)*\s*(>=|<=)\s*\(*\s*(?:([0-9]+)|'([0-9]+)'::bigint)\s*\)*",
+        value,
+    )
+    if matched is None:
+        return "<unrecognized>"
+    number = matched.group(2) or matched.group(3)
+    return EPOCH_COLUMN + matched.group(1) + str(int(number))
+
+
+@dataclass(frozen=True)
+class FenceState:
+    identity: str
+    project_id: str | None
+    column: ColumnState
+    minimums: tuple[int, ...]
+    maximums: tuple[int, ...]
+    holds: tuple[str, ...]
+    retired: tuple[str, ...]
+
+    @property
+    def lower_epoch(self) -> int | None:
+        return max(self.minimums) if self.minimums else None
+
+    @property
+    def upper_epoch(self) -> int | None:
+        return min(self.maximums) if self.maximums else None
+
+    @property
+    def complete(self) -> bool:
+        return (
+            self.project_id is not None
+            and self.column == "valid"
+            and bool(self.minimums and self.maximums)
+        )
+
+    @property
+    def closed(self) -> bool:
+        low, high = self.lower_epoch, self.upper_epoch
+        return bool(self.holds) or (low is not None and high is not None and low > high)
+
+    @property
+    def epoch(self) -> int | None:
+        if self.complete and self.lower_epoch == self.upper_epoch:
+            return self.lower_epoch
+        return None
+
+    def as_record(self) -> dict[str, Any]:
+        return {
+            "identity": self.identity,
+            "project_id": self.project_id,
+            "column": self.column,
+            "lower_epoch": self.lower_epoch,
+            "upper_epoch": self.upper_epoch,
+            "holds": list(self.holds),
+            "retired": list(self.retired),
+            "closed": self.closed,
+        }
+
+
+def fence_state(metadata: FenceMetadata) -> FenceState:
+    owners: list[str] = []
+    minima: list[int] = []
+    maxima: list[int] = []
+    holds: list[str] = []
+    retired: list[str] = []
+    for name, raw in sorted(metadata.constraints.items()):
+        if not name.startswith(FENCE_PREFIX):
+            continue
+        suffix = name[len(FENCE_PREFIX) :]
+        predicate = _predicate(raw)
+        if re.fullmatch("owner_[0-9a-f]{32}", suffix):
+            owners.append(suffix[6:])
+            expected = "1"
+        elif re.fullmatch("retired_[0-9a-f]{32}", suffix):
+            retired.append(suffix[8:])
+            expected = "1"
+        elif suffix == "setup" or re.fullmatch("hold_[0-9a-f]{32}", suffix):
+            holds.append("setup" if suffix == "setup" else suffix[5:])
+            expected = "0"
+        elif re.fullmatch("(?:min|max)_[1-9][0-9]*", suffix):
+            value = int(suffix[4:])
+            check_epoch(value)
+            (minima if suffix.startswith("min_") else maxima).append(value)
+            expected = EPOCH_COLUMN + (">=" if suffix.startswith("min_") else "<=") + str(value)
+        else:
+            raise MigrationRefused("unrecognized constraint in the reserved write-fence namespace")
+        if predicate != expected:
+            raise MigrationRefused(f"write fence constraint {name} has an unexpected predicate")
+    if len(owners) > 1:
+        raise MigrationRefused("the table carries write fences from more than one project")
+    if not owners and (minima or maxima or holds or retired):
+        raise MigrationRefused("write fence constraints have no project owner")
+    return FenceState(
+        metadata.identity,
+        owners[0] if owners else None,
+        metadata.column,
+        tuple(sorted(minima)),
+        tuple(sorted(maxima)),
+        tuple(sorted(holds)),
+        tuple(sorted(retired)),
+    )
+
+
+class WriteFence:
+    """A table's provisioning capability. Serialize executor commands per project state directory.
+
+    Writes carry a number from their own immutable placement. Replaying
+    a stale administrative command cannot lower the native minimum. A partial command can leave the
+    table closed, so an I/O error calls for inspection/resumption, not an assumption of rollback.
+    """
+
+    def __init__(self, backend: FenceBackend, table: str, *, project_id: str) -> None:
+        _identity(project_id, "project_id")
+        if not isinstance(table, str) or not table or "\0" in table:
+            raise MigrationRefused("write fence table must be a nonempty identifier")
+        if table in (DRAIN_TABLE, BACKFILL_TABLE, WATERMARK_TABLE):
+            raise MigrationRefused("write fences cannot take over an SDK metadata table")
+        self._backend = backend
+        self.table = table
+        self.project_id = project_id
+
+    def state(self) -> FenceState:
+        state = fence_state(self._backend.metadata(self.table))
+        if state.project_id is not None and state.project_id != self.project_id:
+            raise MigrationRefused("the write fence belongs to another project")
+        if state.column == "conflict":
+            raise MigrationRefused("the reserved write-epoch column has an incompatible definition")
+        return state
+
+    def _ready(self) -> FenceState:
+        state = self.state()
+        if not state.complete:
+            raise MigrationRefused("the write fence is not fully provisioned")
+        return state
+
+    def prepare(self, epoch: int) -> FenceState:
+        epoch = check_epoch(epoch)
+        state = self.state()
+        if state.complete and "setup" not in state.holds:
+            if state.epoch != epoch:
+                raise MigrationRefused("provisioning cannot change an existing write epoch")
+            return state
+        if state.project_id is None and state.column != "absent":
+            raise MigrationRefused("the reserved write-epoch column is not owned by this project")
+        if state.lower_epoch is not None and state.lower_epoch > epoch:
+            raise MigrationRefused("a write epoch cannot move backwards")
+        if state.project_id is None:
+            self._backend.add_constraint(self.table, FENCE_PREFIX + "owner_" + self.project_id, "1")
+        self._backend.add_constraint(self.table, SETUP, "0")
+        self._backend.add_column(self.table)
+        self._bounds(epoch)
+        self._backend.drain(self.table, project_id=self.project_id, hold="setup")
+        self._backend.drop_constraint(self.table, SETUP)
+        return self._ready()
+
+    def freeze(self, request_id: str) -> FenceState:
+        _identity(request_id, "request_id")
+        state = self._ready()
+        if request_id in state.retired:
+            raise MigrationRefused("a completed write barrier id cannot be reused")
+        self._backend.add_constraint(self.table, FENCE_PREFIX + "hold_" + request_id, "0")
+        # Always drain, even on a retry: existence of the constraint proves admission is closed,
+        # not that an INSERT which captured older metadata has finished.
+        self._backend.drain(self.table, project_id=self.project_id, hold=request_id)
+        return self._ready()
+
+    def resume(self, request_id: str) -> FenceState:
+        _identity(request_id, "request_id")
+        self._backend.restore(self.table, project_id=self.project_id, hold=request_id)
+        return self.freeze(request_id)
+
+    def resume_prepare(self, epoch: int) -> FenceState:
+        epoch = check_epoch(epoch)
+        self._backend.restore(self.table, project_id=self.project_id, hold="setup")
+        return self.prepare(epoch)
+
+    def advance(self, epoch: int) -> FenceState:
+        epoch = check_epoch(epoch)
+        state = self._ready()
+        if not state.holds:
+            raise MigrationRefused("changing a write epoch needs a named write barrier")
+        if state.lower_epoch is not None and epoch < state.lower_epoch:
+            raise MigrationRefused("a write epoch cannot move backwards")
+        self._bounds(epoch)
+        return self._ready()
+
+    def _bounds(self, epoch: int) -> None:
+        self._backend.add_constraint(
+            self.table, FENCE_PREFIX + "min_" + str(epoch), f"{EPOCH_COLUMN} >= {epoch}"
+        )
+        self._backend.add_constraint(
+            self.table, FENCE_PREFIX + "max_" + str(epoch), f"{EPOCH_COLUMN} <= {epoch}"
+        )
+        # Remove only bounds weaker/older than the one just installed, by their exact names. A
+        # stale caller must never remove a newer constraint discovered after its earlier check.
+        state = self.state()
+        if state.lower_epoch is not None and state.lower_epoch > epoch:
+            raise MigrationRefused(
+                "another executor installed a newer write epoch; table stays closed"
+            )
+        for old in state.maximums:
+            if old < epoch:
+                self._backend.drop_constraint(self.table, FENCE_PREFIX + "max_" + str(old))
+        for old in state.minimums:
+            if old < epoch:
+                self._backend.drop_constraint(self.table, FENCE_PREFIX + "min_" + str(old))
+
+    def release(self, request_id: str) -> FenceState:
+        _identity(request_id, "request_id")
+        state = self._ready()
+        if state.epoch is None:
+            raise MigrationRefused("cannot release a barrier with an incomplete epoch change")
+        if request_id not in state.holds:
+            return state
+        # Record completion before releasing admission. A delayed invocation must never recreate
+        # a completed hold, nor reuse its id for a later generation and inherit an old release.
+        self._backend.add_constraint(self.table, FENCE_PREFIX + "retired_" + request_id, "1")
+        self._backend.drop_constraint(self.table, FENCE_PREFIX + "hold_" + request_id)
+        return self._ready()

sde_demo/__init__.py

@@ -0,0 +1 @@
+"""Customer-side Weather starter; data and local credentials stay in this process."""

sde_demo/__main__.py

@@ -0,0 +1,183 @@
+"""Run the customer-side Weather starter; engine credentials and values stay local."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import sde
+from sde._local_state import transaction
+
+from .model import WORKLOADS, model
+from .project import DemoRefused, config, connections, public_keys, read, setup, write
+from .resources import ResourceRefused
+
+
+def operator(root: Path, action: str, plan: Path | None) -> dict[str, Any]:
+    from sde.local_cutover import _local_status
+
+    from . import resources
+
+    with transaction(root):
+        settings, logical = config(root, ready=action != "status"), model()
+        keys = public_keys(settings["public_keys"])
+        if action == "status":
+            return _local_status(
+                root / "state", model=logical, project_id=settings["project_id"], public_key=keys
+            )
+        resources.verify(root)
+        with connections(root, settings) as (operators, runtime):
+            local = sde.LocalCutover(
+                root / "state",
+                model=logical,
+                project_id=settings["project_id"],
+                public_key=keys,
+                operators=operators,
+                runtime={name: [adapter] for name, adapter in runtime.items()},
+            )
+            if action == "resume":
+                return local.resume().as_record()
+            if action == "abandon":
+                return local.abandon().as_record()
+            if plan is None:
+                raise DemoRefused("Stage, index and execute require --plan from the controller.")
+            if action == "index":
+                return local.index(
+                    sde.load_index_plan(
+                        read(plan),
+                        model=logical,
+                        project_id=settings["project_id"],
+                        public_key=keys,
+                    )
+                ).as_record()
+            if action == "stage":
+                return local.stage(
+                    sde.load_staging_plan(
+                        read(plan),
+                        model=logical,
+                        project_id=settings["project_id"],
+                        public_key=keys,
+                    )
+                ).as_record()
+            return local.execute(
+                sde.load_cutover_plan(
+                    read(plan), model=logical, project_id=settings["project_id"], public_key=keys
+                )
+            ).as_record()
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--directory", type=Path)
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("model", help="print the neutral Weather model for the controller")
+    start = commands.add_parser("setup", help="allocate loopback-only demo resources and enroll")
+    start.add_argument("--bootstrap", type=Path, required=True)
+    reset = commands.add_parser(
+        "reset", help="remove this allocation after stopping demo processes"
+    )
+    reset.add_argument("--confirm-allocation", required=True)
+    for command in (start, reset):
+        command.add_argument("--postgres-admin-env", default="SDE_POSTGRES_DSN")
+        command.add_argument("--clickhouse-admin-env", default="SDE_CLICKHOUSE_DSN")
+    commands.add_parser("doctor", help="check the local map, connections and runtime grants")
+    workload = commands.add_parser("run", help="run a bounded logical workload with runtime rights")
+    workload.add_argument("--iterations", type=int, default=10)
+    workload.add_argument("--batch-size", type=int, default=10)
+    workload.add_argument("--interval-ms", type=int, default=100)
+    workload.add_argument("--recovery-ms", type=int, default=10000)
+    workload.add_argument("--workload", choices=WORKLOADS, default="mixed")
+    verify = commands.add_parser(
+        "verify-runs", help="verify earlier completed Python/TypeScript runs"
+    )
+    verify.add_argument("--run-id", action="append", required=True)
+    query = commands.add_parser(
+        "query-count", help="execute the current approved Weather COUNT locally"
+    )
+    query.add_argument("--record", type=Path, required=True)
+    op = commands.add_parser(
+        "operator", help="use the existing local staging/index/cutover executor"
+    )
+    op.add_argument("action", choices=("status", "stage", "index", "execute", "resume", "abandon"))
+    op.add_argument("--plan", type=Path)
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "model":
+            result = sde.neutral_declaration(model())
+        else:
+            if args.directory is None:
+                raise DemoRefused("Provide --directory for this customer-side demo allocation.")
+            root = args.directory.absolute()
+            if args.command in ("setup", "reset"):
+                admin = {
+                    dialect: os.environ[variable]
+                    for dialect, variable in (
+                        ("postgres", args.postgres_admin_env),
+                        ("clickhouse", args.clickhouse_admin_env),
+                    )
+                    if os.environ.get(variable)
+                }
+                if args.command == "setup":
+                    result = setup(root, read(args.bootstrap), admin)
+                else:
+                    from .resources import reset as reset_resources
+
+                    with transaction(root):
+                        allocation = read(root / "resources.json")["allocation_id"]
+                        if args.confirm_allocation != allocation:
+                            raise DemoRefused(
+                                "Confirm the allocation_id from local resources.json."
+                            )
+                        write(root / "reset-request.json", {"allocation_id": allocation}, once=True)
+                        result = reset_resources(root, admin)
+            elif args.command == "doctor":
+                from .diagnostics import doctor
+
+                result = doctor(root)
+            elif args.command == "operator":
+                result = operator(root, args.action, args.plan)
+            elif args.command == "verify-runs":
+                from .verification import verify_runs
+
+                result = verify_runs(root, args.run_id)
+            elif args.command == "query-count":
+                from .query_count import run_count_query
+
+                result = run_count_query(root, read(args.record))
+            else:
+                from .runtime import run
+
+                result = run(
+                    root,
+                    iterations=args.iterations,
+                    batch_size=args.batch_size,
+                    interval_ms=args.interval_ms,
+                    recovery_ms=args.recovery_ms,
+                    workload=args.workload,
+                )
+        print(json.dumps(result, sort_keys=True, allow_nan=False))
+        return 0
+    except (DemoRefused, ResourceRefused) as exc:
+        print(json.dumps({"error": "demo_refused", "message": str(exc)}), file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(
+            json.dumps(
+                {
+                    "error": type(exc).__name__,
+                    "message": "Operation incomplete. Inspect local state and the run report "
+                    "before retrying; uncertain writes must not be replayed.",
+                }
+            ),
+            file=sys.stderr,
+        )
+        return 3
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sde_demo/diagnostics.py

@@ -0,0 +1,92 @@
+"""Local diagnostics using existing native identity and grant qualification."""
+
+from __future__ import annotations
+
+import platform
+from pathlib import Path
+from typing import Any
+
+import sde
+from sde._local_state import transaction
+from sde.engines._operator import NativeOperator
+
+from . import resources
+from .model import model
+from .project import config, connections, public_keys, require_drivers
+
+
+def doctor(root: Path) -> dict[str, Any]:
+    with transaction(root):
+        settings = config(root)
+        require_drivers(binding["dialect"] for binding in settings["engines"].values())
+        resources.verify(root)
+        return _diagnose(root, settings)
+
+
+def _diagnose(root: Path, settings: dict[str, Any]) -> dict[str, Any]:
+    logical = model()
+    placement = sde.load_local_map(
+        root / "state",
+        model=logical,
+        project_id=settings["project_id"],
+        public_key=public_keys(settings["public_keys"]),
+    )
+    tables: dict[str, set[str]] = {name: {"sde_map_state"} for name in settings["engines"]}
+    for group in placement.groups.values():
+        for material in group.all():
+            tables[material.engine].update(material.layout.tables.values())
+    results = {}
+    with connections(root, settings) as (operators, runtime):
+        for name, operator in operators.items():
+            native = NativeOperator(operator, [runtime[name]])
+            native.qualify(sorted(tables[name]), sorted(tables[name]))
+            if operator.dialect == "postgres":
+                version = str(native.rows("SHOW server_version")[0][0])
+                tls = bool(
+                    native.rows(
+                        "SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()",
+                        engine=runtime[name],
+                    )[0][0]
+                )
+            else:
+                version = str(native.rows("SELECT version()")[0][0])
+                tls = runtime[name]._cx.url.startswith("https://")
+            results[name] = {
+                "dialect": operator.dialect,
+                "version": version,
+                "native_identity": "matched",
+                "runtime_privileges": "qualified",
+                "connection_tls": tls,
+            }
+        with sde.Session(
+            logical, placement, runtime, project_id=settings["project_id"]
+        ) as session:
+            storage = session.measure_storage()
+        # Whether each engine can give the window a group's size, by the class of the reason -
+        # never the size itself. An engine that holds no group's source has nothing to measure.
+        measured = {size.engine for size in storage.sizes}
+        for name, result in results.items():
+            sources = [
+                group for group, spot in placement.groups.items() if spot.source.engine == name
+            ]
+            reasons = sorted(
+                {storage.unavailable[group] for group in sources if group in storage.unavailable}
+            )
+            result["storage_telemetry"] = (
+                "no_source" if not sources
+                else "available" if name in measured and not reasons
+                else "unavailable: " + ", ".join(reasons)
+            )
+    return {
+        "protocol": 1,
+        "status": "ready",
+        "scope": "local-weather-demo",
+        "sdk_version": sde.__version__,
+        "python_version": platform.python_version(),
+        "project_id": settings["project_id"],
+        "model_version": logical.version,
+        "map_version": placement.map_version,
+        "signature": "verified",
+        "engines": results,
+        "production_qualification": "not_performed",
+    }

sde_demo/model.py

@@ -0,0 +1,75 @@
+"""The fixed, portable model used by the customer Weather starter."""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from typing import Any
+from uuid import UUID
+
+import sde
+from sde.testing.loader import model_from_neutral
+
+DECLARATION: dict[str, Any] = {
+    "entities": [
+        {
+            "name": "WeatherReading",
+            "fields": [
+                {"name": "at", "type": "timestamptz"},
+                {"name": "celsius", "type": "decimal(8,2)"},
+                {"name": "humidity", "type": "int64"},
+                {"name": "id", "type": "uuid"},
+                {"name": "station", "type": "string"},
+            ],
+            "key": ["station", "at"],
+            "residency": "EU",
+        }
+    ],
+    "relations": [],
+    "atomic": [],
+    "cost_ceiling": {"amount": "500.00", "currency": "EUR"},
+}
+BASE_TIME = datetime(2026, 1, 1, tzinfo=UTC)
+STATION_TEMPLATE = "weather-{run_id}-{worker}"
+SEED_TEMPLATE = "sde-weather-v1:{run_id}:{worker}:{sequence}"
+CELSIUS_BASE_CENTS, CELSIUS_MODULUS = 1525, 1000
+HUMIDITY_BASE, HUMIDITY_MODULUS = 30, 70
+UUID_VERSION = 4
+WORKLOADS = ("mixed", "point", "analytics", "fleet", "alerts")
+"""What a run can drive. The CLI offers and the runtime accepts this one tuple."""
+GENERATOR_SPEC = {
+    "kind": "sde-weather-generator",
+    "version": 1,
+    "worker": 0,
+    "station_template": STATION_TEMPLATE,
+    "seed_template": SEED_TEMPLATE,
+    "base_time": BASE_TIME.isoformat(timespec="microseconds").replace("+00:00", "Z"),
+    "timestamp_step_microseconds": 1,
+    "celsius_base_cents": CELSIUS_BASE_CENTS,
+    "celsius_modulus": CELSIUS_MODULUS,
+    "humidity_base": HUMIDITY_BASE,
+    "humidity_modulus": HUMIDITY_MODULUS,
+    "id": "first 16 bytes of SHA256(UTF8(seed)), RFC4122 UUID version bits",
+    "uuid_version": UUID_VERSION,
+}
+GENERATOR_ID = "weather-v1:" + hashlib.sha256(sde.canonical_bytes(GENERATOR_SPEC)).hexdigest()
+
+
+def model() -> sde.LogicalModel:
+    return model_from_neutral(DECLARATION)
+
+
+def reading(run_id: str, worker: int, sequence: int) -> dict[str, Any]:
+    """Deterministic synthetic values; the run namespace prevents keys crossing runs."""
+    identity = hashlib.sha256(
+        SEED_TEMPLATE.format(run_id=run_id, worker=worker, sequence=sequence).encode()
+    ).digest()[:16]
+    cents = CELSIUS_BASE_CENTS + sequence % CELSIUS_MODULUS
+    return {
+        "station": STATION_TEMPLATE.format(run_id=run_id, worker=worker),
+        "at": BASE_TIME + timedelta(microseconds=sequence),
+        "celsius": Decimal(f"{cents // 100}.{cents % 100:02d}"),
+        "humidity": HUMIDITY_BASE + sequence % HUMIDITY_MODULUS,
+        "id": UUID(bytes=identity, version=UUID_VERSION),
+    }