smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/cutover.py

@@ -0,0 +1,286 @@
+"""Load a signed local cutover authorization without touching an engine or adopting a map."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import re
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from .canonical import CanonicalError, canonical_bytes
+from .errors import MapError, MigrationRefused
+from .generation import GENERATIONS_SINCE, MAX_EPOCH, check_map_project, json_numbers
+from .model import LogicalModel
+from .placement import PlacementMap, _verify_signature, load_map
+from .shapes import enumerate_shapes
+from .verification import VerificationRequest
+
+CUTOVER_PROTOCOL = 1
+"""A move: the maintained copy that takes over is in another engine binding than the source."""
+CUTOVER_RELAYOUT_PROTOCOL = 2
+"""A relayout: the copy that takes over is in the source's own engine binding, under fresh names -
+the cutover that follows a staging protocol 2 preparation. Separate for the reason given there:
+each protocol stays strict, and an older operator refuses a relayout by its number."""
+_FIELDS = {
+    "kind",
+    "protocol",
+    "plan_id",
+    "project_id",
+    "group",
+    "pause_budget_ms",
+    "query_impact_digest",
+    "verification",
+    "before",
+    "success",
+    "abort",
+    "signature",
+}
+Outcome = Literal["success", "abort"]
+
+
+def _hex(value: Any, width: int, name: str, subject: str = "cutover") -> str:
+    if not isinstance(value, str) or re.fullmatch(f"[0-9a-f]{{{width}}}", value) is None:
+        raise MigrationRefused(f"{subject} {name} must be {width} lowercase hexadecimal digits")
+    return value
+
+
+def _positive(value: Any, name: str) -> int:
+    if type(value) is not int or not 1 <= value <= MAX_EPOCH:
+        raise MigrationRefused(f"cutover {name} must be a positive safe integer")
+    return value
+
+
+def _record(value: Any, name: str, subject: str = "cutover") -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise MigrationRefused(f"{subject} {name} must be an object")
+    return value
+
+
+def _signature(record: dict[str, Any], subject: str = "cutover") -> None:
+    signature = _record(record.get("signature"), "signature", subject)
+    if set(signature) not in ({"alg", "value"}, {"alg", "value", "key_id"}):
+        raise MigrationRefused(f"{subject} signatures have missing or unknown fields")
+    value = signature.get("value")
+    malformed = f"{subject} signatures must use ed25519 and canonical base64"
+    if signature.get("alg") != "ed25519" or not isinstance(value, str):
+        raise MigrationRefused(malformed)
+    try:
+        decoded = base64.b64decode(value, validate=True)
+    except ValueError as exc:
+        raise MigrationRefused(malformed) from exc
+    if len(decoded) != 64 or base64.b64encode(decoded).decode() != value:
+        raise MigrationRefused(malformed)
+    if "key_id" in signature and not isinstance(signature["key_id"], str):
+        raise MigrationRefused(f"{subject} signature key_id must be a string")
+
+
+@dataclass(frozen=True)
+class CutoverPlan:
+    plan_id: str
+    project_id: str
+    group: str
+    before: PlacementMap
+    success: PlacementMap
+    abort: PlacementMap
+    verification: VerificationRequest
+    pause_budget_ms: int
+    query_impact_digest: str
+    verified_with: str | None
+    fingerprint: str | None = field(default=None, init=False)
+    _document: bytes = field(default=b"", init=False, repr=False)
+
+    def _loaded(self) -> None:
+        if self.fingerprint is None or not self._document:
+            raise MigrationRefused("cutover execution requires an immutable loaded plan")
+
+    @property
+    def source_epoch(self) -> int:
+        self._loaded()
+        value = self.before.placement_of(self.group).write_epoch
+        assert value is not None
+        return value
+
+    @property
+    def maintenance_epoch(self) -> int:
+        return self.source_epoch + 1
+
+    @property
+    def activation_epoch(self) -> int:
+        return self.source_epoch + 2
+
+    def as_record(self) -> dict[str, Any]:
+        self._loaded()
+        result: dict[str, Any] = json.loads(self._document)
+        return result
+
+    def candidate_payload(self, outcome: Outcome) -> bytes:
+        if outcome not in ("success", "abort"):
+            raise MigrationRefused("cutover outcome must be success or abort")
+        return canonical_bytes(self.as_record()[outcome])
+
+    def check_current(self, current: PlacementMap) -> None:
+        self._loaded()
+        check_map_project(current, self.project_id)
+        if not current.signed or current.fingerprint != self.before.fingerprint:
+            raise MigrationRefused("cutover plan does not name the current placement map")
+
+
+def _load(
+    raw: Mapping[str, Any],
+    *,
+    model: LogicalModel,
+    project_id: str,
+    public_key: bytes | Mapping[str, bytes],
+) -> CutoverPlan:
+    body = _record(json_numbers(deepcopy(raw)), "plan")
+    if set(body) != _FIELDS:
+        raise MigrationRefused("cutover plan has missing or unknown fields")
+    if type(body["protocol"]) is not int or body["protocol"] not in (
+        CUTOVER_PROTOCOL,
+        CUTOVER_RELAYOUT_PROTOCOL,
+    ):
+        raise MigrationRefused("unsupported cutover plan protocol")
+    protocol: int = body["protocol"]
+    if body["kind"] != "sde-cutover":
+        raise MigrationRefused("unsupported cutover document kind")
+    identity = _hex(body["plan_id"], 32, "plan_id")
+    local = _hex(body["project_id"], 32, "project_id")
+    if local != project_id:
+        raise MigrationRefused("cutover plan belongs to another locally configured project")
+    budget = _positive(body["pause_budget_ms"], "pause_budget_ms")
+    approval = _hex(body["query_impact_digest"], 64, "query_impact_digest")
+    group = body["group"]
+    if not isinstance(group, str) or not group:
+        raise MigrationRefused("cutover group must be a nonempty string")
+    _signature(body)
+    verified_with = _verify_signature(body, public_key)
+    documents = {name: _record(body[name], name) for name in ("before", "success", "abort")}
+    maps: dict[str, PlacementMap] = {}
+    for name, document in documents.items():
+        _signature(document)
+        for raw_spot in _record(document.get("groups"), "groups").values():
+            raw_spot = _record(raw_spot, "group")
+            derived = raw_spot.get("derived", [])
+            if not isinstance(derived, list):
+                raise MigrationRefused("cutover derived copies must be an array")
+            for raw_material in (raw_spot.get("source"), *derived):
+                raw_material = _record(raw_material, "materialization")
+                layout = _record(raw_material.get("layout"), "layout")
+                if layout.get("auto"):
+                    raise MigrationRefused("cutover maps require explicit physical layouts")
+        parsed = load_map(document, model=model, public_key=public_key, require_signature=True)
+        # Generations arrived in contract 4 and contract 5 keeps them. The three candidates share
+        # every top-level attribute, contract included, which the comparison below enforces.
+        if parsed.contract < GENERATIONS_SINCE:
+            raise MigrationRefused(
+                f"cutover protocol {protocol} requires placement map contract "
+                f"{GENERATIONS_SINCE} or later"
+            )
+        check_map_project(parsed, project_id)
+        _positive(parsed.map_version, "map_version")
+        maps[name] = parsed
+    before, success, abort = (maps[name] for name in ("before", "success", "abort"))
+    if not before.map_version < success.map_version < abort.map_version:
+        raise MigrationRefused("cutover versions must increase from before to success to abort")
+    request = VerificationRequest.from_record(body["verification"])
+    if not request.requires_signature:
+        raise MigrationRefused("cutover verification must require the signed before map")
+    request.check_session(before, project_id=project_id, group=group)
+    spot = before.placement_of(group)
+    if len(spot.derived) != 1 or spot.also_write != spot.derived:
+        raise MigrationRefused("cutover requires exactly one derived copy maintained by fan-out")
+    source, target = spot.source, spot.derived[0]
+    same = source.engine == target.engine
+    if protocol == CUTOVER_PROTOCOL and same:
+        raise MigrationRefused("cutover source and target must use different engine bindings")
+    if protocol == CUTOVER_RELAYOUT_PROTOCOL and not same:
+        raise MigrationRefused(
+            "a relayout (cutover protocol 2) activates a copy in the source's own engine binding"
+        )
+    epoch = spot.write_epoch
+    assert epoch is not None
+    if epoch > MAX_EPOCH - 2:
+        raise MigrationRefused("cutover needs two available write generations")
+    affected = {shape.id for shape in enumerate_shapes(model) if shape.group == group}
+    if any(before.routing.get(shape, source.id) != source.id for shape in affected):
+        raise MigrationRefused("cutover before-map reads must still use the source")
+    before_raw = documents["before"]
+    expected_routes = {
+        shape: value for shape, value in before.routing.items() if shape not in affected
+    }
+    stable = {
+        key: value
+        for key, value in before_raw.items()
+        if key not in {"signature", "map_version", "groups", "routing"}
+    }
+    for name, material, terminal_epoch in (
+        ("success", target, epoch + 2),
+        ("abort", source, epoch + 1),
+    ):
+        document, parsed = documents[name], maps[name]
+        unchanged = {
+            key: value
+            for key, value in document.items()
+            if key not in {"signature", "map_version", "groups", "routing"}
+        }
+        if canonical_bytes(unchanged) != canonical_bytes(stable) or set(parsed.groups) != set(
+            before.groups
+        ):
+            raise MigrationRefused("cutover cannot change other map attributes or groups")
+        if dict(parsed.routing) != expected_routes:
+            raise MigrationRefused("cutover terminal routing must preserve the unaffected groups")
+        for other in before.groups:
+            if other != group and canonical_bytes(document["groups"][other]) != canonical_bytes(
+                before_raw["groups"][other]
+            ):
+                raise MigrationRefused("cutover cannot change an unaffected group")
+        terminal = document["groups"][group]
+        if set(terminal) != {"source", "write_epoch"} or terminal["write_epoch"] != terminal_epoch:
+            raise MigrationRefused(
+                "cutover terminal group must contain only its source and decision generation"
+            )
+        candidate = terminal["source"]
+        if not isinstance(candidate.get("id"), str) or not candidate["id"]:
+            raise MigrationRefused("cutover terminal source must have a nonempty string id")
+        original = (
+            before_raw["groups"][group]["source"]
+            if name == "abort"
+            else before_raw["groups"][group]["derived"][0]
+        )
+        expected = {
+            key: value for key, value in original.items() if key not in {"id", "lag_budget_ms"}
+        }
+        actual = {key: value for key, value in candidate.items() if key != "id"}
+        if (
+            canonical_bytes(actual) != canonical_bytes(expected)
+            or parsed.groups[group].source.engine != material.engine
+        ):
+            raise MigrationRefused(
+                "cutover terminal map changed the authorized physical materialization"
+            )
+    plan = CutoverPlan(
+        identity, local, group, before, success, abort, request, budget, approval, verified_with
+    )
+    document_bytes = canonical_bytes(body)
+    unsigned = {key: value for key, value in body.items() if key != "signature"}
+    object.__setattr__(plan, "_document", document_bytes)
+    object.__setattr__(plan, "fingerprint", hashlib.sha256(canonical_bytes(unsigned)).hexdigest())
+    return plan
+
+
+def load_cutover_plan(
+    raw: Mapping[str, Any],
+    *,
+    model: LogicalModel,
+    project_id: str,
+    public_key: bytes | Mapping[str, bytes],
+) -> CutoverPlan:
+    """Verify exact authorization and candidates. No I/O, reservation, activation or watermark."""
+    try:
+        return _load(raw, model=model, project_id=project_id, public_key=public_key)
+    except (MapError, CanonicalError) as exc:
+        raise MigrationRefused(f"cutover document refused: {exc}") from exc

sde/engines/_clickhouse_connection.py

@@ -0,0 +1,224 @@
+"""Pure normalization of the portable ClickHouse DSN profile.
+
+No sockets, CA reads or driver factories run here. The connection owner qualifies local trust
+material before creating a client. Never include a raw DSN or its credentials in parser errors.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import math
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+from urllib.parse import parse_qsl, unquote_to_bytes, urlsplit
+
+from ..errors import EngineError
+
+CONNECT_TIMEOUT_SECONDS = 10
+HANDSHAKE_TIMEOUT_SECONDS = 15
+MIN_TIMEOUT_SECONDS = 0.001
+MAX_TIMEOUT_SECONDS = 2147483.647
+_QUERY_KEYS = frozenset({"secure", "verify", "ca_cert", "connect_timeout", "send_receive_timeout"})
+_SCHEMES = frozenset({"clickhouse", "clickhouses", "http", "https"})
+_PERCENT = re.compile(r"%(?![0-9a-fA-F]{2})")
+_DNS_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", re.ASCII)
+_NUMERIC_HOST = re.compile(r"(?:[0-9]+|0x[0-9a-f]+)(?:\.(?:[0-9]+|0x[0-9a-f]+))*\.?", re.ASCII)
+_SECONDS = re.compile(r"(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", re.ASCII)
+
+
+@dataclass(frozen=True)
+class ConnectionParameters:
+    host: str = field(repr=False)
+    port: int
+    username: str = field(repr=False)
+    password: str = field(repr=False)
+    database: str = field(repr=False)
+    secure: bool
+    ca_cert: str | None = field(default=None, repr=False)
+    connect_timeout: float = CONNECT_TIMEOUT_SECONDS
+    send_receive_timeout: float = HANDSHAKE_TIMEOUT_SECONDS
+
+    @property
+    def interface(self) -> Literal["http", "https"]:
+        return "https" if self.secure else "http"
+
+
+def _require(condition: bool, message: str) -> None:
+    if not condition:
+        raise EngineError(message)
+
+
+def _no_controls(value: str) -> None:
+    _require(
+        all(ord(character) >= 32 and ord(character) != 127 for character in value),
+        "ClickHouse connection fields cannot contain control characters",
+    )
+
+
+def _decode(value: str) -> str:
+    result = unquote_to_bytes(value).decode("utf-8", errors="strict")
+    _no_controls(result)
+    return result
+
+
+def _host(value: str) -> str:
+    _require(
+        value.isascii() and "%" not in value, "ClickHouse host must be ASCII DNS or an IP address"
+    )
+    value = value.lower()
+    try:
+        address = ipaddress.ip_address(value)
+    except ValueError:
+        _require(
+            _NUMERIC_HOST.fullmatch(value) is None, "Legacy numeric IPv4 forms are not supported"
+        )
+        labels = value[:-1].split(".") if value.endswith(".") else value.split(".")
+        _require(
+            len(value) <= 253 and all(_DNS_LABEL.fullmatch(label) is not None for label in labels),
+            "ClickHouse host is not a valid DNS name or IP address",
+        )
+        return value
+    return str(address)
+
+
+def _seconds(value: str | None, default: float) -> float:
+    if value is None:
+        return default
+    _require(
+        _SECONDS.fullmatch(value) is not None, "ClickHouse timeout must be positive finite seconds"
+    )
+    result = float(value)
+    _require(
+        math.isfinite(result) and MIN_TIMEOUT_SECONDS <= result <= MAX_TIMEOUT_SECONDS,
+        "ClickHouse timeout must be positive finite seconds",
+    )
+    return result
+
+
+def parse_dsn(dsn: str) -> ConnectionParameters:
+    """Parse the shared URI profile once; errors name rules, never connection values."""
+    try:
+        _require(isinstance(dsn, str) and bool(dsn), "Provide a ClickHouse connection URI")
+        dsn.encode("utf-8", errors="strict")
+        _require(
+            not any(character.isspace() for character in dsn),
+            "Encode whitespace in connection URIs",
+        )
+        _no_controls(dsn)
+        _require(
+            "#" not in dsn and "\\" not in dsn and _PERCENT.search(dsn) is None,
+            "ClickHouse URI fragments, raw backslashes and malformed percent escapes are refused",
+        )
+        parsed = urlsplit(dsn)
+        scheme = parsed.scheme.lower()
+        _require(
+            scheme in _SCHEMES and bool(parsed.netloc),
+            "Unsupported ClickHouse URI scheme or authority",
+        )
+        _require(
+            parsed.hostname is not None and parsed.netloc.count("@") <= 1,
+            "ClickHouse URI needs one unambiguous host",
+        )
+        authority = parsed.netloc.rsplit("@", 1)[-1]
+        _require("\\" not in authority, "ClickHouse authority cannot contain a backslash")
+        if authority.startswith("["):
+            closing = authority.find("]")
+            _require(
+                closing > 0 and (len(authority) == closing + 1 or authority[closing + 1] == ":"),
+                "ClickHouse IPv6 authority is malformed",
+            )
+            declared_port = authority[closing + 2 :] if len(authority) > closing + 1 else None
+        else:
+            _require(authority.count(":") <= 1, "Enclose IPv6 hosts in URI brackets")
+            declared_port = authority.rsplit(":", 1)[1] if ":" in authority else None
+        port = None
+        if declared_port is not None:
+            _require(
+                re.fullmatch(r"[0-9]+", declared_port, flags=re.ASCII) is not None,
+                "ClickHouse port must be an explicit positive integer",
+            )
+            port = int(declared_port)
+            _require(0 < port <= 65535, "ClickHouse port is outside the valid range")
+        host = _host(parsed.hostname or "")
+        username = _decode(parsed.username) if parsed.username is not None else "default"
+        password = _decode(parsed.password) if parsed.password is not None else ""
+        _require(
+            bool(username) and ":" not in username,
+            "ClickHouse username must be nonempty and contain no colon",
+        )
+        _require(
+            parsed.path.startswith("/") and parsed.path.count("/") == 1 and "\\" not in parsed.path,
+            "ClickHouse URI needs one explicit database path segment",
+        )
+        database = _decode(parsed.path[1:])
+        _require(
+            bool(database) and "/" not in database and database not in (".", ".."),
+            "ClickHouse database must be a nonempty single segment",
+        )
+        options: dict[str, str] = {}
+        for name, value in parse_qsl(
+            parsed.query,
+            keep_blank_values=True,
+            strict_parsing=True,
+            encoding="utf-8",
+            errors="strict",
+        ):
+            _no_controls(name)
+            _no_controls(value)
+            _require(
+                name in _QUERY_KEYS and name not in options,
+                "ClickHouse query parameters must be known and appear once",
+            )
+            options[name] = value
+        requested = options.get("secure")
+        _require(
+            requested is None or requested in ("true", "false"),
+            "ClickHouse secure must be literal true or false",
+        )
+        secure = scheme in ("https", "clickhouses")
+        if scheme == "clickhouse":
+            _require(
+                requested is not None or port not in (443, 8443),
+                "Select TLS or explicit plain transport for an ambiguous ClickHouse port",
+            )
+            secure = requested == "true"
+        elif requested is not None:
+            _require(
+                (requested == "true") == secure, "ClickHouse scheme and secure setting disagree"
+            )
+        _require(
+            "verify" not in options or options["verify"] == "true",
+            "ClickHouse verification must remain enabled; use literal true or omit verify",
+        )
+        _require(
+            secure or not {"verify", "ca_cert"} & options.keys(),
+            "ClickHouse TLS options cannot apply to plain HTTP",
+        )
+        ca_cert = options.get("ca_cert")
+        _require(
+            ca_cert is None or (bool(ca_cert) and Path(ca_cert).is_absolute()),
+            "ClickHouse CA must be an absolute local file path",
+        )
+        if port is None:
+            port = {"http": 80, "https": 443}.get(scheme, 8443 if secure else 8123)
+        return ConnectionParameters(
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            database=database,
+            secure=secure,
+            ca_cert=ca_cert,
+            connect_timeout=_seconds(options.get("connect_timeout"), CONNECT_TIMEOUT_SECONDS),
+            send_receive_timeout=_seconds(
+                options.get("send_receive_timeout"), HANDSHAKE_TIMEOUT_SECONDS
+            ),
+        )
+    except EngineError:
+        raise
+    except (ValueError, TypeError, UnicodeError, AttributeError):
+        raise EngineError(
+            "Invalid ClickHouse URI encoding, authority or connection setting"
+        ) from None