smart-data-engine-sdk 0.1.0.dev0__py3-none-any.whl

sde/__init__.py

@@ -0,0 +1,226 @@
+"""Smart Data Engine - client library.
+
+You declare entities and relations. We decide which database engine each colocation group lives in,
+what its physical layout is there, and when it should move - and your code never names a table or an
+engine, which is exactly what lets us change both without touching it.
+
+    from datetime import datetime from decimal import Decimal from typing import Annotated from uuid
+    import UUID import sde
+
+    @sde.entity
+    class User:
+        id: UUID email: str
+
+        class Meta:
+            pii = ["email"]
+
+    @sde.entity
+    class Order:
+        id: UUID user: sde.Ref[User] total: Annotated[Decimal, sde.precision(12, 2)] created_at:
+        datetime
+
+        class Meta:
+            atomic_with = ["Payment"] residency = "EU"
+
+Everything about storage is our decision. The four things you declare - atomicity, residency,
+personal data and a cost ceiling - are the ones that cannot be read from traffic no matter how long
+we watch it.
+
+This library is Apache-2.0 and it works without an account: hand it a placement map you wrote
+yourself and it will route, create schema and run, with no key and no network. That is a supported
+mode, not a loophole.
+"""
+
+from __future__ import annotations
+
+from .canonical import CanonicalError, canonical_bytes, canonical_str, digest16
+from .capabilities import members_of, satisfies
+from .entity import Ref, clear_registry, entity, registry
+from .errors import (
+    DeclarationError,
+    EngineError,
+    MapError,
+    MapRolledBack,
+    MigrationRefused,
+    ModelPlanningError,
+    SdeError,
+)
+from .explain import Cost, Explains, PlanFinding, QueryPlan, QueryPlanRefused, explain
+from .groups import Group, colocation_groups, group_of
+from .hashing import NameMap, hash_identifiers, load_or_create_salt
+from .infer import InferredModel, Note, infer_model, infer_models
+from .internal import internal_failures, reset_internal_failures
+from .layout import (
+    DIALECTS,
+    FIXED_SCHEMA,
+    ORDERBOOK_KEY,
+    ORDERBOOK_SHAPE,
+    ORDERBOOK_TABLE,
+    DerivedLayout,
+    can_store,
+    default_layout,
+    denormalized_layout,
+    fixed_schema_mismatch,
+    group_columns,
+    snake_case,
+    stored_types,
+)
+from .migration import (
+    BACKFILL_TABLE,
+    CHUNK_ROWS,
+    DIALECT_PRECISION,
+    PRECISION_INDEPENDENT,
+    BackfillProgress,
+    Difference,
+    EntityProgress,
+    Migratable,
+    VerifyReport,
+    backfill,
+    precision_refusal,
+    verify,
+)
+from .model import CONTRACT, LogicalModel, build_model, neutral_declaration
+from .placement import (
+    ALSO_WRITE_SINCE,
+    MAP_CONTRACT,
+    MAP_CONTRACT_FLOOR,
+    RESERVED_TABLES,
+    GroupPlacement,
+    Materialization,
+    PhysicalLayout,
+    PlacementMap,
+    load_map,
+)
+from .routing import Router, resolve
+from .schema import CompatibilityViews, compatibility_views, schema_is_fixed, schema_statements
+from .session import Engine, Session
+from .shapes import SHAPE_KINDS, WRITE_KINDS, OperationShape, enumerate_shapes
+from .telemetry import (
+    MEASURED_FIELDS,
+    CopyFreshness,
+    FanOutStats,
+    GroupFeatures,
+    Histogram,
+    Recorder,
+    ShapeStats,
+    Window,
+    has_time_dimension,
+)
+from .types import Float32, Int32, Json, Timestamp, precision
+from .watermark import (
+    WATERMARK_TABLE,
+    Protection,
+    WatermarkCheck,
+    WatermarkStore,
+    enforce_forward_only,
+)
+
+__version__ = "0.1.0.dev0"
+
+__all__ = [
+    "ALSO_WRITE_SINCE",
+    "BACKFILL_TABLE",
+    "CHUNK_ROWS",
+    "CONTRACT",
+    "DIALECTS",
+    "DIALECT_PRECISION",
+    "FIXED_SCHEMA",
+    "MAP_CONTRACT",
+    "MAP_CONTRACT_FLOOR",
+    "MEASURED_FIELDS",
+    "ORDERBOOK_KEY",
+    "ORDERBOOK_SHAPE",
+    "ORDERBOOK_TABLE",
+    "PRECISION_INDEPENDENT",
+    "RESERVED_TABLES",
+    "SHAPE_KINDS",
+    "WATERMARK_TABLE",
+    "WRITE_KINDS",
+    "BackfillProgress",
+    "CanonicalError",
+    "CompatibilityViews",
+    "CopyFreshness",
+    "Cost",
+    "DeclarationError",
+    "DerivedLayout",
+    "Difference",
+    "Engine",
+    "EngineError",
+    "EntityProgress",
+    "Explains",
+    "FanOutStats",
+    "Float32",
+    "Group",
+    "GroupFeatures",
+    "GroupPlacement",
+    "Histogram",
+    "InferredModel",
+    "Int32",
+    "Json",
+    "LogicalModel",
+    "MapError",
+    "MapRolledBack",
+    "Materialization",
+    "Migratable",
+    "MigrationRefused",
+    "ModelPlanningError",
+    "NameMap",
+    "Note",
+    "OperationShape",
+    "PhysicalLayout",
+    "PlacementMap",
+    "PlanFinding",
+    "Protection",
+    "QueryPlan",
+    "QueryPlanRefused",
+    "Recorder",
+    "Ref",
+    "Router",
+    "SdeError",
+    "Session",
+    "ShapeStats",
+    "Timestamp",
+    "VerifyReport",
+    "WatermarkCheck",
+    "WatermarkStore",
+    "Window",
+    "__version__",
+    "backfill",
+    "build_model",
+    "can_store",
+    "canonical_bytes",
+    "canonical_str",
+    "clear_registry",
+    "colocation_groups",
+    "compatibility_views",
+    "default_layout",
+    "denormalized_layout",
+    "digest16",
+    "enforce_forward_only",
+    "entity",
+    "enumerate_shapes",
+    "explain",
+    "fixed_schema_mismatch",
+    "group_columns",
+    "group_of",
+    "has_time_dimension",
+    "hash_identifiers",
+    "infer_model",
+    "infer_models",
+    "internal_failures",
+    "load_map",
+    "load_or_create_salt",
+    "members_of",
+    "neutral_declaration",
+    "precision",
+    "precision_refusal",
+    "registry",
+    "reset_internal_failures",
+    "resolve",
+    "satisfies",
+    "schema_is_fixed",
+    "schema_statements",
+    "snake_case",
+    "stored_types",
+    "verify",
+]

sde/canonical.py

@@ -0,0 +1,141 @@
+"""Canonical encoding: the one place where the cross-language contract lives.
+
+Every SDE library has to produce byte-identical output from this. A definition amounting to
+"whatever the Python implementation does" is not a definition, so the rules are spelled out here and
+in ``docs/format-contract.md``, and the conformance vectors pin them.
+
+The rules, in the order they are applied:
+
+1. UTF-8, no byte order mark.
+2. Object keys are NFC-normalised, then sorted by Unicode code point. Normalise first: sorting first
+   would order ``é`` (U+00E9) and ``e`` + U+0301 differently while both normalise to the same key.
+3. No insignificant whitespace. ``{"a":1,"b":[2,3]}`` and nothing else.
+4. Every string is NFC-normalised.
+5. Escaping is minimal and exhaustive: only ``"``, ``\\`` and C0 control characters are escaped,
+   controls using the short forms where JSON defines them and ``\\u00XX`` otherwise. Everything else
+   is emitted as raw UTF-8. This matters because JSON writers disagree: some escape all non-ASCII,
+   some escape U+2028, some escape forward slashes. Any of those would change the hash.
+6. Floating point values are rejected outright. Their textual form differs across languages and the
+   difference is unfixable after the fact. Note the distinction that cost a spec revision: a *field*
+   may have type ``float64``; the IR records the *name* of that type, which is a string. There is no
+   float literal anywhere in the encoding.
+7. Integers are emitted as their shortest decimal form, no leading ``+``, no exponent.
+
+Ordering of arrays is not this module's business. Where order carries no meaning the caller sorts;
+where it does, the caller records an explicit index in each element rather than relying on position.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import unicodedata
+from typing import Final
+
+__all__ = ["CanonicalError", "canonical_bytes", "canonical_str", "digest16"]
+
+# Short escape forms JSON defines. Everything else below 0x20 gets \u00XX.
+_SHORT_ESCAPES: Final[dict[int, str]] = {
+    0x08: "\\b",
+    0x09: "\\t",
+    0x0A: "\\n",
+    0x0C: "\\f",
+    0x0D: "\\r",
+    0x22: '\\"',
+    0x5C: "\\\\",
+}
+
+
+class CanonicalError(ValueError):
+    """A value cannot be encoded canonically.
+
+    Raised rather than coerced on purpose: silently accepting a float, a NaN or a non-string key
+    would produce bytes that another implementation cannot reproduce, and the failure would surface
+    much later as two versions of the same model.
+    """
+
+
+def _escape(text: str) -> str:
+    normalised = unicodedata.normalize("NFC", text)
+    out: list[str] = ['"']
+    for char in normalised:
+        code = ord(char)
+        short = _SHORT_ESCAPES.get(code)
+        if short is not None:
+            out.append(short)
+        elif code < 0x20:
+            out.append(f"\\u{code:04x}")
+        else:
+            out.append(char)
+    out.append('"')
+    return "".join(out)
+
+
+def _encode(value: object, path: str) -> str:
+    # bool before int: bool is a subclass of int in Python and would otherwise encode as 1/0.
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    if value is None:
+        return "null"
+    if isinstance(value, str):
+        return _escape(value)
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        raise CanonicalError(
+            f"float at {path}: floating point is not representable in canonical form, "
+            "because its textual form differs between languages. Use an integer, or a decimal "
+            "string, or the name of a float type if you meant to describe a type."
+        )
+    if isinstance(value, (list, tuple)):
+        items = [_encode(item, f"{path}[{i}]") for i, item in enumerate(value)]
+        return "[" + ",".join(items) + "]"
+    if isinstance(value, dict):
+        pairs: list[tuple[str, object]] = []
+        for key, item in value.items():
+            if not isinstance(key, str):
+                raise CanonicalError(
+                    f"non-string key {key!r} at {path}: object keys must be strings, because "
+                    "key ordering is defined over Unicode code points"
+                )
+            pairs.append((unicodedata.normalize("NFC", key), item))
+        # Sort after normalising, on the normalised key.
+        pairs.sort(key=lambda pair: pair[0])
+        seen: set[str] = set()
+        parts: list[str] = []
+        for key, item in pairs:
+            if key in seen:
+                raise CanonicalError(
+                    f"duplicate key {key!r} at {path} after NFC normalisation: two keys that "
+                    "differ "
+                    "only in Unicode composition are the same key here"
+                )
+            seen.add(key)
+            parts.append(_escape(key) + ":" + _encode(item, f"{path}.{key}"))
+        return "{" + ",".join(parts) + "}"
+    raise CanonicalError(
+        f"{type(value).__name__} at {path} has no canonical form. The canonical encoding accepts "
+        "only null, bool, int, str, list and dict; anything richer has to be reduced to those by "
+        "the caller, so that the reduction is visible and testable."
+    )
+
+
+def canonical_str(value: object) -> str:
+    """Canonical form as text. Prefer :func:`canonical_bytes` for hashing."""
+    return _encode(value, "$")
+
+
+def canonical_bytes(value: object) -> bytes:
+    """Canonical form as UTF-8 bytes. This is what gets hashed and what vectors compare."""
+    return canonical_str(value).encode("utf-8")
+
+
+def digest16(value: object) -> str:
+    """The identifier form used for ``model_version`` and ``shape.id``.
+
+    Lowercase hex, first 8 bytes of SHA-256 over the canonical bytes. Sixteen characters is short
+    enough to appear in logs and error messages without wrapping, and 64 bits of collision
+    resistance is ample for the number of models and shapes one application declares.
+    """
+    return hashlib.sha256(canonical_bytes(value)).hexdigest()[:16]

sde/capabilities.py

@@ -0,0 +1,62 @@
+"""Asking an engine adapter whether it will answer a call, which is not the same as its type.
+
+Two optional protocols decide whether an engine takes part in something: :class:`
+sde.watermark.WatermarkStore` for the forward-only map check, and :class:`sde.migration.Migratable`
+for a migration. Both were originally asked with ``isinstance(engine, Protocol)``, which is the
+obvious spelling and the wrong question.
+
+A ``runtime_checkable`` protocol resolves its members with ``hasattr`` up to Python 3.11 and with
+:func:`inspect.getattr_static` from 3.12 onwards - and the second deliberately ignores
+``__getattr__``. So an object that forwards to a wrapped adapter answers ``hasattr`` for every
+member of the protocol, passes ``isinstance`` on 3.11, and fails it on 3.12. This library supports
+and tests all three, which makes that **the same client, the same wrapper, and a different answer
+per interpreter**.
+
+Wrapping an engine adapter is an ordinary thing for a client to do - metrics, logging, a retry, a
+connection pool - and the consequence was two wrong diagnoses shipped as helpful messages: "this
+engine has nowhere to keep the bookkeeping, so you have no rollback protection", and "this engine
+cannot take part in a migration". Both named the client's *engine* for a property of their own
+wrapper, and the second refuses a migration outright.
+
+Found from a test, not from review: a two-line proxy over the real ClickHouse adapter, written to
+make one write fail, was refused as an engine with no row-level operations. The version dependence
+came from CI on 3.11, which is the part no single machine would have shown.
+
+So the question asked here is the one that matters - **will this object respond to these calls** -
+and it is asked with ordinary attribute access, which honours every way Python has of providing an
+attribute.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = ["members_of", "satisfies"]
+
+
+def members_of(protocol: type) -> tuple[str, ...]:
+    """The public members a protocol names: its methods and its annotated data attributes.
+
+    Both halves are needed and neither is enough. Methods have class attributes, so ``dir`` finds
+    them; an annotated attribute with no value - ``dialect: str`` - exists only in
+    ``__annotations__``. Assembled from two public sources rather than from
+    ``__protocol_attrs__``, which is an implementation detail that did not exist in every version
+    this library supports.
+    """
+    named = {name for name in dir(protocol) if not name.startswith("_")}
+    named.update(
+        name
+        for name in getattr(protocol, "__annotations__", {})
+        if not name.startswith("_")
+    )
+    return tuple(sorted(named))
+
+
+def satisfies(obj: Any, protocol: type) -> bool:
+    """Whether every member the protocol names can be reached on this object.
+
+    Presence rather than callability, deliberately. A member that exists and is not callable fails
+    at the call with a message naming it, which is a better failure than a capability check that
+    quietly answers "no" and sends the reader to look at their engine.
+    """
+    return all(hasattr(obj, name) for name in members_of(protocol))