smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/frozen_verification.py

@@ -0,0 +1,152 @@
+"""Drain named native barriers and compare stable copies, including target-only rows.
+
+This is a client-side data operation. It neither chooses a placement nor activates a map. It
+leaves its barriers installed on success and failure; the executor owns the durable decision
+which permits release, rollback or activation.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from time import perf_counter_ns
+from typing import Any, cast
+
+from .errors import MigrationRefused
+from .generation import Fencable, check_epoch
+from .inspection import InspectionContext
+from .migration import CHUNK_ROWS, VerifyReport, verify
+from .verification import VerificationRequest
+from .write_fence import FenceState, WriteFence
+
+
+@dataclass(frozen=True)
+class FrozenTable:
+    engine: str
+    materialization: str
+    table: str
+    identity: str
+    project_id: str
+    epoch: int
+    hold_id: str
+
+    def as_record(self) -> dict[str, Any]:
+        return {
+            "engine": self.engine,
+            "materialization": self.materialization,
+            "table": self.table,
+            "identity": self.identity,
+            "project_id": self.project_id,
+            "epoch": self.epoch,
+            "hold_id": self.hold_id,
+        }
+
+
+@dataclass(frozen=True)
+class FrozenVerifyReport:
+    comparison: VerifyReport
+    barriers: tuple[FrozenTable, ...]
+    elapsed_ms: int
+
+    @property
+    def matched(self) -> bool:
+        return (
+            self.comparison.matched and self.comparison.rows_source == self.comparison.rows_target
+        )
+
+    def as_record(self) -> dict[str, Any]:
+        return {
+            "protocol": 1,
+            "comparison": self.comparison.as_record(),
+            "barriers": [barrier.as_record() for barrier in self.barriers],
+            "elapsed_ms": self.elapsed_ms,
+            "matched": self.matched,
+        }
+
+
+def _matches(state: FenceState, wanted: FrozenTable) -> None:
+    if state.identity != wanted.identity or state.project_id != wanted.project_id:
+        raise MigrationRefused("a frozen comparison table changed identity or project")
+    if not state.complete or state.epoch != wanted.epoch or wanted.hold_id not in state.holds:
+        raise MigrationRefused("a frozen comparison lost its named barrier or write generation")
+
+
+def verify_frozen(
+    context: InspectionContext,
+    group: str,
+    *,
+    request: VerificationRequest,
+    hold_id: str,
+    epochs: Mapping[str, int],
+    chunk_rows: int = CHUNK_ROWS,
+    at: str | None = None,
+) -> FrozenVerifyReport:
+    """Acquire/drain each barrier and compare exact logical content; do not release the holds.
+
+    `epochs` names materialization ids, including the source. They can differ during operator
+    maintenance and therefore cannot be inferred from a runtime session's single group epoch.
+    A caller must budget this work and handle uncertain engine I/O as recovery, not as success.
+    """
+    if not isinstance(hold_id, str) or re.fullmatch("[0-9a-f]{32}", hold_id) is None:
+        raise MigrationRefused(
+            "a frozen comparison hold id must be 32 lowercase hexadecimal digits"
+        )
+    if chunk_rows < 1:
+        raise MigrationRefused("a frozen comparison needs a positive chunk size")
+    request.check_session(context.placement, project_id=context.project_id, group=group)
+    if at is not None:
+        request.check_time(at)
+    spot = context.placement.placement_of(group)
+    materials = (spot.source, *spot.also_write)
+    if not spot.also_write or set(epochs) != {material.id for material in materials}:
+        raise MigrationRefused("frozen comparison epochs must name exactly the source and copy ids")
+    checked_epochs = {name: check_epoch(value) for name, value in epochs.items()}
+    planned: list[tuple[WriteFence, FrozenTable]] = []
+    for material in sorted(materials, key=lambda value: (value.engine, value.id)):
+        engine = context.engines[material.engine]
+        if not callable(getattr(engine, "write_fence", None)) or not callable(
+            getattr(engine, "validate_schema", None)
+        ):
+            raise MigrationRefused(
+                "frozen comparison requires native write fences and schema checks"
+            )
+        native = cast(Fencable, engine)
+        native.validate_schema(material.layout)
+        for table in sorted(material.layout.tables.values()):
+            fence = native.write_fence(table, project_id=context.project_id)
+            state = fence.state()
+            epoch = checked_epochs[material.id]
+            if hold_id in state.retired:
+                raise MigrationRefused("a frozen comparison cannot reuse a retired barrier id")
+            if not state.complete or state.epoch != epoch:
+                raise MigrationRefused(
+                    "a frozen comparison table is not at the expected write generation"
+                )
+            planned.append(
+                (
+                    fence,
+                    FrozenTable(
+                        material.engine,
+                        material.id,
+                        table,
+                        state.identity,
+                        context.project_id,
+                        epoch,
+                        hold_id,
+                    ),
+                )
+            )
+    started = perf_counter_ns()
+    for fence, wanted in planned:
+        # Constraint presence alone does not drain an old ClickHouse INSERT. Always execute the
+        # native drain, including when resuming an already installed hold.
+        _matches(fence.freeze(hold_id), wanted)
+    comparison = verify(context, group, request=request, chunk_rows=chunk_rows, at=at)
+    for fence, wanted in planned:
+        _matches(fence.state(), wanted)
+    return FrozenVerifyReport(
+        comparison,
+        tuple(wanted for _, wanted in planned),
+        (perf_counter_ns() - started) // 1_000_000,
+    )

sde/generation.py

@@ -0,0 +1,131 @@
+"""Wire vocabulary for generation-bearing maps, independent of drivers and placement classes."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Protocol, cast
+
+from .errors import MigrationRefused
+
+if TYPE_CHECKING:
+    from .model import LogicalModel
+    from .physical import PhysicalFinding
+    from .placement import PhysicalLayout, PlacementMap
+    from .session import Engine
+    from .write_fence import WriteFence
+
+
+class Fencable(Protocol):
+    def write_fence(self, table: str, *, project_id: str) -> WriteFence: ...
+    def validate_schema(
+        self, layout: PhysicalLayout, *, keys: Mapping[str, Sequence[str]] | None = None
+    ) -> tuple[PhysicalFinding, ...]: ...
+
+
+DRAIN_TABLE = "__sde_fence_drains"
+EPOCH_COLUMN = "__sde_write_epoch"
+MAX_EPOCH = 9_007_199_254_740_991
+GENERATIONS_SINCE = 4
+"""The placement map contract that introduced ``project_id`` and per-group ``write_epoch``."""
+
+
+def check_epoch(epoch: int) -> int:
+    if type(epoch) not in (int, float) or not 1 <= epoch <= MAX_EPOCH or int(epoch) != epoch:
+        raise MigrationRefused("write epoch must be a positive safe integer")
+    return int(epoch)
+
+
+def json_numbers(value: Any) -> Any:
+    """JSON has one number type. Match integral JS numbers before checking/signing a v4 map.
+
+    Canonical encoding itself remains strict. Nonintegral/nonfinite numbers survive this pass and
+    are refused by the map's canonical boundary. Earlier map contracts retain their old behavior.
+    """
+    if isinstance(value, float) and value.is_integer():
+        return int(value)
+    if isinstance(value, dict):
+        return {key: json_numbers(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [json_numbers(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(json_numbers(item) for item in value)
+    return value
+
+
+def check_map_project(placement: PlacementMap, project_id: str | None) -> str | None:
+    if placement.contract < 4:
+        return None
+    if project_id is None or project_id != placement.project_id:
+        raise MigrationRefused(
+            "this generation-bearing map needs its locally configured project_id; "
+            "do not learn that identity from the supplied map"
+        )
+    if placement.fingerprint is None:
+        raise MigrationRefused("generation-bearing sessions need an immutable loaded placement map")
+    return project_id
+
+
+def validate_generations(
+    model: LogicalModel,
+    placement: PlacementMap,
+    engines: Mapping[str, Engine],
+    project_id: str | None,
+) -> tuple[PhysicalFinding, ...]:
+    """Check generations and columns; return how tables differ from the declared physical design.
+
+    The physical design is reported rather than refused: a running application must not stop
+    because a table's sort key or index differs from the map (requirement 3.6). Columns, types
+    and generations still refuse.
+    """
+    if placement.contract < 4:
+        return ()
+    findings: list[PhysicalFinding] = []
+    local_project = check_map_project(placement, project_id)
+    assert local_project is not None
+    if model.version != placement.model_version:
+        raise MigrationRefused("the generation-bearing map names another session model")
+    for name in sorted(placement.groups):
+        spot = placement.groups[name]
+        for material in spot.all():
+            factory = getattr(engines[material.engine], "write_fence", None)
+            if not callable(factory) or not callable(
+                getattr(engines[material.engine], "validate_schema", None)
+            ):
+                raise MigrationRefused(
+                    f"engine {material.engine} does not implement write generations"
+                )
+            keys = {entity: model.entity(entity).key for entity in material.layout.tables}
+            findings.extend(
+                cast(Fencable, engines[material.engine]).validate_schema(
+                    material.layout, keys=keys
+                )
+            )
+            for table in sorted(material.layout.tables.values()):
+                state = (
+                    cast(Fencable, engines[material.engine])
+                    .write_fence(table, project_id=local_project)
+                    .state()
+                )
+                if not state.complete or state.epoch != spot.write_epoch:
+                    raise MigrationRefused(
+                        f"the write generation for {name} is not active in {material.engine}; "
+                        "provision the signed map or load the current map before opening a session"
+                    )
+    return tuple(findings)
+
+
+def stamp_values(
+    placement: PlacementMap, group: str, values: Mapping[str, Any]
+) -> Mapping[str, Any]:
+    epoch = placement.placement_of(group).write_epoch
+    if epoch is None:
+        return values
+    if EPOCH_COLUMN in values:
+        raise MigrationRefused("the write-epoch column is reserved for the SDK")
+    return {**values, EPOCH_COLUMN: epoch}
+
+
+def logical_row(placement: PlacementMap, row: Any) -> Any:
+    if placement.contract >= 4 and isinstance(row, dict):
+        return {key: value for key, value in row.items() if key != EPOCH_COLUMN}
+    return row

sde/groups.py

@@ -0,0 +1,97 @@
+"""Colocation groups: the unit of placement.
+
+Entities that are queried together, or that must change together, live in the same engine. The graph
+has an edge for every relation and for every declared atomicity, and a group is a connected
+component of it.
+
+This looks like a limitation and is the opposite. A join across two engines means pulling both sides
+over the network and joining in the client's process: slow, memory-hungry, and the single easiest
+way for a young product to embarrass itself. Making colocation a constraint turns that problem into
+something the planner simply respects, and the consistency contract falls straight out of it - one
+group, one engine, that engine's transaction semantics, and no distributed transactions anywhere.
+
+The obvious worry is that every relation being an edge collapses a normalised model into one group,
+leaving nothing to place. In practice it does not, and the reason is worth understanding because it
+is the product's whole thesis. Take a typical application: ``User``, ``Order``, ``OrderLine``,
+``Product``, ``Event``. The first four are related and become one group. ``Event`` references
+nothing and becomes its own. That is exactly the split that matters: the transactional core belongs
+in a row store, the event stream belongs in a column store, and the entities nobody joins are
+precisely the ones that were sitting in the wrong engine all along.
+
+A client who wants two related entities in different engines can have that, and finds out about the
+cost honestly: the relation stops being traversable, and the error at model-planning time says which
+entities would have to share a group for the query to be possible.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .model import LogicalModel
+
+__all__ = ["Group", "colocation_groups", "group_of"]
+
+
+@dataclass(frozen=True)
+class Group:
+    """A set of entities placed together.
+
+    ``name`` is the alphabetically first member. It exists so that logs, proposals and error
+    messages can say ``group "order"`` instead of a hash, and it is only meaningful within one model
+    version - change the membership and you have changed the model, which changes its version.
+    """
+
+    name: str
+    members: tuple[str, ...]
+
+    def __contains__(self, entity: str) -> bool:
+        return entity in self.members
+
+
+def colocation_groups(model: LogicalModel) -> tuple[Group, ...]:
+    """Connected components of the colocation graph, deterministically ordered.
+
+    Determinism here is not a nicety. The group name reaches the placement map, the telemetry and
+    the planner's decisions, so two runs over the same model have to produce the same names or the
+    control plane sees a model whose groups keep being renamed.
+    """
+    names = sorted(e.name for e in model.entities)
+    parent: dict[str, str] = {n: n for n in names}
+
+    def find(x: str) -> str:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(a: str, b: str) -> None:
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            # Always attach to the alphabetically smaller root, so the representative of a component
+            # does not depend on the order edges were visited in.
+            parent[max(ra, rb)] = min(ra, rb)
+
+    for relation in sorted(model.relations, key=lambda r: (r.source, r.name, r.target)):
+        union(relation.source, relation.target)
+    for atomic in model.atomic:
+        first = atomic[0]
+        for other in atomic[1:]:
+            union(first, other)
+
+    buckets: dict[str, list[str]] = {}
+    for name in names:
+        buckets.setdefault(find(name), []).append(name)
+
+    groups = [
+        Group(name=min(members), members=tuple(sorted(members)))
+        for members in buckets.values()
+    ]
+    groups.sort(key=lambda g: g.name)
+    return tuple(groups)
+
+
+def group_of(groups: tuple[Group, ...], entity: str) -> Group:
+    for group in groups:
+        if entity in group:
+            return group
+    raise KeyError(f"{entity} is not in any group, which means it is not in the model")

sde/hashing.py

@@ -0,0 +1,242 @@
+"""Hiding identifier names from us, and from the model, without hiding them from the planner's job.
+
+A client in a regulated environment may not want us to see that they have a table called
+``patient_diagnosis``. Requirement 11.3 lets them hash entity, field and relation names with a salt
+that never leaves their infrastructure, and requirement 11.4 says the consequence must not be a
+worse placement: no scoring feature may read the *meaning* of a name, so a hashed model has to be
+placed identically to an unhashed one.
+
+That second requirement is the interesting one. It is enforced by a test rather than by review, and
+the test is worth more than a dozen unit tests of the planner, because it fails the moment anyone
+writes ``if "log" in table_name`` anywhere in the decision path.
+
+# Two consequences worth knowing before turning this on
+
+**Hashing is a model change.** A group's name is its alphabetically first member, and hashing
+changes which member that is; operation shape identifiers include the group and entity names, so
+they change too. The canonical IR is different, so ``model_version`` is different, so the placement
+map is different. Switching hashing on or off is therefore not a setting - it is a new model that
+needs a new map, exactly as requirement 1.3 describes. Saying this plainly is cheaper than having a
+client discover it when their existing map is refused.
+
+**Your own tables get opaque names.** The physical layout comes from the map, which is keyed by
+hashed names, so a table in the client's own database ends up called ``e_9c1f2a7b3d40``. For a
+regulated deployment that is the point. For most deployments it is a real cost, and it should be
+weighed rather than accepted by default - which is why this is off unless asked for.
+
+# What is not hidden
+
+Types, cardinalities, latencies, call counts, the shape of the graph. All of that is what the
+planner actually reasons from, and none of it is a name. What the model loses is semantic signal:
+``orders`` tells a language model something that ``e_9c1f2a7b3d40`` does not, so proposals get
+measurably worse (requirement 18.10). The deterministic path is unaffected, which is the whole point
+of it not reading names.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import secrets
+import unicodedata
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from .errors import DeclarationError
+from .model import EntitySpec, FieldSpec, LogicalModel, RelationSpec, assemble
+
+__all__ = ["NameMap", "hash_identifiers", "load_or_create_salt"]
+
+# Twelve hex characters is 48 bits. Collisions matter here - two entities hashing to one name would
+# merge them in the IR - so this is checked rather than assumed: hash_identifiers refuses on
+# collision instead of producing a model with one entity where there were two.
+DIGEST_CHARS = 12
+
+_ENTITY_PREFIX = "e_"
+_FIELD_PREFIX = "f_"
+_RELATION_PREFIX = "r_"
+
+
+def load_or_create_salt(path: Path | None = None) -> bytes:
+    """Read the client's salt, generating it on first use.
+
+    The salt never appears in a model, a telemetry window or anything sent to us: it is the whole
+    mechanism, and a salt we hold is a mechanism we could reverse. Stored with owner-only
+    permissions, because a readable salt in a shared container image is the same as no salt.
+
+    The file is read **verbatim**. An earlier version called ``.strip()`` on it, to be forgiving
+    about a trailing newline in a hand-made file, and that was a serious bug rather than a kindness.
+    ``bytes.strip()`` removes six byte *values* - space, tab, newline, carriage return, ``\x0b`` and
+    ``\x0c`` - and a random 32-byte salt begins or ends with one of them about 5% of the time.
+
+    When it did, the process that generated the salt used all 32 bytes and every process afterwards
+    used the stripped remainder. So one client computed **two different model versions** for one
+    declared model: the map issued for one is refused by the other, and a map that was accepted
+    names different tables. The TypeScript library takes the salt as bytes from its caller and
+    strips nothing, so a Python service and a Node service sharing one salt file disagreed too - the
+    exact failure the byte contract exists to prevent, arriving through the file rather than the
+    encoder.
+
+    A file created with ``echo`` therefore includes its newline, and the salt is those bytes
+    including it. That is consistent and checkable. Being forgiving would mean choosing an encoding,
+    hex or base64, and that changes what is on disk - which changes every name derived from it.
+    """
+    location = path or Path.home() / ".sde" / "salt"
+    if location.exists():
+        salt = location.read_bytes()
+        if len(salt) < 16:
+            raise DeclarationError(
+                f"the salt at {location} is shorter than 16 bytes. A short salt is "
+                "guessable, and a guessable salt means the names are not hidden. Delete the file "
+                "to have a new one generated - but note that a new salt is a new model version "
+                "and needs a new map."
+            )
+        return salt
+
+    location.parent.mkdir(parents=True, exist_ok=True)
+    salt = secrets.token_bytes(32)
+    # Written before the mode is set, then narrowed. os.open with 0o600 would be tighter, and is
+    # what this should become if the file ever holds more than a salt.
+    location.write_bytes(salt)
+    os.chmod(location, 0o600)
+    return salt
+
+
+@dataclass(frozen=True)
+class NameMap:
+    """The translation between what the client wrote and what we see.
+
+    Held only in the client's process. The library needs it because the placement map is keyed by
+    hashed names while the application still says ``session.save("User", ...)`` - so every lookup
+    crosses this boundary, and it is the only place that does.
+    """
+
+    entities: Mapping[str, str]
+    fields: Mapping[str, Mapping[str, str]]
+    relations: Mapping[str, Mapping[str, str]]
+
+    def entity(self, name: str) -> str:
+        try:
+            return self.entities[name]
+        except KeyError:
+            raise DeclarationError(
+                f"{name!r} is not in this model, so it has no hashed name. Either it was never "
+                "declared, or the model was rebuilt without it."
+            ) from None
+
+    def field(self, entity: str, name: str) -> str:
+        return self.fields[entity][name]
+
+    def relation(self, entity: str, name: str) -> str:
+        return self.relations[entity][name]
+
+
+def _digest(salt: bytes, prefix: str, *parts: str) -> str:
+    """HMAC over the parts, NFC-normalised and joined by a separator no identifier can contain.
+
+    Two details, both of which a port can get wrong silently, so both are pinned in the format
+    contract.
+
+    **NFC first.** The canonical encoder normalises before it emits bytes, which is why two
+    libraries that declare ``Zamówienie`` in different normal forms compute the *same* model
+    version. Hashing a name before normalising it would throw that away: the same identifier written
+    two ways would give two digests, two model versions, and a placement map issued for one service
+    that the other refuses. Nothing about that is visible in ASCII, so it would have shipped and
+    then failed for a client whose entity names are not English.
+
+    **The separator, not concatenation.** Hashing ``("User", "id")`` as ``"Userid"`` would collide
+    with ``("Use", "rid")``. Unlikely is not a guarantee when the consequence is two fields becoming
+    one column. U+0000 cannot occur in an identifier, so the join is unambiguous.
+
+    The prefix is deliberately outside the HMAC. It labels the digest for a human reading a table
+    name; it carries no secret and adding it to the message would only make the derivation harder to
+    reproduce.
+    """
+    message = "\x00".join(unicodedata.normalize("NFC", part) for part in parts)
+    digest = hmac.new(salt, message.encode("utf-8"), hashlib.sha256).hexdigest()
+    return prefix + digest[:DIGEST_CHARS]
+
+
+def hash_identifiers(model: LogicalModel, salt: bytes) -> tuple[LogicalModel, NameMap]:
+    """Return an equivalent model with every identifier replaced by a keyed digest.
+
+    Field and relation names are hashed *with their entity* in the input, so the same field name on
+    two entities produces two different digests. That is not paranoia: leaving them independent
+    would let us learn that two entities share a field called ``email`` even though we cannot read
+    the name, which is exactly the kind of structural leak hashing is meant to close.
+    """
+    if len(salt) < 16:
+        raise DeclarationError("the salt must be at least 16 bytes")
+
+    entity_names: dict[str, str] = {}
+    for spec in model.entities:
+        hashed = _digest(salt, _ENTITY_PREFIX, spec.name)
+        if hashed in entity_names.values():
+            clash = next(k for k, v in entity_names.items() if v == hashed)
+            raise DeclarationError(
+                f"{spec.name!r} and {clash!r} hash to the same name. Refused rather than merged: a "
+                "model with one entity where there were two would place both in one engine and "
+                "write both into one table. Change the salt."
+            )
+        entity_names[spec.name] = hashed
+
+    field_names: dict[str, dict[str, str]] = {}
+    relation_names: dict[str, dict[str, str]] = {}
+
+    entities: list[EntitySpec] = []
+    for spec in model.entities:
+        mapping: dict[str, str] = {}
+        for spec_field in spec.fields:
+            mapping[spec_field.name] = _digest(
+                salt, _FIELD_PREFIX, spec.name, spec_field.name
+            )
+        if len(set(mapping.values())) != len(mapping):
+            raise DeclarationError(
+                f"two fields of {spec.name!r} hash to the same name. Refused rather than merged. "
+                "Change the salt."
+            )
+        field_names[spec.name] = mapping
+
+        entities.append(
+            EntitySpec(
+                name=entity_names[spec.name],
+                fields=tuple(
+                    FieldSpec(name=mapping[f.name], type=f.type, nullable=f.nullable)
+                    for f in spec.fields
+                ),
+                key=tuple(mapping[k] for k in spec.key),
+                pii=tuple(mapping[p] for p in spec.pii),
+                # Residency is not an identifier. It is a jurisdiction, it is a hard constraint on
+                # placement, and hashing it would make the constraint unenforceable.
+                residency=spec.residency,
+            )
+        )
+
+    relations: list[RelationSpec] = []
+    for relation in model.relations:
+        hashed = _digest(salt, _RELATION_PREFIX, relation.source, relation.name)
+        relation_names.setdefault(relation.source, {})[relation.name] = hashed
+        relations.append(
+            RelationSpec(
+                name=hashed,
+                source=entity_names[relation.source],
+                target=entity_names[relation.target],
+            )
+        )
+
+    atomic = tuple(
+        tuple(sorted(entity_names[member] for member in group)) for group in model.atomic
+    )
+
+    hashed_model = assemble(
+        entities=tuple(entities),
+        relations=tuple(relations),
+        atomic=tuple(sorted(atomic)),
+        # The cost ceiling is a number and a currency, not an identifier.
+        cost_ceiling=model.cost_ceiling,
+    )
+    return hashed_model, NameMap(
+        entities=entity_names, fields=field_names, relations=relation_names
+    )