smart-data-engine-sdk 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sde/__init__.py +226 -0
- sde/canonical.py +141 -0
- sde/capabilities.py +62 -0
- sde/engines/__init__.py +0 -0
- sde/engines/clickhouse.py +689 -0
- sde/engines/orderbook.py +454 -0
- sde/engines/postgres.py +672 -0
- sde/entity.py +170 -0
- sde/errors.py +88 -0
- sde/explain.py +300 -0
- sde/groups.py +97 -0
- sde/hashing.py +242 -0
- sde/infer.py +461 -0
- sde/internal.py +90 -0
- sde/layout.py +660 -0
- sde/logging.py +132 -0
- sde/migration.py +820 -0
- sde/model.py +482 -0
- sde/placement.py +818 -0
- sde/py.typed +0 -0
- sde/routing.py +85 -0
- sde/schema.py +370 -0
- sde/session.py +507 -0
- sde/shapes.py +153 -0
- sde/telemetry.py +736 -0
- sde/testing/__init__.py +14 -0
- sde/testing/loader.py +175 -0
- sde/testing/memory.py +318 -0
- sde/types.py +228 -0
- sde/watermark.py +222 -0
- smart_data_engine_sdk-0.1.0.dev0.dist-info/METADATA +152 -0
- smart_data_engine_sdk-0.1.0.dev0.dist-info/RECORD +35 -0
- smart_data_engine_sdk-0.1.0.dev0.dist-info/WHEEL +4 -0
- smart_data_engine_sdk-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
- smart_data_engine_sdk-0.1.0.dev0.dist-info/licenses/NOTICE +13 -0
sde/types.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The neutral type vocabulary, and the mapping from Python's types onto it.
|
|
2
|
+
|
|
3
|
+
Why a vocabulary at all: ``Decimal`` in Python and ``BigDecimal`` in Java have to land on the same
|
|
4
|
+
bytes in the canonical IR, or the same model gets two versions and the control plane sees two
|
|
5
|
+
models. So no language's own type names ever reach the IR. Each library maps its host language onto
|
|
6
|
+
this closed set, and the mapping is part of the published contract rather than an implementation
|
|
7
|
+
detail.
|
|
8
|
+
|
|
9
|
+
The set:
|
|
10
|
+
|
|
11
|
+
bool int32 int64 float32 float64 decimal(p,s) string bytes uuid date timestamp timestamptz json
|
|
12
|
+
|
|
13
|
+
``decimal`` is the only parameterised member, written ``decimal(12,2)`` - precision then scale, no
|
|
14
|
+
spaces. It is parameterised because a decimal without precision is not a storable type in any of the
|
|
15
|
+
engines we place data in, and asking the engine to pick would make the physical schema depend on
|
|
16
|
+
something the model did not say.
|
|
17
|
+
|
|
18
|
+
**Floats are in the vocabulary, and this needed a correction to the specification.** The first draft
|
|
19
|
+
of the format contract said "no floating point anywhere", conflating two different things. Canonical
|
|
20
|
+
*encoding* must contain no float literals, because their textual form differs between languages -
|
|
21
|
+
that rule stands and ``canonical.py`` enforces it. But a *field* may perfectly well be a float:
|
|
22
|
+
sensor readings are the obvious case, and IoT at scale is one of the segments this product is aimed
|
|
23
|
+
at. A field of type ``float64`` is recorded in the IR as the string ``"float64"``, so there is no
|
|
24
|
+
float literal involved and no conflict. Forbidding the type would have meant telling a client to
|
|
25
|
+
store temperatures as decimals, which is worse engineering than the rule was worth.
|
|
26
|
+
|
|
27
|
+
Two mappings are defaults rather than one-to-one, and both are documented here because a silent
|
|
28
|
+
default in a type mapping is the kind of thing that surfaces two years later as a timezone bug:
|
|
29
|
+
|
|
30
|
+
* ``int`` maps to ``int64``. Python's integers are unbounded, so any choice is a narrowing; 64 bits
|
|
31
|
+
is what every target engine has a native type for. Use :data:`Int32` to say so explicitly.
|
|
32
|
+
* ``datetime`` maps to ``timestamptz``. A naive timestamp is a latent bug in a system that may move
|
|
33
|
+
data between engines and regions, so the safe reading is the default and the narrow one has to be
|
|
34
|
+
asked for by name with :data:`Timestamp`.
|
|
35
|
+
|
|
36
|
+
Anything with no mapping - a bare ``Decimal``, a custom class, ``complex`` - is a
|
|
37
|
+
:class:`~sde.errors.DeclarationError`. Guessing would produce a physical schema the model did not
|
|
38
|
+
ask for.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import datetime as _dt
|
|
44
|
+
import decimal as _decimal
|
|
45
|
+
import types as _pytypes
|
|
46
|
+
import typing
|
|
47
|
+
import uuid as _uuid
|
|
48
|
+
from dataclasses import dataclass
|
|
49
|
+
from typing import Annotated, Any, Final, get_args, get_origin
|
|
50
|
+
|
|
51
|
+
from .errors import DeclarationError
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"NEUTRAL_TYPES",
|
|
55
|
+
"Float32",
|
|
56
|
+
"Int32",
|
|
57
|
+
"Json",
|
|
58
|
+
"Timestamp",
|
|
59
|
+
"precision",
|
|
60
|
+
"resolve_type",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
NEUTRAL_TYPES: Final[frozenset[str]] = frozenset(
|
|
64
|
+
{
|
|
65
|
+
"bool",
|
|
66
|
+
"int32",
|
|
67
|
+
"int64",
|
|
68
|
+
"float32",
|
|
69
|
+
"float64",
|
|
70
|
+
"string",
|
|
71
|
+
"bytes",
|
|
72
|
+
"uuid",
|
|
73
|
+
"date",
|
|
74
|
+
"timestamp",
|
|
75
|
+
"timestamptz",
|
|
76
|
+
"json",
|
|
77
|
+
# decimal is parameterised and validated by pattern, not by membership here.
|
|
78
|
+
}
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class _Marker:
|
|
84
|
+
"""Base for the annotations that disambiguate a mapping."""
|
|
85
|
+
|
|
86
|
+
kind: str
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class _Precision(_Marker):
|
|
91
|
+
digits: int
|
|
92
|
+
scale: int
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def precision(digits: int, scale: int) -> _Precision:
|
|
96
|
+
"""Annotate a ``Decimal`` field with its precision and scale.
|
|
97
|
+
|
|
98
|
+
``total: Annotated[Decimal, precision(12, 2)]``
|
|
99
|
+
|
|
100
|
+
Both are required. A decimal without them is not a storable type, and letting the engine choose
|
|
101
|
+
would make the physical schema depend on something the model never stated.
|
|
102
|
+
"""
|
|
103
|
+
if digits < 1 or scale < 0 or scale > digits:
|
|
104
|
+
raise DeclarationError(
|
|
105
|
+
f"precision({digits}, {scale}) is not a usable decimal: digits must be at least 1 and "
|
|
106
|
+
"scale must be between 0 and digits"
|
|
107
|
+
)
|
|
108
|
+
return _Precision(kind="precision", digits=digits, scale=scale)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
Int32 = Annotated[int, _Marker(kind="int32")]
|
|
112
|
+
"""A 32-bit integer, said explicitly. Bare ``int`` maps to ``int64``."""
|
|
113
|
+
|
|
114
|
+
Float32 = Annotated[float, _Marker(kind="float32")]
|
|
115
|
+
"""A single-precision float. Bare ``float`` maps to ``float64``."""
|
|
116
|
+
|
|
117
|
+
Timestamp = Annotated[_dt.datetime, _Marker(kind="naive")]
|
|
118
|
+
"""A timestamp without a zone. Bare ``datetime`` maps to ``timestamptz`` on purpose."""
|
|
119
|
+
|
|
120
|
+
Json = Annotated[object, _Marker(kind="json")]
|
|
121
|
+
"""An opaque JSON document. Use when the shape genuinely is not fixed, not to avoid declaring it."""
|
|
122
|
+
|
|
123
|
+
_SIMPLE: Final[dict[Any, str]] = {
|
|
124
|
+
bool: "bool",
|
|
125
|
+
int: "int64",
|
|
126
|
+
float: "float64",
|
|
127
|
+
str: "string",
|
|
128
|
+
bytes: "bytes",
|
|
129
|
+
_uuid.UUID: "uuid",
|
|
130
|
+
_dt.date: "date",
|
|
131
|
+
_dt.datetime: "timestamptz",
|
|
132
|
+
dict: "json",
|
|
133
|
+
list: "json",
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _markers(annotation: object) -> tuple[object, tuple[_Marker, ...]]:
|
|
138
|
+
"""Peel ``Annotated`` and ``Optional`` off an annotation.
|
|
139
|
+
|
|
140
|
+
Returns the bare type plus any markers found. Nullability is handled by the caller, which needs
|
|
141
|
+
to record it separately in the IR rather than as part of the type name.
|
|
142
|
+
"""
|
|
143
|
+
markers: list[_Marker] = []
|
|
144
|
+
current = annotation
|
|
145
|
+
while get_origin(current) is Annotated:
|
|
146
|
+
args = get_args(current)
|
|
147
|
+
current = args[0]
|
|
148
|
+
markers.extend(m for m in args[1:] if isinstance(m, _Marker))
|
|
149
|
+
return current, tuple(markers)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _is_optional(annotation: object) -> tuple[bool, object]:
|
|
153
|
+
"""Peel ``| None`` off, and refuse anything wider.
|
|
154
|
+
|
|
155
|
+
``str | None`` is a nullable string. ``int | str`` is not a field: it has no single physical
|
|
156
|
+
representation, so there is no column we could create for it and no index we could put on it.
|
|
157
|
+
Refusing here rather than falling through to the vocabulary check matters only for the error
|
|
158
|
+
message, and the error message is the whole product for someone who mistyped an annotation.
|
|
159
|
+
"""
|
|
160
|
+
origin = get_origin(annotation)
|
|
161
|
+
if origin is typing.Union or origin is _pytypes.UnionType:
|
|
162
|
+
args = get_args(annotation)
|
|
163
|
+
non_none = [a for a in args if a is not type(None)]
|
|
164
|
+
if len(non_none) != 1:
|
|
165
|
+
raise DeclarationError(
|
|
166
|
+
f"{annotation!r} is a union of several types. A field has one type; a union of two "
|
|
167
|
+
"real types has no single physical representation, so it cannot be placed. Model "
|
|
168
|
+
"it "
|
|
169
|
+
"as separate nullable fields, or as json if the shape really is not fixed."
|
|
170
|
+
)
|
|
171
|
+
return len(non_none) != len(args), non_none[0]
|
|
172
|
+
return False, annotation
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def resolve_type(annotation: object, *, field: str, entity: str) -> tuple[str, bool]:
|
|
176
|
+
"""Map a Python annotation onto ``(neutral_type, nullable)``.
|
|
177
|
+
|
|
178
|
+
Raises :class:`~sde.errors.DeclarationError` naming the field, because the reader has to fix
|
|
179
|
+
their declaration and a traceback into this module tells them nothing.
|
|
180
|
+
"""
|
|
181
|
+
nullable, inner = _is_optional(annotation)
|
|
182
|
+
bare, markers = _markers(inner)
|
|
183
|
+
# Optional may sit inside Annotated as well: Annotated[int | None, ...].
|
|
184
|
+
nested_nullable, bare = _is_optional(bare)
|
|
185
|
+
nullable = nullable or nested_nullable
|
|
186
|
+
|
|
187
|
+
kinds = {m.kind for m in markers}
|
|
188
|
+
|
|
189
|
+
if bare is _decimal.Decimal:
|
|
190
|
+
found = [m for m in markers if isinstance(m, _Precision)]
|
|
191
|
+
if not found:
|
|
192
|
+
raise DeclarationError(
|
|
193
|
+
f"{entity}.{field} is a Decimal without precision. Write "
|
|
194
|
+
f"Annotated[Decimal, precision(digits, scale)] - a decimal without precision is "
|
|
195
|
+
"not "
|
|
196
|
+
"a storable type in any engine we place data in, and choosing for you would make "
|
|
197
|
+
"the physical schema depend on something your model did not say."
|
|
198
|
+
)
|
|
199
|
+
p = found[0]
|
|
200
|
+
return f"decimal({p.digits},{p.scale})", nullable
|
|
201
|
+
|
|
202
|
+
if "json" in kinds:
|
|
203
|
+
return "json", nullable
|
|
204
|
+
if "int32" in kinds:
|
|
205
|
+
return "int32", nullable
|
|
206
|
+
if "float32" in kinds:
|
|
207
|
+
return "float32", nullable
|
|
208
|
+
if "naive" in kinds:
|
|
209
|
+
if bare is not _dt.datetime:
|
|
210
|
+
raise DeclarationError(
|
|
211
|
+
f"{entity}.{field} is annotated as a naive timestamp but is not a datetime"
|
|
212
|
+
)
|
|
213
|
+
return "timestamp", nullable
|
|
214
|
+
|
|
215
|
+
origin = get_origin(bare)
|
|
216
|
+
if origin in (dict, list):
|
|
217
|
+
return "json", nullable
|
|
218
|
+
|
|
219
|
+
mapped = _SIMPLE.get(bare)
|
|
220
|
+
if mapped is not None:
|
|
221
|
+
return mapped, nullable
|
|
222
|
+
|
|
223
|
+
raise DeclarationError(
|
|
224
|
+
f"{entity}.{field} has type {bare!r}, which has no place in the neutral type vocabulary "
|
|
225
|
+
f"({', '.join(sorted(NEUTRAL_TYPES))}, decimal(p,s)). Map it yourself - to a string, to "
|
|
226
|
+
"json, to a decimal with stated precision - so that the choice is visible in the model "
|
|
227
|
+
"instead of being guessed here."
|
|
228
|
+
)
|
sde/watermark.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Refusing a placement map that goes backwards, and the durable state that makes it possible.
|
|
2
|
+
|
|
3
|
+
A signed map for version 3 verifies correctly forever - that is what a signature is. So replacing
|
|
4
|
+
the client's map file with an older signed one loads cleanly, routes writes to the previous
|
|
5
|
+
placement, and **nothing protests**. Today that costs a client a stale schema. Once the migration
|
|
6
|
+
state travels in the map, it costs them writes: a library reverted from dual-write to single-write
|
|
7
|
+
in the middle of a migration drops exactly the rows the migration exists not to drop.
|
|
8
|
+
|
|
9
|
+
Refusing it needs one thing the library has never had: **memory**. Everything else here is a pure
|
|
10
|
+
function of a document, a model and a key, which is why it can be verified by reading it. This
|
|
11
|
+
module is the exception, and each of the three obvious places to keep that memory is worse than the
|
|
12
|
+
one chosen:
|
|
13
|
+
|
|
14
|
+
- **in the process** protects until the first restart, and a restart is when a swapped file is
|
|
15
|
+
read. A protection that lapses exactly when it is needed;
|
|
16
|
+
- **in a file** needs a configured path, and in a container that path is usually ephemeral - so it
|
|
17
|
+
degrades to the first option while continuing to look like the third. The worst property
|
|
18
|
+
available: a guarantee that is present in the code and absent in production;
|
|
19
|
+
- **with us** would mean the library asking our service whether it may start, which is the one
|
|
20
|
+
thing this product promises it will never need to do. Our outage would become the client's.
|
|
21
|
+
|
|
22
|
+
So it lives **in the client's own engines**, in a table this library owns. The library already
|
|
23
|
+
creates tables there; this is one more, it holds no client data, and we still never see a row of it.
|
|
24
|
+
|
|
25
|
+
Four properties, and the first two are what make it safe rather than merely present.
|
|
26
|
+
|
|
27
|
+
**Append-only, and the watermark is `max(map_version)`.** No update, no key enforcement, no
|
|
28
|
+
row-level contention - and therefore identical semantics in PostgreSQL and in ClickHouse, which is
|
|
29
|
+
the engine that has no unique constraint to offer. A stale row can never lower the bar. It also
|
|
30
|
+
leaves an audit trail for free: which map versions this deployment has seen, and when.
|
|
31
|
+
|
|
32
|
+
**Every participating engine is written, and the watermark is the maximum over all of them.**
|
|
33
|
+
Losing an engine cannot lose the protection, and one engine lagging cannot weaken it.
|
|
34
|
+
|
|
35
|
+
**An engine that cannot store it does not participate, and that is reported rather than hidden.**
|
|
36
|
+
The orderbook engine has a schema fixed in its own source and no room for bookkeeping, so a client
|
|
37
|
+
whose only engine is that one has no rollback protection and cannot have any. The honest maximum is
|
|
38
|
+
to say so - :class:`WatermarkCheck` carries it and ``Session`` exposes it, because a protection
|
|
39
|
+
whose state cannot be read is a protection taken on trust.
|
|
40
|
+
|
|
41
|
+
**Only signed maps are checked.** An unsigned map is the client's own document: hand-writing one and
|
|
42
|
+
pointing a library at it is the no-account mode, and their business what they replace it with. A
|
|
43
|
+
signed map is one we issued, which is precisely when we are the authority on what the newest version
|
|
44
|
+
is. In pure no-account mode this module does nothing at all - no table, no query, no cost.
|
|
45
|
+
|
|
46
|
+
The escape hatch is deliberately not a parameter. A legitimate rollback - we issued a bad map -
|
|
47
|
+
means clearing the bookkeeping, and the refusal says how. A parameter called ``allow_rollback``
|
|
48
|
+
would be set once during an incident and left set.
|
|
49
|
+
|
|
50
|
+
**A limitation worth stating.** The watermark is per engine and the format has no field naming which
|
|
51
|
+
stream of maps a document belongs to, so an engine shared by two independent map streams would have
|
|
52
|
+
the higher one refusing the lower. The fix is a separate database per stream, which a shared engine
|
|
53
|
+
wants regardless; inventing a stream identifier would mean a new key in a signed document, which is
|
|
54
|
+
a loosening of the format and a contract bump in every language at once.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
from collections.abc import Mapping
|
|
60
|
+
from dataclasses import dataclass
|
|
61
|
+
from typing import Any, Literal, Protocol, runtime_checkable
|
|
62
|
+
|
|
63
|
+
from .capabilities import satisfies
|
|
64
|
+
from .errors import MapRolledBack
|
|
65
|
+
from .logging import log
|
|
66
|
+
from .placement import WATERMARK_TABLE, PlacementMap
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
"WATERMARK_TABLE",
|
|
70
|
+
"Protection",
|
|
71
|
+
"WatermarkCheck",
|
|
72
|
+
"WatermarkStore",
|
|
73
|
+
"enforce_forward_only",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
Protection = Literal["enforced", "unavailable", "not_applicable"]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@runtime_checkable
|
|
80
|
+
class WatermarkStore(Protocol):
|
|
81
|
+
"""What an engine adapter needs to offer to take part.
|
|
82
|
+
|
|
83
|
+
A separate protocol from :class:`~sde.session.Engine`, and optional. Adding two methods to
|
|
84
|
+
``Engine`` would break every adapter anybody has written against it - including the fakes in
|
|
85
|
+
somebody else's test suite - for a capability one of our own three engines cannot provide
|
|
86
|
+
anyway. So participation is discovered rather than required, and non-participation is a
|
|
87
|
+
reportable state instead of a crash.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
def map_watermark(self) -> int | None: ...
|
|
91
|
+
def record_map_version(self, version: int, *, model_version: str) -> None: ...
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass(frozen=True)
|
|
95
|
+
class WatermarkCheck:
|
|
96
|
+
"""What the check did, in a form a client can assert on.
|
|
97
|
+
|
|
98
|
+
Exposed rather than kept private on purpose. A protection whose state cannot be read is a
|
|
99
|
+
protection taken on trust, and this product's whole argument is that its guarantees are
|
|
100
|
+
checkable by reading the code and now by reading this.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
protection: Protection
|
|
104
|
+
map_version: int
|
|
105
|
+
highest_seen: int | None
|
|
106
|
+
participating: tuple[str, ...]
|
|
107
|
+
unable: tuple[str, ...]
|
|
108
|
+
why: str
|
|
109
|
+
|
|
110
|
+
def as_record(self) -> dict[str, Any]:
|
|
111
|
+
return {
|
|
112
|
+
"protection": self.protection,
|
|
113
|
+
"map_version": self.map_version,
|
|
114
|
+
"highest_seen": self.highest_seen,
|
|
115
|
+
"participating": list(self.participating),
|
|
116
|
+
"unable": list(self.unable),
|
|
117
|
+
"why": self.why,
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _split(engines: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
|
122
|
+
"""Which engines can keep the bookkeeping and which cannot, both sorted.
|
|
123
|
+
|
|
124
|
+
Asked with :func:`sde.capabilities.satisfies` rather than `isinstance`, because a
|
|
125
|
+
runtime_checkable protocol resolves members without consulting `__getattr__` - so a client
|
|
126
|
+
wrapping one of our adapters for metrics or retries would be reported here as an engine with
|
|
127
|
+
nowhere to keep the bookkeeping, and would silently lose rollback protection under a message
|
|
128
|
+
blaming their engine's schema.
|
|
129
|
+
"""
|
|
130
|
+
able = sorted(name for name, engine in engines.items() if satisfies(engine, WatermarkStore))
|
|
131
|
+
unable = sorted(set(engines) - set(able))
|
|
132
|
+
return tuple(able), tuple(unable)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def enforce_forward_only(
|
|
136
|
+
placement: PlacementMap, engines: Mapping[str, Any]
|
|
137
|
+
) -> WatermarkCheck:
|
|
138
|
+
"""Refuse a signed map older than the newest one these engines have seen.
|
|
139
|
+
|
|
140
|
+
Raises :class:`~sde.errors.MapRolledBack`. Equal is allowed - restarting a process against the
|
|
141
|
+
same map is the ordinary case - and only strictly lower is refused.
|
|
142
|
+
"""
|
|
143
|
+
if not placement.signed:
|
|
144
|
+
return WatermarkCheck(
|
|
145
|
+
protection="not_applicable",
|
|
146
|
+
map_version=placement.map_version,
|
|
147
|
+
highest_seen=None,
|
|
148
|
+
participating=(),
|
|
149
|
+
unable=tuple(sorted(engines)),
|
|
150
|
+
why=(
|
|
151
|
+
"this map is unsigned, so it is your own document rather than one we issued. "
|
|
152
|
+
"Replacing it with another is the no-account mode working as documented, and there "
|
|
153
|
+
"is no newest version for us to be the authority on."
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
able, unable = _split(engines)
|
|
158
|
+
if not able:
|
|
159
|
+
check = WatermarkCheck(
|
|
160
|
+
protection="unavailable",
|
|
161
|
+
map_version=placement.map_version,
|
|
162
|
+
highest_seen=None,
|
|
163
|
+
participating=(),
|
|
164
|
+
unable=unable,
|
|
165
|
+
why=(
|
|
166
|
+
f"none of the engines in this map can keep bookkeeping ({list(unable)}), so a map "
|
|
167
|
+
f"that goes backwards cannot be recognised. An engine whose schema is fixed in its "
|
|
168
|
+
f"own source - ours is - has nowhere to put it. Nothing is wrong with your "
|
|
169
|
+
f"configuration; this protection simply does not exist for it."
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
log(
|
|
173
|
+
"sde.map.rollback_unprotected",
|
|
174
|
+
map_version=placement.map_version,
|
|
175
|
+
engines=len(unable),
|
|
176
|
+
)
|
|
177
|
+
return check
|
|
178
|
+
|
|
179
|
+
seen = [store.map_watermark() for store in (engines[name] for name in able)]
|
|
180
|
+
known = [value for value in seen if value is not None]
|
|
181
|
+
highest = max(known) if known else None
|
|
182
|
+
|
|
183
|
+
if highest is not None and placement.map_version < highest:
|
|
184
|
+
raise MapRolledBack(
|
|
185
|
+
f"this map is version {placement.map_version} and version {highest} has already been "
|
|
186
|
+
f"applied against these engines. Refusing to go backwards: an older signed map "
|
|
187
|
+
f"verifies perfectly - that is what a signature is - so nothing else here would notice "
|
|
188
|
+
f"that the file was replaced, and the writes would go to the previous placement. If "
|
|
189
|
+
f"this is deliberate, because the newer map was wrong, clear the bookkeeping: "
|
|
190
|
+
f"DELETE FROM {WATERMARK_TABLE} WHERE map_version > {placement.map_version}; in "
|
|
191
|
+
f"{list(able)}, and on an engine that deletes asynchronously, let the deletion finish "
|
|
192
|
+
f"before restarting. That is a deliberate act with a stated consequence, which is why "
|
|
193
|
+
f"it is not a flag."
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
if highest is None or placement.map_version > highest:
|
|
197
|
+
# Written only when it moves. Recording every start would grow the table by one row per
|
|
198
|
+
# process restart, and the watermark would say nothing more than it does now.
|
|
199
|
+
for name in able:
|
|
200
|
+
store: WatermarkStore = engines[name]
|
|
201
|
+
store.record_map_version(
|
|
202
|
+
placement.map_version, model_version=placement.model_version
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
log(
|
|
206
|
+
"sde.map.forward_only",
|
|
207
|
+
map_version=placement.map_version,
|
|
208
|
+
highest_seen=highest,
|
|
209
|
+
engines=len(able),
|
|
210
|
+
)
|
|
211
|
+
return WatermarkCheck(
|
|
212
|
+
protection="enforced",
|
|
213
|
+
map_version=placement.map_version,
|
|
214
|
+
highest_seen=highest,
|
|
215
|
+
participating=able,
|
|
216
|
+
unable=unable,
|
|
217
|
+
why=(
|
|
218
|
+
f"the highest map version applied against these engines is "
|
|
219
|
+
f"{highest if highest is not None else placement.map_version}, kept in "
|
|
220
|
+
f"{WATERMARK_TABLE} in {list(able)}. A map older than that is refused."
|
|
221
|
+
),
|
|
222
|
+
)
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: smart-data-engine-sdk
|
|
3
|
+
Version: 0.1.0.dev0
|
|
4
|
+
Summary: Smart Data Engine client library: declare a data model, we place it and move it
|
|
5
|
+
Project-URL: Homepage, https://github.com/Smart-Data-Engines/smart-data-engine-sdk
|
|
6
|
+
Project-URL: Repository, https://github.com/Smart-Data-Engines/smart-data-engine-sdk
|
|
7
|
+
Project-URL: Issues, https://github.com/Smart-Data-Engines/smart-data-engine-sdk/issues
|
|
8
|
+
Project-URL: Format contract, https://github.com/Smart-Data-Engines/smart-data-engine-sdk/blob/main/docs/format-contract.md
|
|
9
|
+
Project-URL: Failure semantics, https://github.com/Smart-Data-Engines/smart-data-engine-sdk/blob/main/docs/failure-semantics.md
|
|
10
|
+
Author: Smart Data Engines
|
|
11
|
+
License-Expression: Apache-2.0
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
License-File: NOTICE
|
|
14
|
+
Keywords: clickhouse,database,placement,polystore,postgresql
|
|
15
|
+
Classifier: Development Status :: 3 - Alpha
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: <3.14,>=3.11
|
|
23
|
+
Provides-Extra: clickhouse
|
|
24
|
+
Requires-Dist: clickhouse-connect>=0.7; extra == 'clickhouse'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
28
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
29
|
+
Provides-Extra: postgres
|
|
30
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'postgres'
|
|
31
|
+
Provides-Extra: signed
|
|
32
|
+
Requires-Dist: cryptography>=42; extra == 'signed'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# sde — Smart Data Engine client library for Python
|
|
36
|
+
|
|
37
|
+
Declare your data model. We decide which database engine each part of it lives in, how it is laid out
|
|
38
|
+
there, and when it should move — and move it while your application keeps running.
|
|
39
|
+
|
|
40
|
+
Your code never names a table or an engine. That absence is the point: it is what lets the physical
|
|
41
|
+
schema change underneath you without touching a line of your code.
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from datetime import datetime
|
|
45
|
+
from decimal import Decimal
|
|
46
|
+
from typing import Annotated
|
|
47
|
+
from uuid import UUID
|
|
48
|
+
import sde
|
|
49
|
+
|
|
50
|
+
@sde.entity
|
|
51
|
+
class User:
|
|
52
|
+
id: UUID
|
|
53
|
+
email: str
|
|
54
|
+
|
|
55
|
+
class Meta:
|
|
56
|
+
pii = ["email"]
|
|
57
|
+
|
|
58
|
+
@sde.entity
|
|
59
|
+
class Order:
|
|
60
|
+
id: UUID
|
|
61
|
+
user: sde.Ref[User]
|
|
62
|
+
total: Annotated[Decimal, sde.precision(12, 2)]
|
|
63
|
+
created_at: datetime
|
|
64
|
+
|
|
65
|
+
class Meta:
|
|
66
|
+
residency = "EU"
|
|
67
|
+
|
|
68
|
+
model = sde.build_model()
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## What you declare, and what you do not
|
|
72
|
+
|
|
73
|
+
You declare entities, relations, and four invariants. Everything else about storage is ours to
|
|
74
|
+
decide.
|
|
75
|
+
|
|
76
|
+
The four exist because no amount of watching traffic reveals them:
|
|
77
|
+
|
|
78
|
+
| Declaration | Why traffic cannot tell us |
|
|
79
|
+
|---|---|
|
|
80
|
+
| `atomic_with` | that two entities must commit together is a business rule, not a pattern |
|
|
81
|
+
| `residency` | where data may legally live is not visible in a query |
|
|
82
|
+
| `pii` | which column is personal data determines retention and what may be denormalised |
|
|
83
|
+
| `cost_ceiling` | your budget is not in your workload |
|
|
84
|
+
|
|
85
|
+
Anything beyond those four and the list of engines you have available would be us handing work back
|
|
86
|
+
to you, which is the opposite of what this is for.
|
|
87
|
+
|
|
88
|
+
## Two guarantees, and how to check them yourself
|
|
89
|
+
|
|
90
|
+
**We are not in your data path.** The library connects to your engines directly and answers every
|
|
91
|
+
operation from a locally cached placement map. Our service being down does not make your application
|
|
92
|
+
down. There is no configuration for this — there is no code path that routes your queries through us,
|
|
93
|
+
which you can confirm by reading `routing.py`: it is a dictionary lookup and three conditions.
|
|
94
|
+
|
|
95
|
+
**We never see a row.** Telemetry carries operation *shapes* and counts, never values, and a shape is
|
|
96
|
+
assembled from the structure of the call rather than from a query string — so there is nowhere for a
|
|
97
|
+
value to come from. This is the reason the library is open source: the guarantee is checkable instead
|
|
98
|
+
of promised.
|
|
99
|
+
|
|
100
|
+
## It works without an account
|
|
101
|
+
|
|
102
|
+
Write a placement map by hand, point the library at it, and everything runs: no key, no network, no
|
|
103
|
+
account.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
placement = sde.load_map(json.load(open("placement.json")), model=model)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
An unsigned map is valid. A *signed* map with no key to verify it is not, because a signature is a
|
|
110
|
+
claim that it came from us and an unverifiable claim is worse than no claim. This mode is supported
|
|
111
|
+
and tested, not tolerated — it is also the honest answer to what happens if you stop paying us.
|
|
112
|
+
|
|
113
|
+
## Install
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install smart-data-engine-sdk # core, no dependencies at all
|
|
117
|
+
pip install 'smart-data-engine-sdk[signed]' # verify maps we signed
|
|
118
|
+
pip install 'smart-data-engine-sdk[postgres]' # PostgreSQL engine driver
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
The core has no runtime dependencies. This library goes into your application, so every dependency
|
|
122
|
+
would be one you inherit and a version conflict you might have to resolve.
|
|
123
|
+
|
|
124
|
+
**The distribution is `smart-data-engine-sdk` and the import is `sde`, and neither half of that is a
|
|
125
|
+
preference.** `sde` alone is taken on PyPI by somebody else, and renaming the import to a
|
|
126
|
+
squatting-adjacent misspelling is worse for you than a distribution name that differs from it. The
|
|
127
|
+
obvious distribution name, `smart-data-engine`, is *refused by PyPI*: it answers 400, "the name is
|
|
128
|
+
too similar to an existing project". The project is `smartdata-engine`, registered by somebody else
|
|
129
|
+
with zero releases — PyPI compares names after stripping `._-` and folding `l`/`i` to `1` and `o` to
|
|
130
|
+
`0`, and both reduce to `smartdataeng1ne`. The suffix changes the reduction, and it matches the
|
|
131
|
+
repository name, which is the consistency worth having regardless.
|
|
132
|
+
|
|
133
|
+
**`smart-data-engine-sdk` is unclaimed on PyPI, so those three commands install nothing today.** That
|
|
134
|
+
is the honest state and not a typo: the library is not published yet. Read it as a warning rather
|
|
135
|
+
than a footnote, because three error messages inside this library point at the same commands - if
|
|
136
|
+
someone else registers the name before we do, a person who is already debugging gets told, by code
|
|
137
|
+
they had decided to trust, to install a stranger's package.
|
|
138
|
+
[`../docs/publishing.md`](../docs/publishing.md) is what closes this.
|
|
139
|
+
|
|
140
|
+
## Conformance
|
|
141
|
+
|
|
142
|
+
Everything in `canonical.py`, `model.py`, `shapes.py` and `routing.py` implements a cross-language
|
|
143
|
+
contract, pinned by vectors in `../conformance/vectors`. The Python, TypeScript, Java and Rust
|
|
144
|
+
libraries run the same vectors in their own test runners, so a divergence is a red test for whoever
|
|
145
|
+
caused it rather than an operation written to the wrong engine in production.
|
|
146
|
+
|
|
147
|
+
If you are porting this to another language, `docs/format-contract.md` is meant to be sufficient on
|
|
148
|
+
its own. If it is not, that is a bug in the document.
|
|
149
|
+
|
|
150
|
+
## Licence
|
|
151
|
+
|
|
152
|
+
Apache-2.0.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
sde/__init__.py,sha256=IFbRKfe4CMEU7XIFKTwVG2Gt9NNxvj7jPj8jDDR3gVc,5510
|
|
2
|
+
sde/canonical.py,sha256=NoQDuycFTikcRGjrnqARlyvLpCcmixPKwdWszAkSDbg,5986
|
|
3
|
+
sde/capabilities.py,sha256=cT7RIrgfvCi6uhq16gkwMVynhJxegGtgUzONHQ3SZxU,3188
|
|
4
|
+
sde/entity.py,sha256=epPnf-cM-_c1SvI0IKnNVMqVlvau32wX2kmgWtsypy4,6549
|
|
5
|
+
sde/errors.py,sha256=05MgWvXZOuClhQjNbMpXpiW3rG55wl2_PRU6ZLOGEIU,3879
|
|
6
|
+
sde/explain.py,sha256=BrTA6rbZVhkkeLt81NqDEwdFe7wxVEbBhtlkvKkSnjI,14917
|
|
7
|
+
sde/groups.py,sha256=1g2IERLl2Dt--rMP0Y7suIvzjXNxaOYYn7svhtybK8U,4120
|
|
8
|
+
sde/hashing.py,sha256=8SJ-yvTW6SVdG2FPLoWyaIg8yBKATsdN3VX1pY20Ft8,11562
|
|
9
|
+
sde/infer.py,sha256=myFzi9dgCPnKwzVutOFCng_Gtx5PirkWZpjUm8p0sVc,19372
|
|
10
|
+
sde/internal.py,sha256=3nadKOi-p263hhC_AT8pFfcfxFoENqXR7GFZ5-JpVvw,3678
|
|
11
|
+
sde/layout.py,sha256=1erdR6hJLUjvKVTRxynuzX4p78QyU-Ol9T33BJ61Et4,32924
|
|
12
|
+
sde/logging.py,sha256=R95aqmYVxVU-JOzrbAa8rSyF1HAwP3em7GT74q5LKVs,7620
|
|
13
|
+
sde/migration.py,sha256=jWOuoGpoo43kR8uEcF0qGUoeZtyR9wM7ieY3bECgKzo,37845
|
|
14
|
+
sde/model.py,sha256=zdbrQSRn-UHDuTpoyVqav3e-SKf1L0mv-6uoPVqtPcs,20602
|
|
15
|
+
sde/placement.py,sha256=LetpAUqtu6jmze0sAueuH4Pr1L39jgeLH8f-XKLiJLc,40276
|
|
16
|
+
sde/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
sde/routing.py,sha256=_tvNgpT3F8sTp1TQjVQPjB9dei4QlfFncdTHy1N7ePc,3195
|
|
18
|
+
sde/schema.py,sha256=58c4zfj6xaqXj0UbqPznjmMozr5jYTo8qGoSVj_aklI,19283
|
|
19
|
+
sde/session.py,sha256=1m9WIhHEH9We0gpzFxHInfgFP5VhJsAYoaLAd7SgfO8,25995
|
|
20
|
+
sde/shapes.py,sha256=d_mfam5U4JQhZQ1obihOrYWEBsPqStY3y6wMIGLYi1g,6521
|
|
21
|
+
sde/telemetry.py,sha256=jLXxcDZL4exGxzRZNMuS0xHg5RdD46JbWjo_kmXBlDo,33770
|
|
22
|
+
sde/types.py,sha256=S3YGEN_uYL0ve2ASwzD3yZadeGFuC3l1bvxfAnJQBLE,8960
|
|
23
|
+
sde/watermark.py,sha256=N0Ljs9oiNveV6wboQRZc1JyXtJrepNcBEUN7BGL3PTA,10604
|
|
24
|
+
sde/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
+
sde/engines/clickhouse.py,sha256=9nBNyO1jPFOgVjPqrETaMJuAJo97E1yro4sRTY8F4mQ,35972
|
|
26
|
+
sde/engines/orderbook.py,sha256=UCie44DHDRHwzpxGYFoaQKVsxYEBdJIihZCoWeWpX9o,22567
|
|
27
|
+
sde/engines/postgres.py,sha256=jhC8-7M-FnhXdgG4QXGASKXXokHh2e0sMQe_8CfrMmE,34226
|
|
28
|
+
sde/testing/__init__.py,sha256=T2NEZSj_YVQdpw0-CmjQOiMCHbPZIDRuyUMuwTUIJTY,670
|
|
29
|
+
sde/testing/loader.py,sha256=Bdd9X3ZgEadnkvBaU9LQJiygWZoJTtIxVjmLA4Bbvgs,8889
|
|
30
|
+
sde/testing/memory.py,sha256=FzEkQhPbxjgtszYDXOUUvM5yEGq3nx1bZabuhGINDJk,14900
|
|
31
|
+
smart_data_engine_sdk-0.1.0.dev0.dist-info/METADATA,sha256=iROaVs982_WPrlIHJTWHUI261x2EDZ28ZESuqbyr6t4,6747
|
|
32
|
+
smart_data_engine_sdk-0.1.0.dev0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
33
|
+
smart_data_engine_sdk-0.1.0.dev0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
34
|
+
smart_data_engine_sdk-0.1.0.dev0.dist-info/licenses/NOTICE,sha256=J4n4FX0yCbPCA7zpguXbUyA9R6XmrWCyLqca3Hh79u4,583
|
|
35
|
+
smart_data_engine_sdk-0.1.0.dev0.dist-info/RECORD,,
|