smart-data-engine-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sde/__init__.py +318 -0
- sde/_cutover_project.py +179 -0
- sde/_local_state.py +188 -0
- sde/_operator_deadline.py +50 -0
- sde/_usage.py +314 -0
- sde/bulk.py +79 -0
- sde/canonical.py +141 -0
- sde/capabilities.py +62 -0
- sde/cutover.py +286 -0
- sde/engines/__init__.py +0 -0
- sde/engines/_clickhouse_connection.py +224 -0
- sde/engines/_index_build.py +294 -0
- sde/engines/_operator.py +394 -0
- sde/engines/_staging.py +222 -0
- sde/engines/_storage.py +22 -0
- sde/engines/_write_fences.py +271 -0
- sde/engines/clickhouse.py +1115 -0
- sde/engines/orderbook.py +457 -0
- sde/engines/postgres.py +967 -0
- sde/entity.py +170 -0
- sde/errors.py +103 -0
- sde/explain.py +300 -0
- sde/frozen_verification.py +152 -0
- sde/generation.py +131 -0
- sde/groups.py +97 -0
- sde/hashing.py +242 -0
- sde/index_build.py +313 -0
- sde/index_operator.py +347 -0
- sde/infer.py +461 -0
- sde/inspection.py +62 -0
- sde/internal.py +90 -0
- sde/layout.py +669 -0
- sde/local_cutover.py +801 -0
- sde/logging.py +143 -0
- sde/migration.py +856 -0
- sde/model.py +482 -0
- sde/physical.py +531 -0
- sde/placement.py +1010 -0
- sde/provisioning.py +63 -0
- sde/py.typed +0 -0
- sde/query.py +521 -0
- sde/routing.py +85 -0
- sde/schema.py +466 -0
- sde/session.py +993 -0
- sde/shapes.py +153 -0
- sde/staging.py +264 -0
- sde/staging_operator.py +393 -0
- sde/telemetry.py +1087 -0
- sde/testing/__init__.py +14 -0
- sde/testing/loader.py +175 -0
- sde/testing/memory.py +331 -0
- sde/types.py +228 -0
- sde/verification.py +220 -0
- sde/watermark.py +222 -0
- sde/write_fence.py +283 -0
- sde_demo/__init__.py +1 -0
- sde_demo/__main__.py +183 -0
- sde_demo/diagnostics.py +92 -0
- sde_demo/model.py +75 -0
- sde_demo/project.py +312 -0
- sde_demo/py.typed +0 -0
- sde_demo/query_count.py +301 -0
- sde_demo/resources.py +969 -0
- sde_demo/runtime.py +419 -0
- sde_demo/verification.py +242 -0
- sde_operator/__init__.py +1 -0
- sde_operator/__main__.py +210 -0
- smart_data_engine_sdk-0.1.0.dist-info/METADATA +174 -0
- smart_data_engine_sdk-0.1.0.dist-info/RECORD +73 -0
- smart_data_engine_sdk-0.1.0.dist-info/WHEEL +4 -0
- smart_data_engine_sdk-0.1.0.dist-info/entry_points.txt +3 -0
- smart_data_engine_sdk-0.1.0.dist-info/licenses/LICENSE +201 -0
- smart_data_engine_sdk-0.1.0.dist-info/licenses/NOTICE +13 -0
sde/provisioning.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Prepare physical schema using client-owned provisioning connections before opening runtime."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from typing import TYPE_CHECKING, cast
|
|
7
|
+
|
|
8
|
+
from .capabilities import satisfies
|
|
9
|
+
from .errors import EngineError, MigrationRefused
|
|
10
|
+
from .generation import Fencable, check_map_project
|
|
11
|
+
from .groups import colocation_groups
|
|
12
|
+
from .model import LogicalModel
|
|
13
|
+
from .physical import refuse_findings
|
|
14
|
+
from .placement import PlacementMap
|
|
15
|
+
from .watermark import WatermarkStore
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from .session import Engine
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def prepare_schema(
|
|
22
|
+
model: LogicalModel,
|
|
23
|
+
placement: PlacementMap,
|
|
24
|
+
engines: Mapping[str, Engine],
|
|
25
|
+
*,
|
|
26
|
+
project_id: str | None = None,
|
|
27
|
+
) -> None:
|
|
28
|
+
"""Apply the described schema and initial generation; never advance an existing generation.
|
|
29
|
+
|
|
30
|
+
Credentials remain in these local adapters. The runtime is opened afterwards on separately
|
|
31
|
+
configured connections, so it need not receive the provisioning role's ALTER capability.
|
|
32
|
+
"""
|
|
33
|
+
if model.version != placement.model_version:
|
|
34
|
+
raise MigrationRefused("schema preparation needs the model named by the placement map")
|
|
35
|
+
local_project = check_map_project(placement, project_id)
|
|
36
|
+
needed = sorted(
|
|
37
|
+
{material.engine for spot in placement.groups.values() for material in spot.all()}
|
|
38
|
+
)
|
|
39
|
+
if any(name not in engines for name in needed):
|
|
40
|
+
raise MigrationRefused("schema preparation is missing an engine named by the map")
|
|
41
|
+
if placement.contract >= 4 and any(
|
|
42
|
+
not callable(getattr(engines[name], "write_fence", None)) for name in needed
|
|
43
|
+
):
|
|
44
|
+
raise MigrationRefused("schema preparation needs native write generations on every engine")
|
|
45
|
+
for group in colocation_groups(model):
|
|
46
|
+
spot = placement.placement_of(group.name)
|
|
47
|
+
keys = {name: model.entity(name).key for name in group.members}
|
|
48
|
+
for material in spot.all():
|
|
49
|
+
engine = engines[material.engine]
|
|
50
|
+
# Provisioning is where a person can act on a physical difference, so here it refuses.
|
|
51
|
+
refuse_findings(engine.ensure_schema(material.layout, keys=keys) or (), EngineError)
|
|
52
|
+
if spot.write_epoch is not None:
|
|
53
|
+
for table in sorted(material.layout.tables.values()):
|
|
54
|
+
assert local_project is not None
|
|
55
|
+
cast(Fencable, engine).write_fence(table, project_id=local_project).prepare(
|
|
56
|
+
spot.write_epoch
|
|
57
|
+
)
|
|
58
|
+
if placement.signed:
|
|
59
|
+
# Prepare storage only. Recording this map here would activate a future watermark before
|
|
60
|
+
# the native generations are ready for runtime and could lock out the current map.
|
|
61
|
+
for name in sorted(engines):
|
|
62
|
+
if satisfies(engines[name], WatermarkStore):
|
|
63
|
+
cast(WatermarkStore, engines[name]).map_watermark()
|
sde/py.typed
ADDED
|
File without changes
|
sde/query.py
ADDED
|
@@ -0,0 +1,521 @@
|
|
|
1
|
+
"""Bounded logical read plans. Values stay in the application process, outside shape identities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import UTC, date, datetime
|
|
10
|
+
from decimal import Decimal, InvalidOperation
|
|
11
|
+
from typing import Any, Final, Protocol, cast
|
|
12
|
+
from uuid import UUID
|
|
13
|
+
|
|
14
|
+
from .errors import ModelPlanningError
|
|
15
|
+
from .schema import QUOTE
|
|
16
|
+
|
|
17
|
+
MAX_PAGE_ROWS: Final = 1000
|
|
18
|
+
MAX_ORDER_FIELDS: Final = 32
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class QueryRefused(ModelPlanningError):
|
|
22
|
+
"""A logical query cannot be executed as asked; no query I/O has started."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Range:
|
|
27
|
+
field: str
|
|
28
|
+
low: Any = None
|
|
29
|
+
high: Any = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class ReadColumn:
|
|
34
|
+
name: str
|
|
35
|
+
type: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class ReadFilter:
|
|
40
|
+
column: ReadColumn
|
|
41
|
+
operation: str
|
|
42
|
+
value: Any
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class ReadPlan:
|
|
47
|
+
columns: tuple[ReadColumn, ...]
|
|
48
|
+
filters: tuple[ReadFilter, ...]
|
|
49
|
+
order: tuple[ReadColumn, ...]
|
|
50
|
+
descending: bool
|
|
51
|
+
after: tuple[Any, ...] | None
|
|
52
|
+
limit: int
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class ScanPage:
|
|
57
|
+
rows: tuple[dict[str, Any], ...]
|
|
58
|
+
next_after: Mapping[str, Any] | None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Queryable(Protocol):
|
|
62
|
+
def select_rows(self, table: str, plan: ReadPlan) -> list[dict[str, Any]]: ...
|
|
63
|
+
def count_rows(self, table: str, plan: ReadPlan) -> int: ...
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def query_engine(engine: object) -> Queryable:
|
|
67
|
+
if not all(callable(getattr(engine, name, None)) for name in ("select_rows", "count_rows")):
|
|
68
|
+
raise QueryRefused("this adapter does not support logical reads (select_rows/count_rows)")
|
|
69
|
+
return cast(Queryable, engine)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _text(value: str) -> str:
|
|
73
|
+
try:
|
|
74
|
+
value.encode("utf-8")
|
|
75
|
+
except UnicodeEncodeError:
|
|
76
|
+
raise QueryRefused("query strings must contain Unicode scalar values") from None
|
|
77
|
+
return value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _decimal(value: Any) -> Decimal:
|
|
81
|
+
if isinstance(value, bool) or not isinstance(value, (str, int, Decimal)):
|
|
82
|
+
raise QueryRefused("decimal query values require Decimal, integer or decimal text")
|
|
83
|
+
if (
|
|
84
|
+
isinstance(value, str)
|
|
85
|
+
and re.fullmatch(
|
|
86
|
+
r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", value.strip()
|
|
87
|
+
)
|
|
88
|
+
is None
|
|
89
|
+
):
|
|
90
|
+
raise QueryRefused("invalid decimal query value")
|
|
91
|
+
try:
|
|
92
|
+
number = Decimal(value)
|
|
93
|
+
except InvalidOperation:
|
|
94
|
+
raise QueryRefused("invalid decimal query value") from None
|
|
95
|
+
if not number.is_finite():
|
|
96
|
+
raise QueryRefused("decimal query values must be finite")
|
|
97
|
+
# Check before formatting: a compact exponent must not allocate an unbounded parameter.
|
|
98
|
+
exponent = number.as_tuple().exponent
|
|
99
|
+
assert isinstance(exponent, int)
|
|
100
|
+
scale = max(0, -exponent)
|
|
101
|
+
digits = max(0, number.adjusted() + 1) + scale
|
|
102
|
+
if digits > 76 or scale > 76:
|
|
103
|
+
raise QueryRefused("decimal query values may use at most 76 digits")
|
|
104
|
+
return number
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def query_value(column: ReadColumn, value: Any) -> Any:
|
|
108
|
+
"""Snapshot one typed predicate/cursor value without a driver or network call."""
|
|
109
|
+
if value is None:
|
|
110
|
+
return None
|
|
111
|
+
kind = column.type
|
|
112
|
+
if kind.startswith("decimal("):
|
|
113
|
+
return _decimal(value)
|
|
114
|
+
if kind == "bool":
|
|
115
|
+
if not isinstance(value, bool):
|
|
116
|
+
raise QueryRefused("a boolean query value must be a bool")
|
|
117
|
+
return value
|
|
118
|
+
if kind in ("int32", "int64"):
|
|
119
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
120
|
+
raise QueryRefused("an integer query value must be an integer")
|
|
121
|
+
bits = 32 if kind == "int32" else 64
|
|
122
|
+
if not -(2 ** (bits - 1)) <= value < 2 ** (bits - 1):
|
|
123
|
+
raise QueryRefused(f"integer query value is outside {kind}")
|
|
124
|
+
return value
|
|
125
|
+
if kind in ("float32", "float64"):
|
|
126
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
127
|
+
raise QueryRefused("a float query value must be a finite number")
|
|
128
|
+
try:
|
|
129
|
+
number = float(value)
|
|
130
|
+
except OverflowError:
|
|
131
|
+
raise QueryRefused("float query value is outside float64") from None
|
|
132
|
+
if not math.isfinite(number):
|
|
133
|
+
raise QueryRefused("float query bounds must be finite")
|
|
134
|
+
return number
|
|
135
|
+
if kind == "string":
|
|
136
|
+
if not isinstance(value, str):
|
|
137
|
+
raise QueryRefused("a string query value must be text")
|
|
138
|
+
return _text(value)
|
|
139
|
+
if kind == "uuid":
|
|
140
|
+
if isinstance(value, UUID):
|
|
141
|
+
return value
|
|
142
|
+
if (
|
|
143
|
+
not isinstance(value, str)
|
|
144
|
+
or re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value) is None
|
|
145
|
+
):
|
|
146
|
+
raise QueryRefused("a UUID query value must be UUID or canonical UUID text")
|
|
147
|
+
return UUID(value)
|
|
148
|
+
if kind == "date":
|
|
149
|
+
if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
150
|
+
try:
|
|
151
|
+
return date.fromisoformat(value)
|
|
152
|
+
except ValueError:
|
|
153
|
+
pass
|
|
154
|
+
if isinstance(value, date) and not isinstance(value, datetime):
|
|
155
|
+
return value
|
|
156
|
+
raise QueryRefused("a date query value must be a date or YYYY-MM-DD text")
|
|
157
|
+
if kind in ("timestamp", "timestamptz"):
|
|
158
|
+
if isinstance(value, str):
|
|
159
|
+
if (
|
|
160
|
+
re.fullmatch(
|
|
161
|
+
r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?"
|
|
162
|
+
r"(?:Z|[+-]\d{2}:\d{2}(?::\d{2})?)?",
|
|
163
|
+
value,
|
|
164
|
+
)
|
|
165
|
+
is None
|
|
166
|
+
):
|
|
167
|
+
raise QueryRefused("timestamp query text needs at most six fractional digits")
|
|
168
|
+
try:
|
|
169
|
+
value = datetime.fromisoformat(value)
|
|
170
|
+
except ValueError:
|
|
171
|
+
raise QueryRefused("invalid timestamp query value") from None
|
|
172
|
+
if not isinstance(value, datetime):
|
|
173
|
+
raise QueryRefused("a timestamp query value must be datetime or ISO text")
|
|
174
|
+
try:
|
|
175
|
+
instant = (
|
|
176
|
+
value.replace(tzinfo=UTC) if value.utcoffset() is None else value.astimezone(UTC)
|
|
177
|
+
)
|
|
178
|
+
except (ValueError, OverflowError):
|
|
179
|
+
raise QueryRefused(
|
|
180
|
+
"timestamp query value is outside UTC years 0001 through 9999"
|
|
181
|
+
) from None
|
|
182
|
+
return instant.replace(tzinfo=None) if kind == "timestamp" else instant
|
|
183
|
+
if kind == "bytes" and isinstance(value, (bytes, bytearray, memoryview)):
|
|
184
|
+
return bytes(value)
|
|
185
|
+
raise QueryRefused(f"query predicates are not supported for {kind}")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def plan_read(
|
|
189
|
+
columns: Sequence[ReadColumn],
|
|
190
|
+
key: Sequence[str],
|
|
191
|
+
*,
|
|
192
|
+
where: Mapping[str, Any] | None = None,
|
|
193
|
+
bounds: Range | None = None,
|
|
194
|
+
order_by: str | None = None,
|
|
195
|
+
descending: bool = False,
|
|
196
|
+
after: Mapping[str, Any] | None = None,
|
|
197
|
+
limit: int = 100,
|
|
198
|
+
paginate: bool = True,
|
|
199
|
+
) -> ReadPlan:
|
|
200
|
+
by_name = {column.name: column for column in columns}
|
|
201
|
+
|
|
202
|
+
def field(name: str) -> ReadColumn:
|
|
203
|
+
if not isinstance(name, str) or name not in by_name:
|
|
204
|
+
raise QueryRefused("query refers to a field this entity does not declare")
|
|
205
|
+
return by_name[name]
|
|
206
|
+
|
|
207
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= MAX_PAGE_ROWS:
|
|
208
|
+
raise QueryRefused(f"page limit must be an integer between 1 and {MAX_PAGE_ROWS}")
|
|
209
|
+
if not isinstance(descending, bool):
|
|
210
|
+
raise QueryRefused("descending must be a bool")
|
|
211
|
+
if paginate and not key:
|
|
212
|
+
raise QueryRefused("a paginated read requires an entity key")
|
|
213
|
+
order_names = ([order_by] if order_by is not None else []) + [
|
|
214
|
+
name for name in key if name != order_by
|
|
215
|
+
]
|
|
216
|
+
if paginate and len(order_names) > MAX_ORDER_FIELDS:
|
|
217
|
+
raise QueryRefused(f"a read may order by at most {MAX_ORDER_FIELDS} fields")
|
|
218
|
+
order = tuple(field(name) for name in order_names) if paginate else ()
|
|
219
|
+
if any(column.type in ("json", "float32", "float64") for column in order):
|
|
220
|
+
raise QueryRefused("JSON and floating-point ordering are not supported by this read API")
|
|
221
|
+
if any(
|
|
222
|
+
column.type.startswith("decimal(")
|
|
223
|
+
and int(column.type[len("decimal(") : -1].split(",")[0]) > 76
|
|
224
|
+
for column in order
|
|
225
|
+
):
|
|
226
|
+
raise QueryRefused("decimal ordering supports at most 76 digits")
|
|
227
|
+
filters: list[ReadFilter] = []
|
|
228
|
+
if where is not None:
|
|
229
|
+
if not isinstance(where, Mapping) or any(not isinstance(name, str) for name in where):
|
|
230
|
+
raise QueryRefused("where must map logical field names to scalar values")
|
|
231
|
+
for name in sorted(where):
|
|
232
|
+
column = field(name)
|
|
233
|
+
if column.type == "json":
|
|
234
|
+
raise QueryRefused("JSON predicates are not supported by this read API")
|
|
235
|
+
filters.append(ReadFilter(column, "eq", query_value(column, where[name])))
|
|
236
|
+
if bounds is not None:
|
|
237
|
+
if not isinstance(bounds, Range):
|
|
238
|
+
raise QueryRefused("bounds must be a Range")
|
|
239
|
+
column = field(bounds.field)
|
|
240
|
+
if not column.type.startswith(
|
|
241
|
+
("int32", "int64", "float32", "float64", "decimal(", "date", "timestamp")
|
|
242
|
+
):
|
|
243
|
+
raise QueryRefused("this field has no range-read shape")
|
|
244
|
+
if bounds.low is None and bounds.high is None:
|
|
245
|
+
raise QueryRefused("a range needs at least one bound")
|
|
246
|
+
for operation, value in (("ge", bounds.low), ("lt", bounds.high)):
|
|
247
|
+
if value is not None:
|
|
248
|
+
filters.append(ReadFilter(column, operation, query_value(column, value)))
|
|
249
|
+
position = None
|
|
250
|
+
if after is not None:
|
|
251
|
+
if not paginate:
|
|
252
|
+
raise QueryRefused("an aggregate has no pagination position")
|
|
253
|
+
if not isinstance(after, Mapping) or set(after) != set(order_names):
|
|
254
|
+
raise QueryRefused("after must contain exactly the complete ordering key")
|
|
255
|
+
position = tuple(query_value(column, after[column.name]) for column in order)
|
|
256
|
+
return ReadPlan(tuple(columns), tuple(filters), order, descending, position, limit)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _expression(column: ReadColumn, dialect: str) -> str:
|
|
260
|
+
expression = QUOTE[dialect](column.name)
|
|
261
|
+
if column.type == "string" and dialect == "postgres":
|
|
262
|
+
return f'({expression} COLLATE "C")'
|
|
263
|
+
if column.type == "uuid" and dialect == "clickhouse":
|
|
264
|
+
return f"toString({expression})"
|
|
265
|
+
if column.type in ("float32", "float64"):
|
|
266
|
+
return (
|
|
267
|
+
f"CAST({expression} AS double precision)"
|
|
268
|
+
if dialect == "postgres"
|
|
269
|
+
else f"toFloat64({expression})"
|
|
270
|
+
)
|
|
271
|
+
return expression
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _comparison(
|
|
275
|
+
column: ReadColumn,
|
|
276
|
+
operator: str,
|
|
277
|
+
value: Any,
|
|
278
|
+
dialect: str,
|
|
279
|
+
parameter: Callable[[Any], str],
|
|
280
|
+
) -> str:
|
|
281
|
+
expression = _expression(column, dialect)
|
|
282
|
+
if value is None:
|
|
283
|
+
return f"{expression} IS NULL"
|
|
284
|
+
if column.type.startswith("decimal("):
|
|
285
|
+
_, scale_text = column.type[len("decimal(") : -1].split(",")
|
|
286
|
+
scale = max(int(scale_text), max(0, -value.as_tuple().exponent))
|
|
287
|
+
integer_digits = int(column.type[len("decimal(") : -1].split(",")[0]) - int(scale_text)
|
|
288
|
+
if integer_digits + scale > 76:
|
|
289
|
+
raise QueryRefused("decimal comparison requires more than 76 digits")
|
|
290
|
+
target = (
|
|
291
|
+
f"numeric(76,{scale})" if dialect == "postgres" else f"Nullable(Decimal(76,{scale}))"
|
|
292
|
+
)
|
|
293
|
+
expression = f"CAST({expression} AS {target})"
|
|
294
|
+
bound = f"CAST({parameter(format(value, 'f'))} AS {target})"
|
|
295
|
+
else:
|
|
296
|
+
bound = parameter(
|
|
297
|
+
str(value) if column.type == "uuid" and dialect == "clickhouse" else value
|
|
298
|
+
)
|
|
299
|
+
comparison = f"{expression} {operator} {bound}"
|
|
300
|
+
if column.type in ("float32", "float64") and operator != "=":
|
|
301
|
+
not_nan = (
|
|
302
|
+
f"{expression} <> 'NaN'::double precision"
|
|
303
|
+
if dialect == "postgres"
|
|
304
|
+
else f"NOT isNaN({expression})"
|
|
305
|
+
)
|
|
306
|
+
comparison = f"({not_nan} AND {comparison})"
|
|
307
|
+
return comparison
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def read_sql(
|
|
311
|
+
table: str,
|
|
312
|
+
plan: ReadPlan,
|
|
313
|
+
*,
|
|
314
|
+
dialect: str,
|
|
315
|
+
parameter: Callable[[Any], str],
|
|
316
|
+
count: bool = False,
|
|
317
|
+
) -> str:
|
|
318
|
+
"""One native query. The parameter callback owns the driver's value representation."""
|
|
319
|
+
clauses = [
|
|
320
|
+
_comparison(
|
|
321
|
+
predicate.column,
|
|
322
|
+
{"eq": "=", "ge": ">=", "lt": "<"}[predicate.operation],
|
|
323
|
+
predicate.value,
|
|
324
|
+
dialect,
|
|
325
|
+
parameter,
|
|
326
|
+
)
|
|
327
|
+
for predicate in plan.filters
|
|
328
|
+
]
|
|
329
|
+
if not count and plan.after is not None:
|
|
330
|
+
alternatives: list[str] = []
|
|
331
|
+
operator = "<" if plan.descending else ">"
|
|
332
|
+
for index, (column, value) in enumerate(zip(plan.order, plan.after, strict=True)):
|
|
333
|
+
if value is not None:
|
|
334
|
+
# Bind each occurrence in SQL order. Reusing a prefix would reuse a named
|
|
335
|
+
# ClickHouse parameter but leave PostgreSQL's positional parameters misaligned.
|
|
336
|
+
prefix = [
|
|
337
|
+
_comparison(previous, "=", plan.after[position], dialect, parameter)
|
|
338
|
+
for position, previous in enumerate(plan.order[:index])
|
|
339
|
+
]
|
|
340
|
+
expression = _expression(column, dialect)
|
|
341
|
+
later = _comparison(column, operator, value, dialect, parameter)
|
|
342
|
+
alternatives.append(
|
|
343
|
+
"(" + " AND ".join([*prefix, f"({expression} IS NULL OR {later})"]) + ")"
|
|
344
|
+
)
|
|
345
|
+
clauses.append("(" + " OR ".join(alternatives) + ")" if alternatives else "FALSE")
|
|
346
|
+
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
|
347
|
+
quote = QUOTE[dialect]
|
|
348
|
+
final = " FINAL" if dialect == "clickhouse" else ""
|
|
349
|
+
if count:
|
|
350
|
+
expression = "CAST(count(*) AS text)" if dialect == "postgres" else "toString(count())"
|
|
351
|
+
return f"SELECT {expression} AS sde_count FROM {quote(table)}{final}{where}"
|
|
352
|
+
projection = ", ".join(
|
|
353
|
+
f"toTimeZone({quote(column.name)}, 'UTC') AS {quote(column.name)}"
|
|
354
|
+
if dialect == "clickhouse" and column.type in ("timestamp", "timestamptz")
|
|
355
|
+
else (
|
|
356
|
+
f"CAST({quote(column.name)} AS text) AS {quote(column.name)}"
|
|
357
|
+
if dialect == "postgres"
|
|
358
|
+
else f"toString({quote(column.name)}) AS {quote(column.name)}"
|
|
359
|
+
)
|
|
360
|
+
if column.type.startswith("decimal(")
|
|
361
|
+
else quote(column.name)
|
|
362
|
+
for column in plan.columns
|
|
363
|
+
)
|
|
364
|
+
direction = "DESC" if plan.descending else "ASC"
|
|
365
|
+
order = ", ".join(
|
|
366
|
+
f"{_expression(column, dialect)} {direction} NULLS LAST" for column in plan.order
|
|
367
|
+
)
|
|
368
|
+
return (
|
|
369
|
+
f"SELECT {projection} FROM {quote(table)}{final}{where} ORDER BY {order} "
|
|
370
|
+
f"LIMIT {parameter(plan.limit + 1)}"
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@dataclass(frozen=True)
|
|
375
|
+
class NumericSummary:
|
|
376
|
+
count: int
|
|
377
|
+
non_null_count: int
|
|
378
|
+
minimum: int | Decimal | None
|
|
379
|
+
maximum: int | Decimal | None
|
|
380
|
+
total: int | Decimal | None
|
|
381
|
+
mean: Decimal | None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
class Summarizable(Protocol):
|
|
385
|
+
def summarize_rows(
|
|
386
|
+
self,
|
|
387
|
+
table: str,
|
|
388
|
+
plan: ReadPlan,
|
|
389
|
+
column: ReadColumn,
|
|
390
|
+
) -> Mapping[str, Any]: ...
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def summary_engine(engine: object) -> Summarizable:
|
|
394
|
+
if not callable(getattr(engine, "summarize_rows", None)):
|
|
395
|
+
raise QueryRefused("this adapter does not support numeric summaries (summarize_rows)")
|
|
396
|
+
return cast(Summarizable, engine)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def summary_scale(column: ReadColumn, mean_scale: int) -> int:
|
|
400
|
+
if isinstance(mean_scale, bool) or not isinstance(mean_scale, int) or not 0 <= mean_scale <= 38:
|
|
401
|
+
raise QueryRefused("mean_scale must be an integer between 0 and 38")
|
|
402
|
+
if column.type in ("int32", "int64"):
|
|
403
|
+
return 0
|
|
404
|
+
if column.type.startswith("decimal("):
|
|
405
|
+
precision, scale = map(int, column.type[len("decimal(") : -1].split(","))
|
|
406
|
+
if precision <= 56:
|
|
407
|
+
return scale
|
|
408
|
+
raise QueryRefused("summaries require int32, int64 or decimal precision up to 56")
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def summary_sql(
|
|
412
|
+
table: str,
|
|
413
|
+
plan: ReadPlan,
|
|
414
|
+
column: ReadColumn,
|
|
415
|
+
*,
|
|
416
|
+
dialect: str,
|
|
417
|
+
parameter: Callable[[Any], str],
|
|
418
|
+
) -> str:
|
|
419
|
+
scale = summary_scale(column, 0)
|
|
420
|
+
quoted = QUOTE[dialect](column.name)
|
|
421
|
+
target = f"numeric(76,{scale})" if dialect == "postgres" else f"Nullable(Decimal(76,{scale}))"
|
|
422
|
+
|
|
423
|
+
def text(expression: str) -> str:
|
|
424
|
+
return f"CAST({expression} AS text)" if dialect == "postgres" else f"toString({expression})"
|
|
425
|
+
|
|
426
|
+
fields = [
|
|
427
|
+
text("count(*)") + " AS sde_count",
|
|
428
|
+
text(f"count({quoted})") + " AS sde_present",
|
|
429
|
+
text(f"min({quoted})") + " AS sde_min",
|
|
430
|
+
text(f"max({quoted})") + " AS sde_max",
|
|
431
|
+
text(f"sum(CAST({quoted} AS {target}))") + " AS sde_total",
|
|
432
|
+
]
|
|
433
|
+
clauses = [
|
|
434
|
+
_comparison(
|
|
435
|
+
item.column,
|
|
436
|
+
{"eq": "=", "ge": ">=", "lt": "<"}[item.operation],
|
|
437
|
+
item.value,
|
|
438
|
+
dialect,
|
|
439
|
+
parameter,
|
|
440
|
+
)
|
|
441
|
+
for item in plan.filters
|
|
442
|
+
]
|
|
443
|
+
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
|
444
|
+
final = " FINAL" if dialect == "clickhouse" else ""
|
|
445
|
+
return f"SELECT {', '.join(fields)} FROM {QUOTE[dialect](table)}{final}{where}"
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _decimal_integer(number: Decimal, scale: int) -> int:
|
|
449
|
+
sign, digits, exponent = number.as_tuple()
|
|
450
|
+
assert isinstance(exponent, int)
|
|
451
|
+
coefficient = 0
|
|
452
|
+
for digit in digits:
|
|
453
|
+
coefficient = coefficient * 10 + digit
|
|
454
|
+
shift = exponent + scale
|
|
455
|
+
if shift < 0:
|
|
456
|
+
divisor = int(10**-shift)
|
|
457
|
+
if coefficient % divisor:
|
|
458
|
+
raise ValueError("summary result has fractional digits outside the stored scale")
|
|
459
|
+
coefficient //= divisor
|
|
460
|
+
else:
|
|
461
|
+
coefficient *= int(10**shift)
|
|
462
|
+
return -coefficient if sign else coefficient
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def numeric_summary(
|
|
466
|
+
record: Mapping[str, Any],
|
|
467
|
+
column: ReadColumn,
|
|
468
|
+
mean_scale: int,
|
|
469
|
+
) -> NumericSummary:
|
|
470
|
+
"""Decode exact server text and round a rational mean independently of decimal context."""
|
|
471
|
+
scale = summary_scale(column, mean_scale)
|
|
472
|
+
count, present = int(record["sde_count"]), int(record["sde_present"])
|
|
473
|
+
if not 0 <= present <= count:
|
|
474
|
+
raise ValueError("summary counts are inconsistent")
|
|
475
|
+
if present == 0:
|
|
476
|
+
return NumericSummary(count, 0, None, None, None, None)
|
|
477
|
+
minimum, maximum, total = (_decimal(record[key]) for key in ("sde_min", "sde_max", "sde_total"))
|
|
478
|
+
unscaled = _decimal_integer(total, scale)
|
|
479
|
+
numerator, denominator = abs(unscaled) * 10**mean_scale, present * 10**scale
|
|
480
|
+
rounded, remainder = divmod(numerator, denominator)
|
|
481
|
+
if remainder * 2 > denominator or (remainder * 2 == denominator and rounded % 2):
|
|
482
|
+
rounded += 1
|
|
483
|
+
mean = Decimal((int(unscaled < 0), tuple(int(digit) for digit in str(rounded)), -mean_scale))
|
|
484
|
+
if column.type in ("int32", "int64"):
|
|
485
|
+
return NumericSummary(
|
|
486
|
+
count,
|
|
487
|
+
present,
|
|
488
|
+
_decimal_integer(minimum, 0),
|
|
489
|
+
_decimal_integer(maximum, 0),
|
|
490
|
+
_decimal_integer(total, 0),
|
|
491
|
+
mean,
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
def scaled(value: Decimal) -> Decimal:
|
|
495
|
+
coefficient = _decimal_integer(value, scale)
|
|
496
|
+
return Decimal(
|
|
497
|
+
(
|
|
498
|
+
int(coefficient < 0),
|
|
499
|
+
tuple(int(digit) for digit in str(abs(coefficient))),
|
|
500
|
+
-scale,
|
|
501
|
+
)
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
return NumericSummary(count, present, scaled(minimum), scaled(maximum), scaled(total), mean)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def read_row(columns: Sequence[ReadColumn], row: Mapping[str, Any]) -> dict[str, Any]:
|
|
508
|
+
result = dict(row)
|
|
509
|
+
for column in columns:
|
|
510
|
+
value = result[column.name]
|
|
511
|
+
if value is not None and column.type.startswith("decimal("):
|
|
512
|
+
scale = int(column.type[len("decimal(") : -1].split(",")[1])
|
|
513
|
+
coefficient = _decimal_integer(_decimal(value), scale)
|
|
514
|
+
result[column.name] = Decimal(
|
|
515
|
+
(
|
|
516
|
+
int(coefficient < 0),
|
|
517
|
+
tuple(int(digit) for digit in str(abs(coefficient))),
|
|
518
|
+
-scale,
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
return result
|
sde/routing.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Routing: a lookup and three conditions, and deliberately nothing more.
|
|
2
|
+
|
|
3
|
+
The library does not decide where an operation goes. The planner decided, ahead of time, for every
|
|
4
|
+
shape the model admits, and put the answers in the placement map. This module reads them.
|
|
5
|
+
|
|
6
|
+
That division is the reason it is affordable to have four libraries. Decisions need telemetry,
|
|
7
|
+
history, a cost model and an explanation, and they need to be reproducible and testable - all of
|
|
8
|
+
which live on our side, once. If the library decided anything, that judgement would have to be
|
|
9
|
+
reimplemented in Python, TypeScript, Java and Rust, and kept identical in all four forever. So it
|
|
10
|
+
looks things up.
|
|
11
|
+
|
|
12
|
+
The three conditions that are *not* lookups exist because they are correctness, not judgement:
|
|
13
|
+
|
|
14
|
+
1. Writes go to the source materialisation. There is exactly one, and derived copies are derived.
|
|
15
|
+
2. An operation inside a transaction that has already written goes to the source, because a derived
|
|
16
|
+
copy is behind by design and would not show the write the caller just made.
|
|
17
|
+
3. An operation asking for no staleness goes to the source, for the same reason.
|
|
18
|
+
|
|
19
|
+
Anything else follows the routing table, and if the table has nothing to say the answer is the
|
|
20
|
+
source. That fallback is safe by construction, since the source is always correct and merely
|
|
21
|
+
sometimes slower, and it is what makes a hand-written map a two-line affair rather than a table of
|
|
22
|
+
hashes.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
|
|
29
|
+
from .logging import log
|
|
30
|
+
from .placement import Materialization, PlacementMap
|
|
31
|
+
from .shapes import WRITE_KINDS, OperationShape
|
|
32
|
+
|
|
33
|
+
__all__ = ["Router", "resolve"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Router:
|
|
38
|
+
"""Resolves shapes against one placement map."""
|
|
39
|
+
|
|
40
|
+
placement: PlacementMap
|
|
41
|
+
|
|
42
|
+
def resolve(
|
|
43
|
+
self,
|
|
44
|
+
shape: OperationShape,
|
|
45
|
+
*,
|
|
46
|
+
in_write_transaction: bool = False,
|
|
47
|
+
fresh: bool = False,
|
|
48
|
+
) -> Materialization:
|
|
49
|
+
group = self.placement.placement_of(shape.group)
|
|
50
|
+
|
|
51
|
+
if shape.kind in WRITE_KINDS:
|
|
52
|
+
return group.source
|
|
53
|
+
if in_write_transaction or fresh:
|
|
54
|
+
return group.source
|
|
55
|
+
|
|
56
|
+
target_id = self.placement.routing.get(shape.id)
|
|
57
|
+
if target_id is None:
|
|
58
|
+
# Not an error. A map with no routing table is the normal shape of a hand-written one,
|
|
59
|
+
# and the source is always a correct answer.
|
|
60
|
+
log("sde.route.fallback", group=shape.group, shape=shape.id, kind=shape.kind)
|
|
61
|
+
return group.source
|
|
62
|
+
|
|
63
|
+
materialization = group.by_id(target_id)
|
|
64
|
+
log(
|
|
65
|
+
"sde.route.resolved",
|
|
66
|
+
group=shape.group,
|
|
67
|
+
shape=shape.id,
|
|
68
|
+
kind=shape.kind,
|
|
69
|
+
materialization=materialization.id,
|
|
70
|
+
engine=materialization.engine,
|
|
71
|
+
)
|
|
72
|
+
return materialization
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def resolve(
|
|
76
|
+
placement: PlacementMap,
|
|
77
|
+
shape: OperationShape,
|
|
78
|
+
*,
|
|
79
|
+
in_write_transaction: bool = False,
|
|
80
|
+
fresh: bool = False,
|
|
81
|
+
) -> Materialization:
|
|
82
|
+
"""Functional form, for the conformance vectors and for one-off resolution."""
|
|
83
|
+
return Router(placement).resolve(
|
|
84
|
+
shape, in_write_transaction=in_write_transaction, fresh=fresh
|
|
85
|
+
)
|