smart-data-engine-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. sde/__init__.py +318 -0
  2. sde/_cutover_project.py +179 -0
  3. sde/_local_state.py +188 -0
  4. sde/_operator_deadline.py +50 -0
  5. sde/_usage.py +314 -0
  6. sde/bulk.py +79 -0
  7. sde/canonical.py +141 -0
  8. sde/capabilities.py +62 -0
  9. sde/cutover.py +286 -0
  10. sde/engines/__init__.py +0 -0
  11. sde/engines/_clickhouse_connection.py +224 -0
  12. sde/engines/_index_build.py +294 -0
  13. sde/engines/_operator.py +394 -0
  14. sde/engines/_staging.py +222 -0
  15. sde/engines/_storage.py +22 -0
  16. sde/engines/_write_fences.py +271 -0
  17. sde/engines/clickhouse.py +1115 -0
  18. sde/engines/orderbook.py +457 -0
  19. sde/engines/postgres.py +967 -0
  20. sde/entity.py +170 -0
  21. sde/errors.py +103 -0
  22. sde/explain.py +300 -0
  23. sde/frozen_verification.py +152 -0
  24. sde/generation.py +131 -0
  25. sde/groups.py +97 -0
  26. sde/hashing.py +242 -0
  27. sde/index_build.py +313 -0
  28. sde/index_operator.py +347 -0
  29. sde/infer.py +461 -0
  30. sde/inspection.py +62 -0
  31. sde/internal.py +90 -0
  32. sde/layout.py +669 -0
  33. sde/local_cutover.py +801 -0
  34. sde/logging.py +143 -0
  35. sde/migration.py +856 -0
  36. sde/model.py +482 -0
  37. sde/physical.py +531 -0
  38. sde/placement.py +1010 -0
  39. sde/provisioning.py +63 -0
  40. sde/py.typed +0 -0
  41. sde/query.py +521 -0
  42. sde/routing.py +85 -0
  43. sde/schema.py +466 -0
  44. sde/session.py +993 -0
  45. sde/shapes.py +153 -0
  46. sde/staging.py +264 -0
  47. sde/staging_operator.py +393 -0
  48. sde/telemetry.py +1087 -0
  49. sde/testing/__init__.py +14 -0
  50. sde/testing/loader.py +175 -0
  51. sde/testing/memory.py +331 -0
  52. sde/types.py +228 -0
  53. sde/verification.py +220 -0
  54. sde/watermark.py +222 -0
  55. sde/write_fence.py +283 -0
  56. sde_demo/__init__.py +1 -0
  57. sde_demo/__main__.py +183 -0
  58. sde_demo/diagnostics.py +92 -0
  59. sde_demo/model.py +75 -0
  60. sde_demo/project.py +312 -0
  61. sde_demo/py.typed +0 -0
  62. sde_demo/query_count.py +301 -0
  63. sde_demo/resources.py +969 -0
  64. sde_demo/runtime.py +419 -0
  65. sde_demo/verification.py +242 -0
  66. sde_operator/__init__.py +1 -0
  67. sde_operator/__main__.py +210 -0
  68. smart_data_engine_sdk-0.1.0.dist-info/METADATA +174 -0
  69. smart_data_engine_sdk-0.1.0.dist-info/RECORD +73 -0
  70. smart_data_engine_sdk-0.1.0.dist-info/WHEEL +4 -0
  71. smart_data_engine_sdk-0.1.0.dist-info/entry_points.txt +3 -0
  72. smart_data_engine_sdk-0.1.0.dist-info/licenses/LICENSE +201 -0
  73. smart_data_engine_sdk-0.1.0.dist-info/licenses/NOTICE +13 -0
sde/provisioning.py ADDED
@@ -0,0 +1,63 @@
1
+ """Prepare physical schema using client-owned provisioning connections before opening runtime."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from typing import TYPE_CHECKING, cast
7
+
8
+ from .capabilities import satisfies
9
+ from .errors import EngineError, MigrationRefused
10
+ from .generation import Fencable, check_map_project
11
+ from .groups import colocation_groups
12
+ from .model import LogicalModel
13
+ from .physical import refuse_findings
14
+ from .placement import PlacementMap
15
+ from .watermark import WatermarkStore
16
+
17
+ if TYPE_CHECKING:
18
+ from .session import Engine
19
+
20
+
21
+ def prepare_schema(
22
+ model: LogicalModel,
23
+ placement: PlacementMap,
24
+ engines: Mapping[str, Engine],
25
+ *,
26
+ project_id: str | None = None,
27
+ ) -> None:
28
+ """Apply the described schema and initial generation; never advance an existing generation.
29
+
30
+ Credentials remain in these local adapters. The runtime is opened afterwards on separately
31
+ configured connections, so it need not receive the provisioning role's ALTER capability.
32
+ """
33
+ if model.version != placement.model_version:
34
+ raise MigrationRefused("schema preparation needs the model named by the placement map")
35
+ local_project = check_map_project(placement, project_id)
36
+ needed = sorted(
37
+ {material.engine for spot in placement.groups.values() for material in spot.all()}
38
+ )
39
+ if any(name not in engines for name in needed):
40
+ raise MigrationRefused("schema preparation is missing an engine named by the map")
41
+ if placement.contract >= 4 and any(
42
+ not callable(getattr(engines[name], "write_fence", None)) for name in needed
43
+ ):
44
+ raise MigrationRefused("schema preparation needs native write generations on every engine")
45
+ for group in colocation_groups(model):
46
+ spot = placement.placement_of(group.name)
47
+ keys = {name: model.entity(name).key for name in group.members}
48
+ for material in spot.all():
49
+ engine = engines[material.engine]
50
+ # Provisioning is where a person can act on a physical difference, so here it refuses.
51
+ refuse_findings(engine.ensure_schema(material.layout, keys=keys) or (), EngineError)
52
+ if spot.write_epoch is not None:
53
+ for table in sorted(material.layout.tables.values()):
54
+ assert local_project is not None
55
+ cast(Fencable, engine).write_fence(table, project_id=local_project).prepare(
56
+ spot.write_epoch
57
+ )
58
+ if placement.signed:
59
+ # Prepare storage only. Recording this map here would activate a future watermark before
60
+ # the native generations are ready for runtime and could lock out the current map.
61
+ for name in sorted(engines):
62
+ if satisfies(engines[name], WatermarkStore):
63
+ cast(WatermarkStore, engines[name]).map_watermark()
sde/py.typed ADDED
File without changes
sde/query.py ADDED
@@ -0,0 +1,521 @@
1
+ """Bounded logical read plans. Values stay in the application process, outside shape identities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import re
7
+ from collections.abc import Callable, Mapping, Sequence
8
+ from dataclasses import dataclass
9
+ from datetime import UTC, date, datetime
10
+ from decimal import Decimal, InvalidOperation
11
+ from typing import Any, Final, Protocol, cast
12
+ from uuid import UUID
13
+
14
+ from .errors import ModelPlanningError
15
+ from .schema import QUOTE
16
+
17
+ MAX_PAGE_ROWS: Final = 1000
18
+ MAX_ORDER_FIELDS: Final = 32
19
+
20
+
21
+ class QueryRefused(ModelPlanningError):
22
+ """A logical query cannot be executed as asked; no query I/O has started."""
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Range:
27
+ field: str
28
+ low: Any = None
29
+ high: Any = None
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class ReadColumn:
34
+ name: str
35
+ type: str
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class ReadFilter:
40
+ column: ReadColumn
41
+ operation: str
42
+ value: Any
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class ReadPlan:
47
+ columns: tuple[ReadColumn, ...]
48
+ filters: tuple[ReadFilter, ...]
49
+ order: tuple[ReadColumn, ...]
50
+ descending: bool
51
+ after: tuple[Any, ...] | None
52
+ limit: int
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class ScanPage:
57
+ rows: tuple[dict[str, Any], ...]
58
+ next_after: Mapping[str, Any] | None
59
+
60
+
61
+ class Queryable(Protocol):
62
+ def select_rows(self, table: str, plan: ReadPlan) -> list[dict[str, Any]]: ...
63
+ def count_rows(self, table: str, plan: ReadPlan) -> int: ...
64
+
65
+
66
+ def query_engine(engine: object) -> Queryable:
67
+ if not all(callable(getattr(engine, name, None)) for name in ("select_rows", "count_rows")):
68
+ raise QueryRefused("this adapter does not support logical reads (select_rows/count_rows)")
69
+ return cast(Queryable, engine)
70
+
71
+
72
+ def _text(value: str) -> str:
73
+ try:
74
+ value.encode("utf-8")
75
+ except UnicodeEncodeError:
76
+ raise QueryRefused("query strings must contain Unicode scalar values") from None
77
+ return value
78
+
79
+
80
+ def _decimal(value: Any) -> Decimal:
81
+ if isinstance(value, bool) or not isinstance(value, (str, int, Decimal)):
82
+ raise QueryRefused("decimal query values require Decimal, integer or decimal text")
83
+ if (
84
+ isinstance(value, str)
85
+ and re.fullmatch(
86
+ r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", value.strip()
87
+ )
88
+ is None
89
+ ):
90
+ raise QueryRefused("invalid decimal query value")
91
+ try:
92
+ number = Decimal(value)
93
+ except InvalidOperation:
94
+ raise QueryRefused("invalid decimal query value") from None
95
+ if not number.is_finite():
96
+ raise QueryRefused("decimal query values must be finite")
97
+ # Check before formatting: a compact exponent must not allocate an unbounded parameter.
98
+ exponent = number.as_tuple().exponent
99
+ assert isinstance(exponent, int)
100
+ scale = max(0, -exponent)
101
+ digits = max(0, number.adjusted() + 1) + scale
102
+ if digits > 76 or scale > 76:
103
+ raise QueryRefused("decimal query values may use at most 76 digits")
104
+ return number
105
+
106
+
107
+ def query_value(column: ReadColumn, value: Any) -> Any:
108
+ """Snapshot one typed predicate/cursor value without a driver or network call."""
109
+ if value is None:
110
+ return None
111
+ kind = column.type
112
+ if kind.startswith("decimal("):
113
+ return _decimal(value)
114
+ if kind == "bool":
115
+ if not isinstance(value, bool):
116
+ raise QueryRefused("a boolean query value must be a bool")
117
+ return value
118
+ if kind in ("int32", "int64"):
119
+ if isinstance(value, bool) or not isinstance(value, int):
120
+ raise QueryRefused("an integer query value must be an integer")
121
+ bits = 32 if kind == "int32" else 64
122
+ if not -(2 ** (bits - 1)) <= value < 2 ** (bits - 1):
123
+ raise QueryRefused(f"integer query value is outside {kind}")
124
+ return value
125
+ if kind in ("float32", "float64"):
126
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
127
+ raise QueryRefused("a float query value must be a finite number")
128
+ try:
129
+ number = float(value)
130
+ except OverflowError:
131
+ raise QueryRefused("float query value is outside float64") from None
132
+ if not math.isfinite(number):
133
+ raise QueryRefused("float query bounds must be finite")
134
+ return number
135
+ if kind == "string":
136
+ if not isinstance(value, str):
137
+ raise QueryRefused("a string query value must be text")
138
+ return _text(value)
139
+ if kind == "uuid":
140
+ if isinstance(value, UUID):
141
+ return value
142
+ if (
143
+ not isinstance(value, str)
144
+ or re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value) is None
145
+ ):
146
+ raise QueryRefused("a UUID query value must be UUID or canonical UUID text")
147
+ return UUID(value)
148
+ if kind == "date":
149
+ if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
150
+ try:
151
+ return date.fromisoformat(value)
152
+ except ValueError:
153
+ pass
154
+ if isinstance(value, date) and not isinstance(value, datetime):
155
+ return value
156
+ raise QueryRefused("a date query value must be a date or YYYY-MM-DD text")
157
+ if kind in ("timestamp", "timestamptz"):
158
+ if isinstance(value, str):
159
+ if (
160
+ re.fullmatch(
161
+ r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?"
162
+ r"(?:Z|[+-]\d{2}:\d{2}(?::\d{2})?)?",
163
+ value,
164
+ )
165
+ is None
166
+ ):
167
+ raise QueryRefused("timestamp query text needs at most six fractional digits")
168
+ try:
169
+ value = datetime.fromisoformat(value)
170
+ except ValueError:
171
+ raise QueryRefused("invalid timestamp query value") from None
172
+ if not isinstance(value, datetime):
173
+ raise QueryRefused("a timestamp query value must be datetime or ISO text")
174
+ try:
175
+ instant = (
176
+ value.replace(tzinfo=UTC) if value.utcoffset() is None else value.astimezone(UTC)
177
+ )
178
+ except (ValueError, OverflowError):
179
+ raise QueryRefused(
180
+ "timestamp query value is outside UTC years 0001 through 9999"
181
+ ) from None
182
+ return instant.replace(tzinfo=None) if kind == "timestamp" else instant
183
+ if kind == "bytes" and isinstance(value, (bytes, bytearray, memoryview)):
184
+ return bytes(value)
185
+ raise QueryRefused(f"query predicates are not supported for {kind}")
186
+
187
+
188
+ def plan_read(
189
+ columns: Sequence[ReadColumn],
190
+ key: Sequence[str],
191
+ *,
192
+ where: Mapping[str, Any] | None = None,
193
+ bounds: Range | None = None,
194
+ order_by: str | None = None,
195
+ descending: bool = False,
196
+ after: Mapping[str, Any] | None = None,
197
+ limit: int = 100,
198
+ paginate: bool = True,
199
+ ) -> ReadPlan:
200
+ by_name = {column.name: column for column in columns}
201
+
202
+ def field(name: str) -> ReadColumn:
203
+ if not isinstance(name, str) or name not in by_name:
204
+ raise QueryRefused("query refers to a field this entity does not declare")
205
+ return by_name[name]
206
+
207
+ if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= MAX_PAGE_ROWS:
208
+ raise QueryRefused(f"page limit must be an integer between 1 and {MAX_PAGE_ROWS}")
209
+ if not isinstance(descending, bool):
210
+ raise QueryRefused("descending must be a bool")
211
+ if paginate and not key:
212
+ raise QueryRefused("a paginated read requires an entity key")
213
+ order_names = ([order_by] if order_by is not None else []) + [
214
+ name for name in key if name != order_by
215
+ ]
216
+ if paginate and len(order_names) > MAX_ORDER_FIELDS:
217
+ raise QueryRefused(f"a read may order by at most {MAX_ORDER_FIELDS} fields")
218
+ order = tuple(field(name) for name in order_names) if paginate else ()
219
+ if any(column.type in ("json", "float32", "float64") for column in order):
220
+ raise QueryRefused("JSON and floating-point ordering are not supported by this read API")
221
+ if any(
222
+ column.type.startswith("decimal(")
223
+ and int(column.type[len("decimal(") : -1].split(",")[0]) > 76
224
+ for column in order
225
+ ):
226
+ raise QueryRefused("decimal ordering supports at most 76 digits")
227
+ filters: list[ReadFilter] = []
228
+ if where is not None:
229
+ if not isinstance(where, Mapping) or any(not isinstance(name, str) for name in where):
230
+ raise QueryRefused("where must map logical field names to scalar values")
231
+ for name in sorted(where):
232
+ column = field(name)
233
+ if column.type == "json":
234
+ raise QueryRefused("JSON predicates are not supported by this read API")
235
+ filters.append(ReadFilter(column, "eq", query_value(column, where[name])))
236
+ if bounds is not None:
237
+ if not isinstance(bounds, Range):
238
+ raise QueryRefused("bounds must be a Range")
239
+ column = field(bounds.field)
240
+ if not column.type.startswith(
241
+ ("int32", "int64", "float32", "float64", "decimal(", "date", "timestamp")
242
+ ):
243
+ raise QueryRefused("this field has no range-read shape")
244
+ if bounds.low is None and bounds.high is None:
245
+ raise QueryRefused("a range needs at least one bound")
246
+ for operation, value in (("ge", bounds.low), ("lt", bounds.high)):
247
+ if value is not None:
248
+ filters.append(ReadFilter(column, operation, query_value(column, value)))
249
+ position = None
250
+ if after is not None:
251
+ if not paginate:
252
+ raise QueryRefused("an aggregate has no pagination position")
253
+ if not isinstance(after, Mapping) or set(after) != set(order_names):
254
+ raise QueryRefused("after must contain exactly the complete ordering key")
255
+ position = tuple(query_value(column, after[column.name]) for column in order)
256
+ return ReadPlan(tuple(columns), tuple(filters), order, descending, position, limit)
257
+
258
+
259
+ def _expression(column: ReadColumn, dialect: str) -> str:
260
+ expression = QUOTE[dialect](column.name)
261
+ if column.type == "string" and dialect == "postgres":
262
+ return f'({expression} COLLATE "C")'
263
+ if column.type == "uuid" and dialect == "clickhouse":
264
+ return f"toString({expression})"
265
+ if column.type in ("float32", "float64"):
266
+ return (
267
+ f"CAST({expression} AS double precision)"
268
+ if dialect == "postgres"
269
+ else f"toFloat64({expression})"
270
+ )
271
+ return expression
272
+
273
+
274
+ def _comparison(
275
+ column: ReadColumn,
276
+ operator: str,
277
+ value: Any,
278
+ dialect: str,
279
+ parameter: Callable[[Any], str],
280
+ ) -> str:
281
+ expression = _expression(column, dialect)
282
+ if value is None:
283
+ return f"{expression} IS NULL"
284
+ if column.type.startswith("decimal("):
285
+ _, scale_text = column.type[len("decimal(") : -1].split(",")
286
+ scale = max(int(scale_text), max(0, -value.as_tuple().exponent))
287
+ integer_digits = int(column.type[len("decimal(") : -1].split(",")[0]) - int(scale_text)
288
+ if integer_digits + scale > 76:
289
+ raise QueryRefused("decimal comparison requires more than 76 digits")
290
+ target = (
291
+ f"numeric(76,{scale})" if dialect == "postgres" else f"Nullable(Decimal(76,{scale}))"
292
+ )
293
+ expression = f"CAST({expression} AS {target})"
294
+ bound = f"CAST({parameter(format(value, 'f'))} AS {target})"
295
+ else:
296
+ bound = parameter(
297
+ str(value) if column.type == "uuid" and dialect == "clickhouse" else value
298
+ )
299
+ comparison = f"{expression} {operator} {bound}"
300
+ if column.type in ("float32", "float64") and operator != "=":
301
+ not_nan = (
302
+ f"{expression} <> 'NaN'::double precision"
303
+ if dialect == "postgres"
304
+ else f"NOT isNaN({expression})"
305
+ )
306
+ comparison = f"({not_nan} AND {comparison})"
307
+ return comparison
308
+
309
+
310
+ def read_sql(
311
+ table: str,
312
+ plan: ReadPlan,
313
+ *,
314
+ dialect: str,
315
+ parameter: Callable[[Any], str],
316
+ count: bool = False,
317
+ ) -> str:
318
+ """One native query. The parameter callback owns the driver's value representation."""
319
+ clauses = [
320
+ _comparison(
321
+ predicate.column,
322
+ {"eq": "=", "ge": ">=", "lt": "<"}[predicate.operation],
323
+ predicate.value,
324
+ dialect,
325
+ parameter,
326
+ )
327
+ for predicate in plan.filters
328
+ ]
329
+ if not count and plan.after is not None:
330
+ alternatives: list[str] = []
331
+ operator = "<" if plan.descending else ">"
332
+ for index, (column, value) in enumerate(zip(plan.order, plan.after, strict=True)):
333
+ if value is not None:
334
+ # Bind each occurrence in SQL order. Reusing a prefix would reuse a named
335
+ # ClickHouse parameter but leave PostgreSQL's positional parameters misaligned.
336
+ prefix = [
337
+ _comparison(previous, "=", plan.after[position], dialect, parameter)
338
+ for position, previous in enumerate(plan.order[:index])
339
+ ]
340
+ expression = _expression(column, dialect)
341
+ later = _comparison(column, operator, value, dialect, parameter)
342
+ alternatives.append(
343
+ "(" + " AND ".join([*prefix, f"({expression} IS NULL OR {later})"]) + ")"
344
+ )
345
+ clauses.append("(" + " OR ".join(alternatives) + ")" if alternatives else "FALSE")
346
+ where = " WHERE " + " AND ".join(clauses) if clauses else ""
347
+ quote = QUOTE[dialect]
348
+ final = " FINAL" if dialect == "clickhouse" else ""
349
+ if count:
350
+ expression = "CAST(count(*) AS text)" if dialect == "postgres" else "toString(count())"
351
+ return f"SELECT {expression} AS sde_count FROM {quote(table)}{final}{where}"
352
+ projection = ", ".join(
353
+ f"toTimeZone({quote(column.name)}, 'UTC') AS {quote(column.name)}"
354
+ if dialect == "clickhouse" and column.type in ("timestamp", "timestamptz")
355
+ else (
356
+ f"CAST({quote(column.name)} AS text) AS {quote(column.name)}"
357
+ if dialect == "postgres"
358
+ else f"toString({quote(column.name)}) AS {quote(column.name)}"
359
+ )
360
+ if column.type.startswith("decimal(")
361
+ else quote(column.name)
362
+ for column in plan.columns
363
+ )
364
+ direction = "DESC" if plan.descending else "ASC"
365
+ order = ", ".join(
366
+ f"{_expression(column, dialect)} {direction} NULLS LAST" for column in plan.order
367
+ )
368
+ return (
369
+ f"SELECT {projection} FROM {quote(table)}{final}{where} ORDER BY {order} "
370
+ f"LIMIT {parameter(plan.limit + 1)}"
371
+ )
372
+
373
+
374
+ @dataclass(frozen=True)
375
+ class NumericSummary:
376
+ count: int
377
+ non_null_count: int
378
+ minimum: int | Decimal | None
379
+ maximum: int | Decimal | None
380
+ total: int | Decimal | None
381
+ mean: Decimal | None
382
+
383
+
384
+ class Summarizable(Protocol):
385
+ def summarize_rows(
386
+ self,
387
+ table: str,
388
+ plan: ReadPlan,
389
+ column: ReadColumn,
390
+ ) -> Mapping[str, Any]: ...
391
+
392
+
393
+ def summary_engine(engine: object) -> Summarizable:
394
+ if not callable(getattr(engine, "summarize_rows", None)):
395
+ raise QueryRefused("this adapter does not support numeric summaries (summarize_rows)")
396
+ return cast(Summarizable, engine)
397
+
398
+
399
+ def summary_scale(column: ReadColumn, mean_scale: int) -> int:
400
+ if isinstance(mean_scale, bool) or not isinstance(mean_scale, int) or not 0 <= mean_scale <= 38:
401
+ raise QueryRefused("mean_scale must be an integer between 0 and 38")
402
+ if column.type in ("int32", "int64"):
403
+ return 0
404
+ if column.type.startswith("decimal("):
405
+ precision, scale = map(int, column.type[len("decimal(") : -1].split(","))
406
+ if precision <= 56:
407
+ return scale
408
+ raise QueryRefused("summaries require int32, int64 or decimal precision up to 56")
409
+
410
+
411
+ def summary_sql(
412
+ table: str,
413
+ plan: ReadPlan,
414
+ column: ReadColumn,
415
+ *,
416
+ dialect: str,
417
+ parameter: Callable[[Any], str],
418
+ ) -> str:
419
+ scale = summary_scale(column, 0)
420
+ quoted = QUOTE[dialect](column.name)
421
+ target = f"numeric(76,{scale})" if dialect == "postgres" else f"Nullable(Decimal(76,{scale}))"
422
+
423
+ def text(expression: str) -> str:
424
+ return f"CAST({expression} AS text)" if dialect == "postgres" else f"toString({expression})"
425
+
426
+ fields = [
427
+ text("count(*)") + " AS sde_count",
428
+ text(f"count({quoted})") + " AS sde_present",
429
+ text(f"min({quoted})") + " AS sde_min",
430
+ text(f"max({quoted})") + " AS sde_max",
431
+ text(f"sum(CAST({quoted} AS {target}))") + " AS sde_total",
432
+ ]
433
+ clauses = [
434
+ _comparison(
435
+ item.column,
436
+ {"eq": "=", "ge": ">=", "lt": "<"}[item.operation],
437
+ item.value,
438
+ dialect,
439
+ parameter,
440
+ )
441
+ for item in plan.filters
442
+ ]
443
+ where = " WHERE " + " AND ".join(clauses) if clauses else ""
444
+ final = " FINAL" if dialect == "clickhouse" else ""
445
+ return f"SELECT {', '.join(fields)} FROM {QUOTE[dialect](table)}{final}{where}"
446
+
447
+
448
+ def _decimal_integer(number: Decimal, scale: int) -> int:
449
+ sign, digits, exponent = number.as_tuple()
450
+ assert isinstance(exponent, int)
451
+ coefficient = 0
452
+ for digit in digits:
453
+ coefficient = coefficient * 10 + digit
454
+ shift = exponent + scale
455
+ if shift < 0:
456
+ divisor = int(10**-shift)
457
+ if coefficient % divisor:
458
+ raise ValueError("summary result has fractional digits outside the stored scale")
459
+ coefficient //= divisor
460
+ else:
461
+ coefficient *= int(10**shift)
462
+ return -coefficient if sign else coefficient
463
+
464
+
465
+ def numeric_summary(
466
+ record: Mapping[str, Any],
467
+ column: ReadColumn,
468
+ mean_scale: int,
469
+ ) -> NumericSummary:
470
+ """Decode exact server text and round a rational mean independently of decimal context."""
471
+ scale = summary_scale(column, mean_scale)
472
+ count, present = int(record["sde_count"]), int(record["sde_present"])
473
+ if not 0 <= present <= count:
474
+ raise ValueError("summary counts are inconsistent")
475
+ if present == 0:
476
+ return NumericSummary(count, 0, None, None, None, None)
477
+ minimum, maximum, total = (_decimal(record[key]) for key in ("sde_min", "sde_max", "sde_total"))
478
+ unscaled = _decimal_integer(total, scale)
479
+ numerator, denominator = abs(unscaled) * 10**mean_scale, present * 10**scale
480
+ rounded, remainder = divmod(numerator, denominator)
481
+ if remainder * 2 > denominator or (remainder * 2 == denominator and rounded % 2):
482
+ rounded += 1
483
+ mean = Decimal((int(unscaled < 0), tuple(int(digit) for digit in str(rounded)), -mean_scale))
484
+ if column.type in ("int32", "int64"):
485
+ return NumericSummary(
486
+ count,
487
+ present,
488
+ _decimal_integer(minimum, 0),
489
+ _decimal_integer(maximum, 0),
490
+ _decimal_integer(total, 0),
491
+ mean,
492
+ )
493
+
494
+ def scaled(value: Decimal) -> Decimal:
495
+ coefficient = _decimal_integer(value, scale)
496
+ return Decimal(
497
+ (
498
+ int(coefficient < 0),
499
+ tuple(int(digit) for digit in str(abs(coefficient))),
500
+ -scale,
501
+ )
502
+ )
503
+
504
+ return NumericSummary(count, present, scaled(minimum), scaled(maximum), scaled(total), mean)
505
+
506
+
507
+ def read_row(columns: Sequence[ReadColumn], row: Mapping[str, Any]) -> dict[str, Any]:
508
+ result = dict(row)
509
+ for column in columns:
510
+ value = result[column.name]
511
+ if value is not None and column.type.startswith("decimal("):
512
+ scale = int(column.type[len("decimal(") : -1].split(",")[1])
513
+ coefficient = _decimal_integer(_decimal(value), scale)
514
+ result[column.name] = Decimal(
515
+ (
516
+ int(coefficient < 0),
517
+ tuple(int(digit) for digit in str(abs(coefficient))),
518
+ -scale,
519
+ )
520
+ )
521
+ return result
sde/routing.py ADDED
@@ -0,0 +1,85 @@
1
+ """Routing: a lookup and three conditions, and deliberately nothing more.
2
+
3
+ The library does not decide where an operation goes. The planner decided, ahead of time, for every
4
+ shape the model admits, and put the answers in the placement map. This module reads them.
5
+
6
+ That division is the reason it is affordable to have four libraries. Decisions need telemetry,
7
+ history, a cost model and an explanation, and they need to be reproducible and testable - all of
8
+ which live on our side, once. If the library decided anything, that judgement would have to be
9
+ reimplemented in Python, TypeScript, Java and Rust, and kept identical in all four forever. So it
10
+ looks things up.
11
+
12
+ The three conditions that are *not* lookups exist because they are correctness, not judgement:
13
+
14
+ 1. Writes go to the source materialisation. There is exactly one, and derived copies are derived.
15
+ 2. An operation inside a transaction that has already written goes to the source, because a derived
16
+ copy is behind by design and would not show the write the caller just made.
17
+ 3. An operation asking for no staleness goes to the source, for the same reason.
18
+
19
+ Anything else follows the routing table, and if the table has nothing to say the answer is the
20
+ source. That fallback is safe by construction, since the source is always correct and merely
21
+ sometimes slower, and it is what makes a hand-written map a two-line affair rather than a table of
22
+ hashes.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from dataclasses import dataclass
28
+
29
+ from .logging import log
30
+ from .placement import Materialization, PlacementMap
31
+ from .shapes import WRITE_KINDS, OperationShape
32
+
33
+ __all__ = ["Router", "resolve"]
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Router:
38
+ """Resolves shapes against one placement map."""
39
+
40
+ placement: PlacementMap
41
+
42
+ def resolve(
43
+ self,
44
+ shape: OperationShape,
45
+ *,
46
+ in_write_transaction: bool = False,
47
+ fresh: bool = False,
48
+ ) -> Materialization:
49
+ group = self.placement.placement_of(shape.group)
50
+
51
+ if shape.kind in WRITE_KINDS:
52
+ return group.source
53
+ if in_write_transaction or fresh:
54
+ return group.source
55
+
56
+ target_id = self.placement.routing.get(shape.id)
57
+ if target_id is None:
58
+ # Not an error. A map with no routing table is the normal shape of a hand-written one,
59
+ # and the source is always a correct answer.
60
+ log("sde.route.fallback", group=shape.group, shape=shape.id, kind=shape.kind)
61
+ return group.source
62
+
63
+ materialization = group.by_id(target_id)
64
+ log(
65
+ "sde.route.resolved",
66
+ group=shape.group,
67
+ shape=shape.id,
68
+ kind=shape.kind,
69
+ materialization=materialization.id,
70
+ engine=materialization.engine,
71
+ )
72
+ return materialization
73
+
74
+
75
+ def resolve(
76
+ placement: PlacementMap,
77
+ shape: OperationShape,
78
+ *,
79
+ in_write_transaction: bool = False,
80
+ fresh: bool = False,
81
+ ) -> Materialization:
82
+ """Functional form, for the conformance vectors and for one-off resolution."""
83
+ return Router(placement).resolve(
84
+ shape, in_write_transaction=in_write_transaction, fresh=fresh
85
+ )