altcodepro-polydb-python 2.3.22__py3-none-any.whl → 2.3.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/METADATA +1 -1
- {altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/RECORD +10 -10
- polydb/adapters/AzureTableStorageAdapter.py +95 -109
- polydb/adapters/PostgreSQLAdapter.py +57 -5
- polydb/base/NoSQLKVAdapter.py +56 -0
- polydb/databaseFactory.py +82 -58
- polydb/models.py +34 -2
- {altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/WHEEL +0 -0
- {altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/top_level.txt +0 -0
{altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: altcodepro-polydb-python
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.24
|
|
4
4
|
Summary: Production-ready multi-cloud database abstraction layer with connection pooling, retry logic, and thread safety
|
|
5
5
|
Author: AltCodePro
|
|
6
6
|
Project-URL: Homepage, https://github.com/altcodepro/polydb-python
|
{altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/RECORD
RENAMED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
altcodepro_polydb_python-2.3.
|
|
1
|
+
altcodepro_polydb_python-2.3.24.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
|
|
2
2
|
polydb/PolyDB.py,sha256=MG7-nV59zDvrUQwXNEgV4eetHCeW9TYY9DQqtxu_r7k,23543
|
|
3
3
|
polydb/__init__.py,sha256=zx6yhNfaMDdshdDdcj4F8Vp9phCBA71mRFVi68IUELM,1459
|
|
4
4
|
polydb/advanced_query.py,sha256=cxMB-EB-qT3bWXJlhmjnMCUtrzogORWyoEfS50Dy7go,4280
|
|
5
5
|
polydb/batch.py,sha256=_DjWZa1ZXYSk6MLKqFe0eT7SYVRZtYNqZb9bI8Y2sao,4566
|
|
6
6
|
polydb/cache.py,sha256=JBXF1XEK-fY80ar8SDE893Z1Z116YtXAEG0PaJ0Nkcw,7658
|
|
7
7
|
polydb/cloudDatabaseFactory.py,sha256=Gp6L__YtgrkGahD8B7ItzXMHCoj2ZUGDjXLS9w0TujY,17780
|
|
8
|
-
polydb/databaseFactory.py,sha256=
|
|
8
|
+
polydb/databaseFactory.py,sha256=uBzX7K-k_aFHLQyFemy3XIe3lsOjAvjRFfaC6phITlA,41470
|
|
9
9
|
polydb/decorators.py,sha256=L_WP2uXP_k8Ac49SUm-mthbM4jWI-XYfHXEyKzumOww,43062
|
|
10
10
|
polydb/errors.py,sha256=jF_55fbfg6b6mBc116dzU_uBG9qiIKbr5jvmDuJx-Jw,1573
|
|
11
11
|
polydb/json_safe.py,sha256=R5PrqAGirqjYKPyy-8KH-lSXjLH0FPr2TSGozy4eheU,149
|
|
12
|
-
polydb/models.py,sha256=
|
|
12
|
+
polydb/models.py,sha256=GBdyGNzLu7skKNMp2cZySsM_peEHB3QhhbsnKAS15D8,8942
|
|
13
13
|
polydb/monitoring.py,sha256=UMm3ybyRJjAQi-prXXMLl9zuHhnhMnYBzMD3XWK66y8,9571
|
|
14
14
|
polydb/multitenancy.py,sha256=9kyY98RpKg8xDy9ejB_MyV_YzF7eZd4uxashw5S8vlg,6408
|
|
15
15
|
polydb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -24,7 +24,7 @@ polydb/validation.py,sha256=a1o1d02k3c6PWQwkBbw_0nEmIgrdB5RR8OcpNQMn4cA,4810
|
|
|
24
24
|
polydb/adapters/AzureBlobStorageAdapter.py,sha256=4vD55Z8DBTzBK66jIJbo5bNMY-AQ61MlP0-P2Fv_JgQ,7083
|
|
25
25
|
polydb/adapters/AzureFileStorageAdapter.py,sha256=VZNprqlBXCuWUgtqClNT-NrQmRf-XFYEiRA2BLbf-Sc,7046
|
|
26
26
|
polydb/adapters/AzureQueueAdapter.py,sha256=5PrKAX4OQxUD5nReZKrInF_mjQVdFcj2aYd0Xp-HjjQ,5254
|
|
27
|
-
polydb/adapters/AzureTableStorageAdapter.py,sha256=
|
|
27
|
+
polydb/adapters/AzureTableStorageAdapter.py,sha256=Gu7Xmi1Q79sUHNAEj6YMohF-Oy1Ar35d7OT1qUHKGOE,26198
|
|
28
28
|
polydb/adapters/BlockchainBlobAdapter.py,sha256=D01Yua9mkKfaQrxKYApblIyI6DSP0dtNAh4Tav51HJ4,3299
|
|
29
29
|
polydb/adapters/BlockchainFileAdapter.py,sha256=G749xOVpG20HuKS8zCgi6PMjoJNu-YXK7zitygjLdzM,8335
|
|
30
30
|
polydb/adapters/BlockchainKVAdapter.py,sha256=UFYHyTgvdW-sZUBqyHEG3Cdx6wSTiF2QowEVWL3XPTg,4564
|
|
@@ -36,7 +36,7 @@ polydb/adapters/GCPFilestoreAdapter.py,sha256=yjFQQwsWYWc8mo8XwMViVTWb5_D--xAyTM
|
|
|
36
36
|
polydb/adapters/GCPPubSubAdapter.py,sha256=7XNots2VA0ReEDku-rjg-OTYmftIpx5UgnXYDdXNkOo,8692
|
|
37
37
|
polydb/adapters/GCPStorageAdapter.py,sha256=9yS1Jhcn5_rCRdZ5uOqcRW6Ba-UNb6VOYpwENP-C6Qk,7133
|
|
38
38
|
polydb/adapters/MongoDBAdapter.py,sha256=vX3SAHDLbTnHABGesES9N-gYSQqPqdqFLJgd7pYWZzw,7471
|
|
39
|
-
polydb/adapters/PostgreSQLAdapter.py,sha256=
|
|
39
|
+
polydb/adapters/PostgreSQLAdapter.py,sha256=lUe65kpJkTu6R1Tc1zgl63Fa1PI44UVbWRTKRUSSGak,36579
|
|
40
40
|
polydb/adapters/S3Adapter.py,sha256=5R0zHAL2SkGFjp1L3bp-IU468bXYdSf6nKx974MN104,7586
|
|
41
41
|
polydb/adapters/S3CompatibleAdapter.py,sha256=jpafqbAjA8-irdXBrfXa1QJySIzrcUQ6UrFt5h5FAEc,7006
|
|
42
42
|
polydb/adapters/SQSAdapter.py,sha256=1vfbNoqIDy-b8t2xcxy91SoxSYBPFDfUh7yCQWxdS84,5778
|
|
@@ -50,7 +50,7 @@ polydb/audit/__init__.py,sha256=Z7-y5djq3glQ2Yun6nj-13Efpj3oGz9Qc0veS2g06Y4,245
|
|
|
50
50
|
polydb/audit/context.py,sha256=-A1FMtmr-2snVAHpTrVT80u-D_MCaqX6AoV4Ku2bz_o,1955
|
|
51
51
|
polydb/audit/manager.py,sha256=KzaaOf5bDfr4M-CkCAZBG_U_4xIBCKDLRAf3hsm-DAk,1236
|
|
52
52
|
polydb/audit/models.py,sha256=NapdH5dXU9GMoP9ccbDFaeMWvWbwoh2B5N7Nd3Ci6Cg,3745
|
|
53
|
-
polydb/base/NoSQLKVAdapter.py,sha256=
|
|
53
|
+
polydb/base/NoSQLKVAdapter.py,sha256=aYEnXG3PZVQbpneSgfToJAhJAsRHUxwazIvjRmT-Qak,13588
|
|
54
54
|
polydb/base/ObjectStorageAdapter.py,sha256=VeJ3qXET6H0xd3lJpE8-WSsKs8EyK9S0-9VNR0EJzME,2288
|
|
55
55
|
polydb/base/QueueAdapter.py,sha256=jFgyG-SUK4nhRNxm2NbzUbwnA9b_5iAC-ikLSUpXRwk,799
|
|
56
56
|
polydb/base/SharedFilesAdapter.py,sha256=kXbJmtn_cwEyAZ-1AvFrmesCLSwu43ycTV3S4BmwrO4,853
|
|
@@ -60,7 +60,7 @@ polydb/observability/logging.py,sha256=gv1SshNjaSkmR98Uv7Qbwq3gkeQUzVCA4ichqHKL6
|
|
|
60
60
|
polydb/services/__init__.py,sha256=nMolMXzmhejRSm7f42kU9fCHF0GIpqpejN2kywuBsUQ,241
|
|
61
61
|
polydb/services/compliance_service.py,sha256=-0ncX7PrXy4-NuyWXIv65cLUsqz0jXzb3LtsD8rlrXE,4983
|
|
62
62
|
polydb/services/security_service.py,sha256=q7GzK6sbMnBBsc4fuL4oXTd-ui2b-MYQyrWTBvdVNSk,5062
|
|
63
|
-
altcodepro_polydb_python-2.3.
|
|
64
|
-
altcodepro_polydb_python-2.3.
|
|
65
|
-
altcodepro_polydb_python-2.3.
|
|
66
|
-
altcodepro_polydb_python-2.3.
|
|
63
|
+
altcodepro_polydb_python-2.3.24.dist-info/METADATA,sha256=oChx1wdfnxF14DH2I2WvFTJWrceIEvoJ-88gzaCXlvY,12359
|
|
64
|
+
altcodepro_polydb_python-2.3.24.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
65
|
+
altcodepro_polydb_python-2.3.24.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
|
|
66
|
+
altcodepro_polydb_python-2.3.24.dist-info/RECORD,,
|
|
@@ -31,9 +31,6 @@ _BASE64_RE = re.compile(r"^[A-Za-z0-9+/]*={0,2}$")
|
|
|
31
31
|
# ensures model isolation across the same table
|
|
32
32
|
_MODEL_FIELD = "__polydb_model__"
|
|
33
33
|
|
|
34
|
-
# ─── NEW: HTTP transport tuning ──────────────────────────────────────────────
|
|
35
|
-
# Default Azure SDK has no read timeout. A sync call from an async event loop
|
|
36
|
-
# can park forever. These are overridable via env so ops can tune per env.
|
|
37
34
|
_CONNECT_TIMEOUT = float(os.getenv("AZURE_TABLE_CONNECT_TIMEOUT", "10"))
|
|
38
35
|
_READ_TIMEOUT = float(os.getenv("AZURE_TABLE_READ_TIMEOUT", "30"))
|
|
39
36
|
_RETRY_TOTAL = int(os.getenv("AZURE_TABLE_RETRY_TOTAL", "3"))
|
|
@@ -50,15 +47,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
50
47
|
- Blob overflow for entities > 1MB
|
|
51
48
|
- Model isolation using __polydb_model__
|
|
52
49
|
- Always returns id (derived from RowKey if missing)
|
|
53
|
-
|
|
54
|
-
Performance / safety guarantees:
|
|
55
|
-
- One TableServiceClient per adapter instance.
|
|
56
|
-
- One TableClient cached per table for the adapter's lifetime.
|
|
57
|
-
- create_table_if_not_exists runs ONCE per table per adapter lifetime.
|
|
58
|
-
- Explicit connect/read timeouts on the HTTP transport so a slow or
|
|
59
|
-
mis-configured endpoint fails fast instead of hanging the caller.
|
|
60
|
-
- No mutation of instance state inside read/write paths (was the
|
|
61
|
-
previous `self._table_client = ...` race hazard).
|
|
62
50
|
"""
|
|
63
51
|
|
|
64
52
|
AZURE_TABLE_MAX_SIZE = 60 * 1024 # 1MB
|
|
@@ -84,12 +72,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
84
72
|
self._blob_service = None
|
|
85
73
|
self._client_lock = threading.Lock()
|
|
86
74
|
|
|
87
|
-
|
|
88
|
-
# _ensured_tables: table names for which create_table_if_not_exists
|
|
89
|
-
# has already succeeded on this adapter instance.
|
|
90
|
-
# _table_clients_cache: TableClient handles by table name.
|
|
91
|
-
# _cache_lock: protects both caches from concurrent get/put.
|
|
92
|
-
self._ensured_tables: set[str] = set()
|
|
75
|
+
self._ensured_tables: set = set()
|
|
93
76
|
self._table_clients_cache: Dict[str, Any] = {}
|
|
94
77
|
self._cache_lock = threading.Lock()
|
|
95
78
|
|
|
@@ -102,11 +85,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
102
85
|
|
|
103
86
|
with self._client_lock:
|
|
104
87
|
if not self._client:
|
|
105
|
-
# ─── CHANGED: explicit transport timeouts + retry budget ──
|
|
106
|
-
# connection_timeout / read_timeout are honoured by the
|
|
107
|
-
# azure-core RequestsTransport. retry_total caps the
|
|
108
|
-
# SDK's built-in retry policy so a hard failure surfaces
|
|
109
|
-
# in seconds, not minutes.
|
|
110
88
|
self._client = TableServiceClient.from_connection_string(
|
|
111
89
|
self.connection_string,
|
|
112
90
|
connection_timeout=_CONNECT_TIMEOUT,
|
|
@@ -155,7 +133,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
155
133
|
if v is None:
|
|
156
134
|
return None
|
|
157
135
|
|
|
158
|
-
# Treat empty containers as absent — let the row omit the property.
|
|
159
136
|
if isinstance(v, (list, tuple, dict)) and len(v) == 0:
|
|
160
137
|
return None
|
|
161
138
|
|
|
@@ -187,14 +164,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
187
164
|
def _decode_value(self, v: Any) -> Any:
|
|
188
165
|
if isinstance(v, str):
|
|
189
166
|
if v.startswith(_JSON_PREFIX):
|
|
190
|
-
payload = v[len(_JSON_PREFIX)
|
|
167
|
+
payload = v[len(_JSON_PREFIX):]
|
|
191
168
|
try:
|
|
192
169
|
return json.loads(payload)
|
|
193
170
|
except Exception:
|
|
194
171
|
return v
|
|
195
172
|
|
|
196
173
|
if v.startswith(_BYTES_PREFIX):
|
|
197
|
-
payload = v[len(_BYTES_PREFIX)
|
|
174
|
+
payload = v[len(_BYTES_PREFIX):].strip()
|
|
198
175
|
if (len(payload) % 4) == 1:
|
|
199
176
|
return v
|
|
200
177
|
if not _BASE64_RE.match(payload):
|
|
@@ -248,11 +225,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
248
225
|
# Entity pack/unpack
|
|
249
226
|
# -----------------------------
|
|
250
227
|
def _pack_entity(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
|
|
251
|
-
"""Pack data into Azure Table Storage entity format.
|
|
252
|
-
|
|
253
|
-
This version is much simpler, more readable, and fixes the fragile
|
|
254
|
-
revmap/skey logic that was likely causing normal fields to disappear.
|
|
255
|
-
"""
|
|
256
228
|
entity: JsonDict = {
|
|
257
229
|
"PartitionKey": pk,
|
|
258
230
|
"RowKey": rk,
|
|
@@ -263,27 +235,21 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
263
235
|
for orig_key, orig_val in (data or {}).items():
|
|
264
236
|
orig_key_str = str(orig_key)
|
|
265
237
|
|
|
266
|
-
# Skip Azure-reserved or special keys
|
|
267
238
|
if orig_key_str in self._RESERVED or orig_key_str in ("PartitionKey", "RowKey"):
|
|
268
239
|
continue
|
|
269
240
|
|
|
270
|
-
# Sanitize property name so it is valid for Azure Table Storage
|
|
271
241
|
skey = self._sanitize_prop_name(orig_key)
|
|
272
242
|
|
|
273
|
-
# Prevent key collisions (extremely rare but safe)
|
|
274
243
|
base = skey
|
|
275
244
|
counter = 1
|
|
276
245
|
while skey in entity:
|
|
277
246
|
skey = f"{base}_{counter}"
|
|
278
247
|
counter += 1
|
|
279
248
|
|
|
280
|
-
# Remember the original key name so _unpack_entity can restore it
|
|
281
249
|
keymap[skey] = orig_key_str
|
|
282
250
|
|
|
283
|
-
# Encode the value (must return something Azure Table accepts)
|
|
284
251
|
entity[skey] = self._encode_value(orig_val)
|
|
285
252
|
|
|
286
|
-
# Store keymap only if we actually have fields (internal use)
|
|
287
253
|
if keymap:
|
|
288
254
|
entity["__keymap__"] = json.dumps(keymap, default=json_safe)
|
|
289
255
|
|
|
@@ -319,7 +285,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
319
285
|
if k in ("PartitionKey", "RowKey"):
|
|
320
286
|
continue
|
|
321
287
|
|
|
322
|
-
# keep internal metadata fields too
|
|
323
288
|
if k.startswith("_") or k in (_MODEL_FIELD,):
|
|
324
289
|
out[k] = v
|
|
325
290
|
continue
|
|
@@ -327,14 +292,12 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
327
292
|
orig_key = keymap.get(k, k)
|
|
328
293
|
out[orig_key] = self._decode_value(v)
|
|
329
294
|
|
|
330
|
-
# ✅ guarantee id for tests & ergonomics
|
|
331
295
|
if "id" not in out and rk is not None:
|
|
332
296
|
out["id"] = rk
|
|
333
297
|
|
|
334
298
|
return out
|
|
335
299
|
|
|
336
300
|
def _sanitize_blob_part(self, value: str) -> str:
|
|
337
|
-
# Blob-safe: lowercase, alphanumeric + dash only
|
|
338
301
|
s = str(value).lower()
|
|
339
302
|
s = re.sub(r"[^a-z0-9\-]", "-", s)
|
|
340
303
|
s = re.sub(r"-+", "-", s)
|
|
@@ -346,8 +309,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
346
309
|
def _blob_key(self, pk: str, rk: str, checksum: str) -> str:
|
|
347
310
|
safe_pk = self._sanitize_blob_part(pk)
|
|
348
311
|
safe_rk = self._sanitize_blob_part(rk)
|
|
349
|
-
|
|
350
|
-
# Flat structure (avoid deep paths)
|
|
351
312
|
return f"{safe_pk}-{safe_rk}-{checksum}.json"
|
|
352
313
|
|
|
353
314
|
def _blob_upload(self, blob_key: str, data_bytes: bytes):
|
|
@@ -375,18 +336,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
375
336
|
# Required NoSQLKVAdapter hooks
|
|
376
337
|
# -----------------------------
|
|
377
338
|
def _sanitize_table_name(self, name: str) -> str:
|
|
378
|
-
"""Convert collection_name to valid Azure Table name (alphanumeric only)."""
|
|
379
339
|
if not name:
|
|
380
340
|
return "defaulttable"
|
|
381
341
|
|
|
382
|
-
# Remove all invalid characters, keep only letters and numbers
|
|
383
342
|
sanitized = re.sub(r"[^a-zA-Z0-9]", "", name)
|
|
384
343
|
|
|
385
|
-
# Must start with a letter
|
|
386
344
|
if sanitized and sanitized[0].isdigit():
|
|
387
345
|
sanitized = "t" + sanitized
|
|
388
346
|
|
|
389
|
-
# Must be 3-63 characters
|
|
390
347
|
if len(sanitized) < 3:
|
|
391
348
|
sanitized = sanitized + "table"
|
|
392
349
|
if len(sanitized) > 63:
|
|
@@ -395,10 +352,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
395
352
|
return sanitized.lower()
|
|
396
353
|
|
|
397
354
|
def _get_table_name(self, model: type) -> str:
|
|
398
|
-
"""
|
|
399
|
-
Extract collection_name from UDL model definition.
|
|
400
|
-
"""
|
|
401
|
-
# Primary source: UDL definition
|
|
402
355
|
definition = getattr(model, "__udl_definition__", None)
|
|
403
356
|
if definition:
|
|
404
357
|
metadata = getattr(definition, "x_metadata", {}) or {}
|
|
@@ -406,41 +359,22 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
406
359
|
if collection_name:
|
|
407
360
|
return self._sanitize_table_name(collection_name)
|
|
408
361
|
|
|
409
|
-
# Fallback: __polydb__ metadata
|
|
410
362
|
polydb_meta = getattr(model, "__polydb__", None)
|
|
411
363
|
if isinstance(polydb_meta, dict):
|
|
412
364
|
collection = polydb_meta.get("collection") or polydb_meta.get("collection_name")
|
|
413
365
|
if collection:
|
|
414
366
|
return self._sanitize_table_name(str(collection))
|
|
415
367
|
|
|
416
|
-
# Last resort
|
|
417
368
|
return os.getenv("AZURE_TABLE_NAME", "defaulttable") or "defaulttable"
|
|
418
369
|
|
|
419
370
|
def _get_table_client(self, model: type):
|
|
420
|
-
"""
|
|
421
|
-
Return a TableClient for the model, ensuring the table exists.
|
|
422
|
-
|
|
423
|
-
─── CHANGED ────────────────────────────────────────────────────────
|
|
424
|
-
Both the table existence check AND the TableClient handle are now
|
|
425
|
-
cached per adapter instance. The previous version ran
|
|
426
|
-
create_table_if_not_exists on every read/write/query/delete — a
|
|
427
|
-
hot-loop seed of 100 records produced ~300 unnecessary network
|
|
428
|
-
round-trips. Now:
|
|
429
|
-
|
|
430
|
-
- First call for a table: 1 HTTP call (create_table_if_not_exists)
|
|
431
|
-
+ 1 cheap local TableClient construction.
|
|
432
|
-
- All subsequent calls: O(1) dict lookup, no network.
|
|
433
|
-
"""
|
|
434
371
|
table_name = self._get_table_name(model)
|
|
435
372
|
|
|
436
|
-
# Fast path: already ensured + client cached.
|
|
437
373
|
cached = self._table_clients_cache.get(table_name)
|
|
438
374
|
if cached is not None and table_name in self._ensured_tables:
|
|
439
375
|
return cached
|
|
440
376
|
|
|
441
377
|
with self._cache_lock:
|
|
442
|
-
# Re-check inside the lock to avoid duplicate ensures under
|
|
443
|
-
# concurrent first-touch.
|
|
444
378
|
cached = self._table_clients_cache.get(table_name)
|
|
445
379
|
already_ensured = table_name in self._ensured_tables
|
|
446
380
|
|
|
@@ -449,12 +383,9 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
449
383
|
self._client.create_table_if_not_exists(table_name)
|
|
450
384
|
logger.info(f"✅ Azure Table ensured/created: {table_name}")
|
|
451
385
|
except Exception as e:
|
|
452
|
-
# TableAlreadyExists is normal and safe to ignore
|
|
453
386
|
msg = str(e)
|
|
454
387
|
if "TableAlreadyExists" not in msg and "already exists" not in msg.lower():
|
|
455
388
|
logger.warning(f"Could not create table {table_name}: {e}")
|
|
456
|
-
# Mark as ensured regardless — either it exists now or we
|
|
457
|
-
# logged the failure; we won't retry on every op.
|
|
458
389
|
self._ensured_tables.add(table_name)
|
|
459
390
|
|
|
460
391
|
if cached is None:
|
|
@@ -464,19 +395,13 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
464
395
|
return cached
|
|
465
396
|
|
|
466
397
|
def _restore_overflow_properties(self, entity_dict: JsonDict) -> JsonDict:
|
|
467
|
-
"""Detect and restore any large properties stored in Blob Storage.
|
|
468
|
-
|
|
469
|
-
Called by both _get_raw and _query_raw.
|
|
470
|
-
"""
|
|
471
398
|
restored = {}
|
|
472
399
|
|
|
473
400
|
for k, v in entity_dict.items():
|
|
474
|
-
# Internal fields (starting with _) are kept as-is
|
|
475
401
|
if k.startswith("_"):
|
|
476
402
|
restored[k] = v
|
|
477
403
|
continue
|
|
478
404
|
|
|
479
|
-
# Is this an overflow metadata JSON string?
|
|
480
405
|
if isinstance(v, str) and v.startswith("{") and '"_overflow":' in v:
|
|
481
406
|
try:
|
|
482
407
|
metadata = json.loads(v)
|
|
@@ -491,10 +416,8 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
491
416
|
restored[k] = v
|
|
492
417
|
continue
|
|
493
418
|
|
|
494
|
-
# Download real value from blob
|
|
495
419
|
blob_data = self._blob_download(blob_key)
|
|
496
420
|
|
|
497
|
-
# Optional but very safe: checksum validation
|
|
498
421
|
if checksum:
|
|
499
422
|
actual_checksum = hashlib.md5(blob_data).hexdigest()
|
|
500
423
|
if actual_checksum != checksum:
|
|
@@ -502,7 +425,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
502
425
|
f"Checksum mismatch for blob {blob_key} (property '{k}')"
|
|
503
426
|
)
|
|
504
427
|
|
|
505
|
-
# Restore original value
|
|
506
428
|
actual_value = json.loads(blob_data.decode("utf-8"))
|
|
507
429
|
restored[k] = actual_value
|
|
508
430
|
logger.debug(f"Restored large property '{k}' from blob: {blob_key}")
|
|
@@ -511,11 +433,9 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
511
433
|
|
|
512
434
|
except Exception as e:
|
|
513
435
|
logger.error(f"Failed to restore overflow property '{k}': {e}")
|
|
514
|
-
# Fall back to raw metadata instead of crashing
|
|
515
436
|
restored[k] = v
|
|
516
437
|
continue
|
|
517
438
|
|
|
518
|
-
# Normal (non-overflow) property
|
|
519
439
|
restored[k] = v
|
|
520
440
|
|
|
521
441
|
return restored
|
|
@@ -523,20 +443,11 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
523
443
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
524
444
|
def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
|
|
525
445
|
try:
|
|
526
|
-
# ─── CHANGED: local variable, not self._table_client ────────────
|
|
527
|
-
# The old pattern `self._table_client = self._get_table_client(...)`
|
|
528
|
-
# was a race hazard: two concurrent writes on different models
|
|
529
|
-
# could swap each other's client mid-call. Local variable is
|
|
530
|
-
# safe and identical in cost since the client is now cached.
|
|
531
446
|
table_client = self._get_table_client(model)
|
|
532
447
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
533
448
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
534
|
-
# Pack entity (encoded for Azure Table)
|
|
535
449
|
entity = self._pack_entity(model, safe_pk, safe_rk, data)
|
|
536
|
-
|
|
537
|
-
# SIZE ESTIMATION (use ORIGINAL payload, not packed)
|
|
538
|
-
# ---------------------------------------------------
|
|
539
|
-
MAX_PROPERTY_CHARS = 30 * 1024 # ~30K safe under UTF-16 32K limit
|
|
450
|
+
MAX_PROPERTY_CHARS = 30 * 1024
|
|
540
451
|
|
|
541
452
|
def _is_large_string(val: Any) -> bool:
|
|
542
453
|
return isinstance(val, str) and len(val) > MAX_PROPERTY_CHARS
|
|
@@ -573,7 +484,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
573
484
|
continue
|
|
574
485
|
if k in large_val_dict:
|
|
575
486
|
metadata = large_val_dict[k]
|
|
576
|
-
reference_entity[k] = json.dumps(metadata, default=json_safe)
|
|
487
|
+
reference_entity[k] = json.dumps(metadata, default=json_safe)
|
|
577
488
|
logger.info(f"Overflow reference stored for {k} → {metadata['_blob_key']}")
|
|
578
489
|
else:
|
|
579
490
|
reference_entity[k] = v
|
|
@@ -596,17 +507,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
596
507
|
try:
|
|
597
508
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
598
509
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
599
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
600
510
|
table_client = self._get_table_client(model)
|
|
601
511
|
|
|
602
512
|
entity = table_client.get_entity(safe_pk, safe_rk)
|
|
603
513
|
entity_dict = dict(entity)
|
|
604
514
|
|
|
605
|
-
# Model isolation check
|
|
606
515
|
if entity_dict.get(_MODEL_FIELD) != model.__qualname__:
|
|
607
516
|
return None
|
|
608
517
|
|
|
609
|
-
# Restore any large fields that were moved to blob
|
|
610
518
|
restored_entity = self._restore_overflow_properties(entity_dict)
|
|
611
519
|
|
|
612
520
|
out = self._unpack_entity(restored_entity)
|
|
@@ -625,10 +533,8 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
625
533
|
self, model: type, filters: Dict[str, Any], limit: Optional[int]
|
|
626
534
|
) -> List[JsonDict]:
|
|
627
535
|
try:
|
|
628
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
629
536
|
table_client = self._get_table_client(model)
|
|
630
537
|
|
|
631
|
-
# Build query filter (your original logic kept unchanged)
|
|
632
538
|
eff_filters = dict(filters or {})
|
|
633
539
|
parts: List[str] = []
|
|
634
540
|
for orig_k, orig_v in eff_filters.items():
|
|
@@ -670,12 +576,10 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
670
576
|
for ent in entities:
|
|
671
577
|
ent_dict = dict(ent)
|
|
672
578
|
|
|
673
|
-
# Restore any large fields from blob
|
|
674
579
|
restored_entity = self._restore_overflow_properties(ent_dict)
|
|
675
580
|
|
|
676
581
|
out = self._unpack_entity(restored_entity)
|
|
677
582
|
|
|
678
|
-
# Guarantee 'id' field
|
|
679
583
|
if "id" not in out and "RowKey" in ent_dict:
|
|
680
584
|
out["id"] = ent_dict["RowKey"]
|
|
681
585
|
|
|
@@ -693,12 +597,10 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
693
597
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
694
598
|
def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
|
|
695
599
|
try:
|
|
696
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
697
600
|
table_client = self._get_table_client(model)
|
|
698
601
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
699
602
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
700
603
|
|
|
701
|
-
# read to check model + overflow
|
|
702
604
|
try:
|
|
703
605
|
entity = table_client.get_entity(safe_pk, safe_rk)
|
|
704
606
|
entity_dict = dict(entity)
|
|
@@ -719,16 +621,100 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
719
621
|
except Exception as e:
|
|
720
622
|
raise NoSQLError(f"Azure Table delete failed: {str(e)}")
|
|
721
623
|
|
|
624
|
+
# -----------------------------
|
|
625
|
+
# Pagination overrides
|
|
626
|
+
# -----------------------------
|
|
627
|
+
|
|
628
|
+
@property
|
|
629
|
+
def capabilities(self):
|
|
630
|
+
from ..models import BackendCapabilities
|
|
631
|
+
return BackendCapabilities(
|
|
632
|
+
server_order=False,
|
|
633
|
+
server_filter=True,
|
|
634
|
+
native_cursor=True,
|
|
635
|
+
supports_count=False,
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
def query_paged(self, model: type, request) -> "PageResult":
|
|
639
|
+
"""Use Azure native continuation tokens when no ORDER BY; fall back to in-memory sort."""
|
|
640
|
+
from ..models import PageResult
|
|
641
|
+
|
|
642
|
+
if request.order_by:
|
|
643
|
+
# Azure Table has no ORDER BY — fall back to in-memory sort via base class
|
|
644
|
+
return super().query_paged(model, request)
|
|
645
|
+
|
|
646
|
+
table_client = self._get_table_client(model)
|
|
647
|
+
eff_filters = dict(request.filters or {})
|
|
648
|
+
parts: List[str] = []
|
|
649
|
+
for orig_k, orig_v in eff_filters.items():
|
|
650
|
+
if orig_v is None:
|
|
651
|
+
continue
|
|
652
|
+
if orig_k in ("partition_key", "PartitionKey"):
|
|
653
|
+
sk = "PartitionKey"
|
|
654
|
+
elif orig_k in ("row_key", "RowKey"):
|
|
655
|
+
sk = "RowKey"
|
|
656
|
+
else:
|
|
657
|
+
sk = self._sanitize_prop_name(orig_k)
|
|
658
|
+
ev = self._encode_value(orig_v)
|
|
659
|
+
if ev is None:
|
|
660
|
+
parts.append(f"{sk} eq null")
|
|
661
|
+
elif isinstance(ev, bool):
|
|
662
|
+
parts.append(f"{sk} eq {str(ev).lower()}")
|
|
663
|
+
elif isinstance(ev, (int, float)):
|
|
664
|
+
parts.append(f"{sk} eq {ev}")
|
|
665
|
+
elif isinstance(ev, datetime):
|
|
666
|
+
iso = ev.isoformat()
|
|
667
|
+
if not iso.endswith("Z"):
|
|
668
|
+
iso += "Z"
|
|
669
|
+
parts.append(f"{sk} eq datetime'{iso}'")
|
|
670
|
+
else:
|
|
671
|
+
sval = str(ev).replace("'", "''")
|
|
672
|
+
parts.append(f"{sk} eq '{sval}'")
|
|
673
|
+
query_filter = " and ".join(parts) if parts else None
|
|
674
|
+
|
|
675
|
+
azure_ct = None
|
|
676
|
+
if request.cursor:
|
|
677
|
+
cd = self._decode_cursor(request.cursor)
|
|
678
|
+
if cd.get("type") == "azure":
|
|
679
|
+
azure_ct = cd.get("token")
|
|
680
|
+
|
|
681
|
+
try:
|
|
682
|
+
paged = table_client.query_entities(
|
|
683
|
+
query_filter=query_filter,
|
|
684
|
+
results_per_page=request.limit,
|
|
685
|
+
)
|
|
686
|
+
page_iter = paged.by_page(continuation_token=azure_ct)
|
|
687
|
+
try:
|
|
688
|
+
page = next(iter(page_iter))
|
|
689
|
+
items = []
|
|
690
|
+
for ent in page:
|
|
691
|
+
ent_dict = dict(ent)
|
|
692
|
+
restored = self._restore_overflow_properties(ent_dict)
|
|
693
|
+
out = self._unpack_entity(restored)
|
|
694
|
+
if "id" not in out and "RowKey" in ent_dict:
|
|
695
|
+
out["id"] = ent_dict["RowKey"]
|
|
696
|
+
items.append(out)
|
|
697
|
+
next_azure_ct = page_iter.continuation_token
|
|
698
|
+
except StopIteration:
|
|
699
|
+
items = []
|
|
700
|
+
next_azure_ct = None
|
|
701
|
+
except Exception as e:
|
|
702
|
+
raise NoSQLError(f"Azure Table paged query failed: {str(e)}")
|
|
703
|
+
|
|
704
|
+
if request.fields:
|
|
705
|
+
field_set = set(request.fields)
|
|
706
|
+
items = [{k: v for k, v in item.items() if k in field_set} for item in items]
|
|
707
|
+
|
|
708
|
+
next_cursor = None
|
|
709
|
+
if next_azure_ct:
|
|
710
|
+
next_cursor = self._encode_cursor({"type": "azure", "token": next_azure_ct})
|
|
711
|
+
|
|
712
|
+
return PageResult(items=items, next_cursor=next_cursor, has_more=bool(next_azure_ct))
|
|
713
|
+
|
|
722
714
|
# -----------------------------
|
|
723
715
|
# Lifecycle
|
|
724
716
|
# -----------------------------
|
|
725
717
|
def close(self) -> None:
|
|
726
|
-
"""Close cached clients. Safe to call multiple times.
|
|
727
|
-
|
|
728
|
-
─── NEW ─────────────────────────────────────────────────────────────
|
|
729
|
-
Hook this into PolyDB shutdown so HTTP sessions don't leak on
|
|
730
|
-
application exit. Caches are cleared so a re-init starts clean.
|
|
731
|
-
"""
|
|
732
718
|
with self._cache_lock:
|
|
733
719
|
for tc in self._table_clients_cache.values():
|
|
734
720
|
try:
|
|
@@ -6,6 +6,7 @@ from typing import Any, Iterator, List, Optional, Tuple, Union
|
|
|
6
6
|
import hashlib
|
|
7
7
|
from contextlib import contextmanager
|
|
8
8
|
import json
|
|
9
|
+
import base64
|
|
9
10
|
from decimal import Decimal
|
|
10
11
|
from datetime import datetime, date
|
|
11
12
|
|
|
@@ -16,7 +17,7 @@ from psycopg2.extras import Json
|
|
|
16
17
|
from ..errors import DatabaseError, ConnectionError, InsufficientBalanceError
|
|
17
18
|
from ..retry import retry
|
|
18
19
|
from ..utils import validate_table_name, validate_column_name
|
|
19
|
-
from ..query import QueryBuilder
|
|
20
|
+
from ..query import QueryBuilder, Operator
|
|
20
21
|
from ..types import JsonDict, Lookup
|
|
21
22
|
|
|
22
23
|
|
|
@@ -129,8 +130,6 @@ class PostgreSQLAdapter:
|
|
|
129
130
|
try:
|
|
130
131
|
conn = self._pool.getconn() # type: ignore
|
|
131
132
|
|
|
132
|
-
# Only ping if the connection is in an error state (closed/broken),
|
|
133
|
-
# not on every call. TCP keepalives handle staleness detection.
|
|
134
133
|
if conn.closed:
|
|
135
134
|
self.logger.warning("Closed connection detected from pool, closing and retrying")
|
|
136
135
|
self._pool.putconn(conn, close=True) # type: ignore
|
|
@@ -650,6 +649,61 @@ class PostgreSQLAdapter:
|
|
|
650
649
|
if own_conn and conn:
|
|
651
650
|
self._return_connection(conn)
|
|
652
651
|
|
|
652
|
+
# ---------------------------------------------------------------------
|
|
653
|
+
# PAGED QUERY (generic PageRequest / PageResult)
|
|
654
|
+
# ---------------------------------------------------------------------
|
|
655
|
+
|
|
656
|
+
@property
|
|
657
|
+
def capabilities(self):
|
|
658
|
+
from ..models import BackendCapabilities
|
|
659
|
+
return BackendCapabilities(
|
|
660
|
+
server_order=True,
|
|
661
|
+
server_filter=True,
|
|
662
|
+
native_cursor=False,
|
|
663
|
+
supports_count=True,
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
def query_paged(self, table: str, request, tx: Optional[Any] = None):
|
|
667
|
+
"""Server-side ORDER BY + LIMIT/OFFSET with opaque offset cursor."""
|
|
668
|
+
from ..models import PageResult
|
|
669
|
+
|
|
670
|
+
offset = 0
|
|
671
|
+
if request.cursor:
|
|
672
|
+
try:
|
|
673
|
+
cd = json.loads(base64.b64decode(request.cursor.encode()).decode())
|
|
674
|
+
offset = cd.get("offset", 0)
|
|
675
|
+
except Exception:
|
|
676
|
+
offset = 0
|
|
677
|
+
|
|
678
|
+
builder = QueryBuilder()
|
|
679
|
+
for k, v in (request.filters or {}).items():
|
|
680
|
+
builder.where(k, Operator.EQ, v)
|
|
681
|
+
if request.order_by:
|
|
682
|
+
builder.order_by(request.order_by, descending=request.order_desc)
|
|
683
|
+
builder.skip(offset)
|
|
684
|
+
builder.take(request.limit + 1)
|
|
685
|
+
if request.fields:
|
|
686
|
+
builder.select_fields(request.fields)
|
|
687
|
+
|
|
688
|
+
results = self.query_linq(table, builder, tx=tx)
|
|
689
|
+
if isinstance(results, int):
|
|
690
|
+
return PageResult(items=[], has_more=False)
|
|
691
|
+
|
|
692
|
+
has_more = len(results) > request.limit
|
|
693
|
+
if has_more:
|
|
694
|
+
results = results[:request.limit]
|
|
695
|
+
|
|
696
|
+
next_cursor = None
|
|
697
|
+
if has_more:
|
|
698
|
+
cursor_data = {
|
|
699
|
+
"offset": offset + request.limit,
|
|
700
|
+
"order_by": request.order_by,
|
|
701
|
+
"desc": request.order_desc,
|
|
702
|
+
}
|
|
703
|
+
next_cursor = base64.b64encode(json.dumps(cursor_data).encode()).decode()
|
|
704
|
+
|
|
705
|
+
return PageResult(items=results, next_cursor=next_cursor, has_more=has_more)
|
|
706
|
+
|
|
653
707
|
# ---------------------------------------------------------------------
|
|
654
708
|
# EXECUTE RAW SQL
|
|
655
709
|
# ---------------------------------------------------------------------
|
|
@@ -673,7 +727,6 @@ class PostgreSQLAdapter:
|
|
|
673
727
|
cursor = None
|
|
674
728
|
try:
|
|
675
729
|
cursor = conn.cursor()
|
|
676
|
-
# Log operation shape only, never parameter values
|
|
677
730
|
self.logger.debug("Executing raw SQL (%d params)", len(params or []))
|
|
678
731
|
exec_params = self._serialize_params(params or [])
|
|
679
732
|
self._timed_execute(cursor, sql, exec_params, operation="execute", table="")
|
|
@@ -913,7 +966,6 @@ class PostgreSQLAdapter:
|
|
|
913
966
|
|
|
914
967
|
# ---------------------------------------------------------------------
|
|
915
968
|
# SAVEPOINTS
|
|
916
|
-
# Security: use pg_sql.Identifier to prevent SQL injection via savepoint names.
|
|
917
969
|
# ---------------------------------------------------------------------
|
|
918
970
|
|
|
919
971
|
def begin_savepoint(self, name: str, tx: Any) -> None:
|
polydb/base/NoSQLKVAdapter.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# src/polydb/adapters/NoSQLKVAdapter.py
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import base64
|
|
4
5
|
import hashlib
|
|
5
6
|
import json
|
|
6
7
|
import threading
|
|
@@ -317,3 +318,58 @@ class NoSQLKVAdapter:
|
|
|
317
318
|
results = unique
|
|
318
319
|
|
|
319
320
|
return results
|
|
321
|
+
|
|
322
|
+
# ---------------------------------------------------------------
|
|
323
|
+
# Generic pagination API (opaque cursor, works for all NoSQL)
|
|
324
|
+
# ---------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def capabilities(self) -> "BackendCapabilities":
|
|
328
|
+
from ..models import BackendCapabilities
|
|
329
|
+
return BackendCapabilities()
|
|
330
|
+
|
|
331
|
+
def _encode_cursor(self, data: dict) -> str:
|
|
332
|
+
return base64.b64encode(json.dumps(data, default=json_safe).encode()).decode()
|
|
333
|
+
|
|
334
|
+
def _decode_cursor(self, cursor: str) -> dict:
|
|
335
|
+
try:
|
|
336
|
+
return json.loads(base64.b64decode(cursor.encode()).decode())
|
|
337
|
+
except Exception:
|
|
338
|
+
return {}
|
|
339
|
+
|
|
340
|
+
def query_paged(self, model: type, request: "PageRequest") -> "PageResult":
|
|
341
|
+
"""In-memory pagination with opaque offset cursor. Subclasses may override."""
|
|
342
|
+
from ..models import PageResult
|
|
343
|
+
|
|
344
|
+
offset = 0
|
|
345
|
+
if request.cursor:
|
|
346
|
+
cd = self._decode_cursor(request.cursor)
|
|
347
|
+
offset = cd.get("offset", 0)
|
|
348
|
+
|
|
349
|
+
builder = QueryBuilder()
|
|
350
|
+
for k, v in (request.filters or {}).items():
|
|
351
|
+
builder.where(k, Operator.EQ, v)
|
|
352
|
+
if request.order_by:
|
|
353
|
+
builder.order_by(request.order_by, descending=request.order_desc)
|
|
354
|
+
builder.skip(offset)
|
|
355
|
+
builder.take(request.limit + 1)
|
|
356
|
+
if request.fields:
|
|
357
|
+
builder.select_fields(request.fields)
|
|
358
|
+
|
|
359
|
+
results = self.query_linq(model, builder)
|
|
360
|
+
if isinstance(results, int):
|
|
361
|
+
return PageResult(items=[], has_more=False)
|
|
362
|
+
|
|
363
|
+
has_more = len(results) > request.limit
|
|
364
|
+
if has_more:
|
|
365
|
+
results = results[:request.limit]
|
|
366
|
+
|
|
367
|
+
next_cursor = None
|
|
368
|
+
if has_more:
|
|
369
|
+
next_cursor = self._encode_cursor({
|
|
370
|
+
"offset": offset + request.limit,
|
|
371
|
+
"order_by": request.order_by,
|
|
372
|
+
"desc": request.order_desc,
|
|
373
|
+
})
|
|
374
|
+
|
|
375
|
+
return PageResult(items=results, next_cursor=next_cursor, has_more=has_more)
|
polydb/databaseFactory.py
CHANGED
|
@@ -34,6 +34,7 @@ from .audit.manager import AuditManager
|
|
|
34
34
|
from .audit.context import AuditContext
|
|
35
35
|
from .query import Operator, QueryBuilder
|
|
36
36
|
from .cloudDatabaseFactory import CloudDatabaseFactory
|
|
37
|
+
from .models import PageRequest, PageResult
|
|
37
38
|
import re as _re
|
|
38
39
|
|
|
39
40
|
logger = logging.getLogger(__name__)
|
|
@@ -60,12 +61,7 @@ def _is_unique_violation(exc: BaseException) -> bool:
|
|
|
60
61
|
return any(m in s for m in _UNIQUE_VIOLATION_MARKERS)
|
|
61
62
|
|
|
62
63
|
|
|
63
|
-
def _parse_unique_violation_columns(exc: BaseException) -> list
|
|
64
|
-
"""
|
|
65
|
-
Pull the conflicting column names out of a Postgres unique-violation error.
|
|
66
|
-
Postgres formats them as: Key (col1, col2)=(val1, val2) already exists.
|
|
67
|
-
Returns [] if the message doesn't carry that detail.
|
|
68
|
-
"""
|
|
64
|
+
def _parse_unique_violation_columns(exc: BaseException) -> list:
|
|
69
65
|
m = _UNIQUE_KEY_RE.search(str(exc))
|
|
70
66
|
if not m:
|
|
71
67
|
return []
|
|
@@ -124,12 +120,6 @@ class _ResolvedAdapters:
|
|
|
124
120
|
|
|
125
121
|
|
|
126
122
|
def _extract_meta(model: Union[type, str]) -> ModelMeta:
|
|
127
|
-
"""
|
|
128
|
-
Extract storage metadata from model class.
|
|
129
|
-
|
|
130
|
-
If model is a string, return a default NoSQL meta (UDL resolves the
|
|
131
|
-
class before calling PolyDB, so string fallback is safe).
|
|
132
|
-
"""
|
|
133
123
|
if isinstance(model, type):
|
|
134
124
|
raw = getattr(model, "__polydb__", None)
|
|
135
125
|
if raw:
|
|
@@ -143,7 +133,6 @@ def _extract_meta(model: Union[type, str]) -> ModelMeta:
|
|
|
143
133
|
cache=raw.get("cache", False),
|
|
144
134
|
cache_ttl=raw.get("cache_ttl"),
|
|
145
135
|
)
|
|
146
|
-
# Default for dynamic/string models
|
|
147
136
|
return ModelMeta(storage="nosql", table=None, collection=None)
|
|
148
137
|
|
|
149
138
|
|
|
@@ -237,14 +226,13 @@ class DatabaseFactory:
|
|
|
237
226
|
self._engine_by_name: Dict[str, EngineConfig] = {e.name: e for e in self._engines}
|
|
238
227
|
self._provider_name = self._engines[0].cloud_factory.provider.value
|
|
239
228
|
|
|
240
|
-
#
|
|
229
|
+
# ────────────────────────────────────────────────────────────────────
|
|
241
230
|
# ENGINE ROUTING
|
|
242
|
-
#
|
|
231
|
+
# ────────────────────────────────────────────────────────────────────
|
|
243
232
|
|
|
244
233
|
def _resolve_adapters(
|
|
245
234
|
self, model_name: str, storage: str, override: Optional[EngineOverride] = None
|
|
246
235
|
) -> _ResolvedAdapters:
|
|
247
|
-
# 1. Per-call override
|
|
248
236
|
if override:
|
|
249
237
|
engine = self._engine_by_name.get(override.engine_name)
|
|
250
238
|
if engine is None:
|
|
@@ -255,7 +243,6 @@ class DatabaseFactory:
|
|
|
255
243
|
sql=engine.sql(), nosql=engine.nosql(), engine_name=engine.name
|
|
256
244
|
)
|
|
257
245
|
|
|
258
|
-
# 2. Explicit allow-list
|
|
259
246
|
for engine in self._engines:
|
|
260
247
|
if storage == "sql" and engine.sql_models and model_name in engine.sql_models:
|
|
261
248
|
return _ResolvedAdapters(
|
|
@@ -266,7 +253,6 @@ class DatabaseFactory:
|
|
|
266
253
|
sql=engine.sql(), nosql=engine.nosql(), engine_name=engine.name
|
|
267
254
|
)
|
|
268
255
|
|
|
269
|
-
# 3. Default fallback
|
|
270
256
|
for engine in self._engines:
|
|
271
257
|
if storage == "sql" and engine.is_default_sql:
|
|
272
258
|
return _ResolvedAdapters(
|
|
@@ -291,9 +277,9 @@ class DatabaseFactory:
|
|
|
291
277
|
storage = "sql" if (meta.storage == "sql" and meta.table) else "nosql"
|
|
292
278
|
return self._resolve_adapters(name, storage, override)
|
|
293
279
|
|
|
294
|
-
#
|
|
280
|
+
# ────────────────────────────────────────────────────────────────────
|
|
295
281
|
# ENGINE MANAGEMENT
|
|
296
|
-
#
|
|
282
|
+
# ────────────────────────────────────────────────────────────────────
|
|
297
283
|
|
|
298
284
|
def register_engine(self, engine: EngineConfig) -> None:
|
|
299
285
|
if engine.name in self._engine_by_name:
|
|
@@ -326,9 +312,9 @@ class DatabaseFactory:
|
|
|
326
312
|
return e.nosql()
|
|
327
313
|
return self._engines[0].nosql()
|
|
328
314
|
|
|
329
|
-
#
|
|
315
|
+
# ────────────────────────────────────────────────────────────────────
|
|
330
316
|
# HELPERS
|
|
331
|
-
#
|
|
317
|
+
# ────────────────────────────────────────────────────────────────────
|
|
332
318
|
|
|
333
319
|
def _inject_audit_fields(self, data: JsonDict, is_create: bool = False) -> JsonDict:
|
|
334
320
|
data = dict(data)
|
|
@@ -360,9 +346,9 @@ class DatabaseFactory:
|
|
|
360
346
|
return False
|
|
361
347
|
return meta.storage == "sql" and bool(meta.table)
|
|
362
348
|
|
|
363
|
-
#
|
|
349
|
+
# ────────────────────────────────────────────────────────────────────
|
|
364
350
|
# CREATE
|
|
365
|
-
#
|
|
351
|
+
# ────────────────────────────────────────────────────────────────────
|
|
366
352
|
|
|
367
353
|
def create(
|
|
368
354
|
self,
|
|
@@ -393,13 +379,8 @@ class DatabaseFactory:
|
|
|
393
379
|
except Exception as exc:
|
|
394
380
|
if not _is_unique_violation(exc):
|
|
395
381
|
raise
|
|
396
|
-
# Half-ran scenario / re-activation / replay. The record already
|
|
397
|
-
# exists with these unique-key columns. Preserve idempotent
|
|
398
|
-
# "create or update" semantics by routing to UPDATE keyed on the
|
|
399
|
-
# exact columns that conflicted (parsed from the Postgres error).
|
|
400
382
|
conflict_cols = _parse_unique_violation_columns(exc)
|
|
401
383
|
if not conflict_cols or not all(c in data for c in conflict_cols):
|
|
402
|
-
# Can't determine the conflict — re-raise so the caller sees it.
|
|
403
384
|
raise
|
|
404
385
|
where = {c: data[c] for c in conflict_cols}
|
|
405
386
|
logger.warning(
|
|
@@ -407,8 +388,6 @@ class DatabaseFactory:
|
|
|
407
388
|
meta.table,
|
|
408
389
|
conflict_cols,
|
|
409
390
|
)
|
|
410
|
-
# Drop the conflict columns from the UPDATE SET clause — they're
|
|
411
|
-
# already the matching key.
|
|
412
391
|
update_data = {k: v for k, v in data.items() if k not in conflict_cols}
|
|
413
392
|
result = adapters.sql.update(meta.table, where, update_data)
|
|
414
393
|
else:
|
|
@@ -426,7 +405,10 @@ class DatabaseFactory:
|
|
|
426
405
|
after_plain = self.encryption.decrypt_fields(result, encrypted_fields)
|
|
427
406
|
success = True
|
|
428
407
|
if self._enable_cache and self._cache:
|
|
429
|
-
|
|
408
|
+
try:
|
|
409
|
+
self._cache.invalidate(name)
|
|
410
|
+
except Exception as _ce:
|
|
411
|
+
logger.warning("Cache invalidate failed (non-fatal): %s", _ce)
|
|
430
412
|
return after_plain
|
|
431
413
|
|
|
432
414
|
try:
|
|
@@ -442,9 +424,9 @@ class DatabaseFactory:
|
|
|
442
424
|
except Exception:
|
|
443
425
|
raise
|
|
444
426
|
|
|
445
|
-
#
|
|
427
|
+
# ────────────────────────────────────────────────────────────────────
|
|
446
428
|
# READ
|
|
447
|
-
#
|
|
429
|
+
# ────────────────────────────────────────────────────────────────────
|
|
448
430
|
|
|
449
431
|
def read(
|
|
450
432
|
self,
|
|
@@ -484,10 +466,12 @@ class DatabaseFactory:
|
|
|
484
466
|
raw = [self.encryption.decrypt_fields(r, encrypted_fields) for r in raw]
|
|
485
467
|
if self._cache and use_external_cache and not no_cache:
|
|
486
468
|
ttl = cache_ttl or getattr(meta, "cache_ttl", 300)
|
|
487
|
-
|
|
469
|
+
try:
|
|
470
|
+
self._cache.set(name, query or {}, raw, ttl)
|
|
471
|
+
except Exception as _ce:
|
|
472
|
+
logger.warning("Cache set failed (non-fatal): %s", _ce)
|
|
488
473
|
return raw
|
|
489
474
|
|
|
490
|
-
# Check external cache first
|
|
491
475
|
if self._cache and use_external_cache and not no_cache:
|
|
492
476
|
cached = self._cache.get(name, query or {})
|
|
493
477
|
if cached is not None:
|
|
@@ -520,9 +504,9 @@ class DatabaseFactory:
|
|
|
520
504
|
)
|
|
521
505
|
return rows[0] if rows else None
|
|
522
506
|
|
|
523
|
-
#
|
|
507
|
+
# ────────────────────────────────────────────────────────────────────
|
|
524
508
|
# UPDATE
|
|
525
|
-
#
|
|
509
|
+
# ────────────────────────────────────────────────────────────────────
|
|
526
510
|
|
|
527
511
|
def update(
|
|
528
512
|
self,
|
|
@@ -591,7 +575,10 @@ class DatabaseFactory:
|
|
|
591
575
|
after_plain = self.encryption.decrypt_fields(result, encrypted_fields)
|
|
592
576
|
success = True
|
|
593
577
|
if self._enable_cache and self._cache:
|
|
594
|
-
|
|
578
|
+
try:
|
|
579
|
+
self._cache.invalidate(name)
|
|
580
|
+
except Exception as _ce:
|
|
581
|
+
logger.warning("Cache invalidate failed (non-fatal): %s", _ce)
|
|
595
582
|
return after_plain
|
|
596
583
|
|
|
597
584
|
try:
|
|
@@ -607,9 +594,9 @@ class DatabaseFactory:
|
|
|
607
594
|
except Exception:
|
|
608
595
|
raise
|
|
609
596
|
|
|
610
|
-
#
|
|
597
|
+
# ────────────────────────────────────────────────────────────────────
|
|
611
598
|
# UPSERT
|
|
612
|
-
#
|
|
599
|
+
# ────────────────────────────────────────────────────────────────────
|
|
613
600
|
|
|
614
601
|
def upsert(
|
|
615
602
|
self,
|
|
@@ -648,7 +635,10 @@ class DatabaseFactory:
|
|
|
648
635
|
after_plain = self.encryption.decrypt_fields(result, encrypted_fields)
|
|
649
636
|
success = True
|
|
650
637
|
if self._enable_cache and self._cache:
|
|
651
|
-
|
|
638
|
+
try:
|
|
639
|
+
self._cache.invalidate(name)
|
|
640
|
+
except Exception as _ce:
|
|
641
|
+
logger.warning("Cache invalidate failed (non-fatal): %s", _ce)
|
|
652
642
|
return after_plain
|
|
653
643
|
|
|
654
644
|
try:
|
|
@@ -664,9 +654,9 @@ class DatabaseFactory:
|
|
|
664
654
|
except Exception:
|
|
665
655
|
raise
|
|
666
656
|
|
|
667
|
-
#
|
|
657
|
+
# ────────────────────────────────────────────────────────────────────
|
|
668
658
|
# DELETE
|
|
669
|
-
#
|
|
659
|
+
# ────────────────────────────────────────────────────────────────────
|
|
670
660
|
|
|
671
661
|
def delete(
|
|
672
662
|
self,
|
|
@@ -715,7 +705,10 @@ class DatabaseFactory:
|
|
|
715
705
|
result = adapters.nosql.delete(cls, entity_id, etag=etag)
|
|
716
706
|
success = True
|
|
717
707
|
if self._enable_cache and self._cache:
|
|
718
|
-
|
|
708
|
+
try:
|
|
709
|
+
self._cache.invalidate(name)
|
|
710
|
+
except Exception as _ce:
|
|
711
|
+
logger.warning("Cache invalidate failed (non-fatal): %s", _ce)
|
|
719
712
|
return result
|
|
720
713
|
|
|
721
714
|
try:
|
|
@@ -731,9 +724,9 @@ class DatabaseFactory:
|
|
|
731
724
|
except Exception:
|
|
732
725
|
raise
|
|
733
726
|
|
|
734
|
-
#
|
|
727
|
+
# ────────────────────────────────────────────────────────────────────
|
|
735
728
|
# QUERY (LINQ-style)
|
|
736
|
-
#
|
|
729
|
+
# ────────────────────────────────────────────────────────────────────
|
|
737
730
|
|
|
738
731
|
def query_linq(
|
|
739
732
|
self,
|
|
@@ -765,9 +758,9 @@ class DatabaseFactory:
|
|
|
765
758
|
return result
|
|
766
759
|
return self._run(_op)
|
|
767
760
|
|
|
768
|
-
#
|
|
769
|
-
# PAGINATION
|
|
770
|
-
#
|
|
761
|
+
# ────────────────────────────────────────────────────────────────────
|
|
762
|
+
# PAGINATION (legacy simple)
|
|
763
|
+
# ────────────────────────────────────────────────────────────────────
|
|
771
764
|
|
|
772
765
|
def read_page(
|
|
773
766
|
self,
|
|
@@ -814,9 +807,40 @@ class DatabaseFactory:
|
|
|
814
807
|
return result
|
|
815
808
|
return self._run(_op)
|
|
816
809
|
|
|
817
|
-
#
|
|
810
|
+
# ────────────────────────────────────────────────────────────────────
|
|
811
|
+
# PAGINATION (generic — order_by, cursor, field projection)
|
|
812
|
+
# ────────────────────────────────────────────────────────────────────
|
|
813
|
+
|
|
814
|
+
def query_paged(
|
|
815
|
+
self,
|
|
816
|
+
model: Union[type, str],
|
|
817
|
+
request: PageRequest,
|
|
818
|
+
*,
|
|
819
|
+
engine_override: Optional[EngineOverride] = None,
|
|
820
|
+
) -> PageResult:
|
|
821
|
+
"""Unified pagination with order_by, opaque cursor, and field projection.
|
|
822
|
+
|
|
823
|
+
Routes to the best backend implementation:
|
|
824
|
+
- PostgreSQL: server-side ORDER BY + LIMIT/OFFSET
|
|
825
|
+
- Azure Table (no order_by): native continuation tokens
|
|
826
|
+
- Azure Table (with order_by) / all other NoSQL: in-memory sort + offset cursor
|
|
827
|
+
"""
|
|
828
|
+
name = _model_name(model)
|
|
829
|
+
meta = _extract_meta(model)
|
|
830
|
+
adapters = self._adapters_for(model, meta, engine_override)
|
|
831
|
+
|
|
832
|
+
if self._is_sql(meta, engine_override):
|
|
833
|
+
return adapters.sql.query_paged(meta.table, request)
|
|
834
|
+
|
|
835
|
+
cls = (
|
|
836
|
+
model if isinstance(model, type)
|
|
837
|
+
else type(name, (), {"__polydb__": meta.__dict__})
|
|
838
|
+
)
|
|
839
|
+
return adapters.nosql.query_paged(cls, request)
|
|
840
|
+
|
|
841
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
818
842
|
# BLOB STORAGE
|
|
819
|
-
#
|
|
843
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
820
844
|
|
|
821
845
|
def upload_blob(
|
|
822
846
|
self,
|
|
@@ -865,9 +889,9 @@ class DatabaseFactory:
|
|
|
865
889
|
)
|
|
866
890
|
return storage.list(prefix)
|
|
867
891
|
|
|
868
|
-
#
|
|
892
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
869
893
|
# QUEUE
|
|
870
|
-
#
|
|
894
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
871
895
|
|
|
872
896
|
def send_queue(
|
|
873
897
|
self,
|
|
@@ -914,9 +938,9 @@ class DatabaseFactory:
|
|
|
914
938
|
else queue.delete(message_id, queue_name)
|
|
915
939
|
)
|
|
916
940
|
|
|
917
|
-
#
|
|
941
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
918
942
|
# FILE STORAGE
|
|
919
|
-
#
|
|
943
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
920
944
|
|
|
921
945
|
def write_file(
|
|
922
946
|
self, path: str, data: Union[bytes, str], *, adapter_name: str = "files"
|
|
@@ -936,9 +960,9 @@ class DatabaseFactory:
|
|
|
936
960
|
files = self._engines[0].cloud_factory.get_files(adapter_name)
|
|
937
961
|
return files.list(directory)
|
|
938
962
|
|
|
939
|
-
#
|
|
963
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
940
964
|
# CACHE
|
|
941
|
-
#
|
|
965
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
942
966
|
|
|
943
967
|
def set_cache(self, model: str, key: Any, value: Any, ttl: int = 300) -> None:
|
|
944
968
|
if self._cache:
|
polydb/models.py
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
Data models and configurations
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
from dataclasses import dataclass
|
|
7
|
-
from typing import Optional, List, Callable, Any
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Optional, List, Callable, Any, Dict
|
|
8
8
|
from enum import Enum
|
|
9
9
|
|
|
10
10
|
|
|
@@ -43,6 +43,38 @@ class QueryOptions:
|
|
|
43
43
|
count_only: bool = False
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
@dataclass
|
|
47
|
+
class BackendCapabilities:
|
|
48
|
+
"""Describes what a storage backend can push down natively."""
|
|
49
|
+
|
|
50
|
+
server_order: bool = False
|
|
51
|
+
server_filter: bool = False
|
|
52
|
+
native_cursor: bool = False
|
|
53
|
+
supports_count: bool = True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class PageRequest:
|
|
58
|
+
"""Unified pagination request passed to query_paged()."""
|
|
59
|
+
|
|
60
|
+
filters: Optional[Dict[str, Any]] = None
|
|
61
|
+
order_by: Optional[str] = None
|
|
62
|
+
order_desc: bool = False
|
|
63
|
+
limit: int = 50
|
|
64
|
+
cursor: Optional[str] = None
|
|
65
|
+
fields: Optional[List[str]] = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class PageResult:
|
|
70
|
+
"""Unified pagination result returned by query_paged()."""
|
|
71
|
+
|
|
72
|
+
items: List[Any] = field(default_factory=list)
|
|
73
|
+
next_cursor: Optional[str] = None
|
|
74
|
+
has_more: bool = False
|
|
75
|
+
total: Optional[int] = None
|
|
76
|
+
|
|
77
|
+
|
|
46
78
|
# ============================================================
|
|
47
79
|
# CONFIG CLASSES (TYPED)
|
|
48
80
|
# ============================================================
|
{altcodepro_polydb_python-2.3.22.dist-info → altcodepro_polydb_python-2.3.24.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|