altcodepro-polydb-python 2.3.23__py3-none-any.whl → 2.3.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/METADATA +1 -1
- {altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/RECORD +11 -11
- {altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/WHEEL +1 -1
- polydb/__init__.py +3 -1
- polydb/adapters/AzureTableStorageAdapter.py +111 -110
- polydb/adapters/PostgreSQLAdapter.py +57 -5
- polydb/base/NoSQLKVAdapter.py +56 -0
- polydb/databaseFactory.py +62 -53
- polydb/models.py +69 -2
- {altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/top_level.txt +0 -0
{altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: altcodepro-polydb-python
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.25
|
|
4
4
|
Summary: Production-ready multi-cloud database abstraction layer with connection pooling, retry logic, and thread safety
|
|
5
5
|
Author: AltCodePro
|
|
6
6
|
Project-URL: Homepage, https://github.com/altcodepro/polydb-python
|
{altcodepro_polydb_python-2.3.23.dist-info → altcodepro_polydb_python-2.3.25.dist-info}/RECORD
RENAMED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
altcodepro_polydb_python-2.3.
|
|
1
|
+
altcodepro_polydb_python-2.3.25.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
|
|
2
2
|
polydb/PolyDB.py,sha256=MG7-nV59zDvrUQwXNEgV4eetHCeW9TYY9DQqtxu_r7k,23543
|
|
3
|
-
polydb/__init__.py,sha256=
|
|
3
|
+
polydb/__init__.py,sha256=d9KSz888A336904TPEjWPYBtyJLPB0lTdcaOnYp8byY,1535
|
|
4
4
|
polydb/advanced_query.py,sha256=cxMB-EB-qT3bWXJlhmjnMCUtrzogORWyoEfS50Dy7go,4280
|
|
5
5
|
polydb/batch.py,sha256=_DjWZa1ZXYSk6MLKqFe0eT7SYVRZtYNqZb9bI8Y2sao,4566
|
|
6
6
|
polydb/cache.py,sha256=JBXF1XEK-fY80ar8SDE893Z1Z116YtXAEG0PaJ0Nkcw,7658
|
|
7
7
|
polydb/cloudDatabaseFactory.py,sha256=Gp6L__YtgrkGahD8B7ItzXMHCoj2ZUGDjXLS9w0TujY,17780
|
|
8
|
-
polydb/databaseFactory.py,sha256=
|
|
8
|
+
polydb/databaseFactory.py,sha256=uBzX7K-k_aFHLQyFemy3XIe3lsOjAvjRFfaC6phITlA,41470
|
|
9
9
|
polydb/decorators.py,sha256=L_WP2uXP_k8Ac49SUm-mthbM4jWI-XYfHXEyKzumOww,43062
|
|
10
10
|
polydb/errors.py,sha256=jF_55fbfg6b6mBc116dzU_uBG9qiIKbr5jvmDuJx-Jw,1573
|
|
11
11
|
polydb/json_safe.py,sha256=R5PrqAGirqjYKPyy-8KH-lSXjLH0FPr2TSGozy4eheU,149
|
|
12
|
-
polydb/models.py,sha256=
|
|
12
|
+
polydb/models.py,sha256=pkGpYnEEJmXdoWVpWS2K3CnuO9JZ_91DKp1bpVNQCns,10259
|
|
13
13
|
polydb/monitoring.py,sha256=UMm3ybyRJjAQi-prXXMLl9zuHhnhMnYBzMD3XWK66y8,9571
|
|
14
14
|
polydb/multitenancy.py,sha256=9kyY98RpKg8xDy9ejB_MyV_YzF7eZd4uxashw5S8vlg,6408
|
|
15
15
|
polydb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -24,7 +24,7 @@ polydb/validation.py,sha256=a1o1d02k3c6PWQwkBbw_0nEmIgrdB5RR8OcpNQMn4cA,4810
|
|
|
24
24
|
polydb/adapters/AzureBlobStorageAdapter.py,sha256=4vD55Z8DBTzBK66jIJbo5bNMY-AQ61MlP0-P2Fv_JgQ,7083
|
|
25
25
|
polydb/adapters/AzureFileStorageAdapter.py,sha256=VZNprqlBXCuWUgtqClNT-NrQmRf-XFYEiRA2BLbf-Sc,7046
|
|
26
26
|
polydb/adapters/AzureQueueAdapter.py,sha256=5PrKAX4OQxUD5nReZKrInF_mjQVdFcj2aYd0Xp-HjjQ,5254
|
|
27
|
-
polydb/adapters/AzureTableStorageAdapter.py,sha256=
|
|
27
|
+
polydb/adapters/AzureTableStorageAdapter.py,sha256=CTi8qCVQDujqBGIfLJu2fgA2sNkDt07zrNIdyuQIcl4,27053
|
|
28
28
|
polydb/adapters/BlockchainBlobAdapter.py,sha256=D01Yua9mkKfaQrxKYApblIyI6DSP0dtNAh4Tav51HJ4,3299
|
|
29
29
|
polydb/adapters/BlockchainFileAdapter.py,sha256=G749xOVpG20HuKS8zCgi6PMjoJNu-YXK7zitygjLdzM,8335
|
|
30
30
|
polydb/adapters/BlockchainKVAdapter.py,sha256=UFYHyTgvdW-sZUBqyHEG3Cdx6wSTiF2QowEVWL3XPTg,4564
|
|
@@ -36,7 +36,7 @@ polydb/adapters/GCPFilestoreAdapter.py,sha256=yjFQQwsWYWc8mo8XwMViVTWb5_D--xAyTM
|
|
|
36
36
|
polydb/adapters/GCPPubSubAdapter.py,sha256=7XNots2VA0ReEDku-rjg-OTYmftIpx5UgnXYDdXNkOo,8692
|
|
37
37
|
polydb/adapters/GCPStorageAdapter.py,sha256=9yS1Jhcn5_rCRdZ5uOqcRW6Ba-UNb6VOYpwENP-C6Qk,7133
|
|
38
38
|
polydb/adapters/MongoDBAdapter.py,sha256=vX3SAHDLbTnHABGesES9N-gYSQqPqdqFLJgd7pYWZzw,7471
|
|
39
|
-
polydb/adapters/PostgreSQLAdapter.py,sha256=
|
|
39
|
+
polydb/adapters/PostgreSQLAdapter.py,sha256=lUe65kpJkTu6R1Tc1zgl63Fa1PI44UVbWRTKRUSSGak,36579
|
|
40
40
|
polydb/adapters/S3Adapter.py,sha256=5R0zHAL2SkGFjp1L3bp-IU468bXYdSf6nKx974MN104,7586
|
|
41
41
|
polydb/adapters/S3CompatibleAdapter.py,sha256=jpafqbAjA8-irdXBrfXa1QJySIzrcUQ6UrFt5h5FAEc,7006
|
|
42
42
|
polydb/adapters/SQSAdapter.py,sha256=1vfbNoqIDy-b8t2xcxy91SoxSYBPFDfUh7yCQWxdS84,5778
|
|
@@ -50,7 +50,7 @@ polydb/audit/__init__.py,sha256=Z7-y5djq3glQ2Yun6nj-13Efpj3oGz9Qc0veS2g06Y4,245
|
|
|
50
50
|
polydb/audit/context.py,sha256=-A1FMtmr-2snVAHpTrVT80u-D_MCaqX6AoV4Ku2bz_o,1955
|
|
51
51
|
polydb/audit/manager.py,sha256=KzaaOf5bDfr4M-CkCAZBG_U_4xIBCKDLRAf3hsm-DAk,1236
|
|
52
52
|
polydb/audit/models.py,sha256=NapdH5dXU9GMoP9ccbDFaeMWvWbwoh2B5N7Nd3Ci6Cg,3745
|
|
53
|
-
polydb/base/NoSQLKVAdapter.py,sha256=
|
|
53
|
+
polydb/base/NoSQLKVAdapter.py,sha256=aYEnXG3PZVQbpneSgfToJAhJAsRHUxwazIvjRmT-Qak,13588
|
|
54
54
|
polydb/base/ObjectStorageAdapter.py,sha256=VeJ3qXET6H0xd3lJpE8-WSsKs8EyK9S0-9VNR0EJzME,2288
|
|
55
55
|
polydb/base/QueueAdapter.py,sha256=jFgyG-SUK4nhRNxm2NbzUbwnA9b_5iAC-ikLSUpXRwk,799
|
|
56
56
|
polydb/base/SharedFilesAdapter.py,sha256=kXbJmtn_cwEyAZ-1AvFrmesCLSwu43ycTV3S4BmwrO4,853
|
|
@@ -60,7 +60,7 @@ polydb/observability/logging.py,sha256=gv1SshNjaSkmR98Uv7Qbwq3gkeQUzVCA4ichqHKL6
|
|
|
60
60
|
polydb/services/__init__.py,sha256=nMolMXzmhejRSm7f42kU9fCHF0GIpqpejN2kywuBsUQ,241
|
|
61
61
|
polydb/services/compliance_service.py,sha256=-0ncX7PrXy4-NuyWXIv65cLUsqz0jXzb3LtsD8rlrXE,4983
|
|
62
62
|
polydb/services/security_service.py,sha256=q7GzK6sbMnBBsc4fuL4oXTd-ui2b-MYQyrWTBvdVNSk,5062
|
|
63
|
-
altcodepro_polydb_python-2.3.
|
|
64
|
-
altcodepro_polydb_python-2.3.
|
|
65
|
-
altcodepro_polydb_python-2.3.
|
|
66
|
-
altcodepro_polydb_python-2.3.
|
|
63
|
+
altcodepro_polydb_python-2.3.25.dist-info/METADATA,sha256=jKo3HkAeo__dmf5vAvRukWvNcqqe1tIscypWPzcju3M,12359
|
|
64
|
+
altcodepro_polydb_python-2.3.25.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
65
|
+
altcodepro_polydb_python-2.3.25.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
|
|
66
|
+
altcodepro_polydb_python-2.3.25.dist-info/RECORD,,
|
polydb/__init__.py
CHANGED
|
@@ -8,7 +8,7 @@ __version__ = "2.2.3"
|
|
|
8
8
|
|
|
9
9
|
from .cloudDatabaseFactory import CloudDatabaseFactory
|
|
10
10
|
from .databaseFactory import DatabaseFactory
|
|
11
|
-
from .models import CloudProvider, PartitionConfig
|
|
11
|
+
from .models import CloudProvider, PartitionConfig, MongoConfig, CosmosMongoConfig
|
|
12
12
|
from .query import QueryBuilder, Operator
|
|
13
13
|
from .audit.context import AuditContext
|
|
14
14
|
from .cache import RedisCacheEngine as CacheEngine
|
|
@@ -36,6 +36,8 @@ __all__ = [
|
|
|
36
36
|
# Models & Config
|
|
37
37
|
"CloudProvider",
|
|
38
38
|
"PartitionConfig",
|
|
39
|
+
"MongoConfig",
|
|
40
|
+
"CosmosMongoConfig",
|
|
39
41
|
# Query
|
|
40
42
|
"QueryBuilder",
|
|
41
43
|
"Operator",
|
|
@@ -31,9 +31,6 @@ _BASE64_RE = re.compile(r"^[A-Za-z0-9+/]*={0,2}$")
|
|
|
31
31
|
# ensures model isolation across the same table
|
|
32
32
|
_MODEL_FIELD = "__polydb_model__"
|
|
33
33
|
|
|
34
|
-
# ─── NEW: HTTP transport tuning ──────────────────────────────────────────────
|
|
35
|
-
# Default Azure SDK has no read timeout. A sync call from an async event loop
|
|
36
|
-
# can park forever. These are overridable via env so ops can tune per env.
|
|
37
34
|
_CONNECT_TIMEOUT = float(os.getenv("AZURE_TABLE_CONNECT_TIMEOUT", "10"))
|
|
38
35
|
_READ_TIMEOUT = float(os.getenv("AZURE_TABLE_READ_TIMEOUT", "30"))
|
|
39
36
|
_RETRY_TOTAL = int(os.getenv("AZURE_TABLE_RETRY_TOTAL", "3"))
|
|
@@ -50,15 +47,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
50
47
|
- Blob overflow for entities > 1MB
|
|
51
48
|
- Model isolation using __polydb_model__
|
|
52
49
|
- Always returns id (derived from RowKey if missing)
|
|
53
|
-
|
|
54
|
-
Performance / safety guarantees:
|
|
55
|
-
- One TableServiceClient per adapter instance.
|
|
56
|
-
- One TableClient cached per table for the adapter's lifetime.
|
|
57
|
-
- create_table_if_not_exists runs ONCE per table per adapter lifetime.
|
|
58
|
-
- Explicit connect/read timeouts on the HTTP transport so a slow or
|
|
59
|
-
mis-configured endpoint fails fast instead of hanging the caller.
|
|
60
|
-
- No mutation of instance state inside read/write paths (was the
|
|
61
|
-
previous `self._table_client = ...` race hazard).
|
|
62
50
|
"""
|
|
63
51
|
|
|
64
52
|
AZURE_TABLE_MAX_SIZE = 60 * 1024 # 1MB
|
|
@@ -84,12 +72,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
84
72
|
self._blob_service = None
|
|
85
73
|
self._client_lock = threading.Lock()
|
|
86
74
|
|
|
87
|
-
|
|
88
|
-
# _ensured_tables: table names for which create_table_if_not_exists
|
|
89
|
-
# has already succeeded on this adapter instance.
|
|
90
|
-
# _table_clients_cache: TableClient handles by table name.
|
|
91
|
-
# _cache_lock: protects both caches from concurrent get/put.
|
|
92
|
-
self._ensured_tables: set[str] = set()
|
|
75
|
+
self._ensured_tables: set = set()
|
|
93
76
|
self._table_clients_cache: Dict[str, Any] = {}
|
|
94
77
|
self._cache_lock = threading.Lock()
|
|
95
78
|
|
|
@@ -102,11 +85,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
102
85
|
|
|
103
86
|
with self._client_lock:
|
|
104
87
|
if not self._client:
|
|
105
|
-
# ─── CHANGED: explicit transport timeouts + retry budget ──
|
|
106
|
-
# connection_timeout / read_timeout are honoured by the
|
|
107
|
-
# azure-core RequestsTransport. retry_total caps the
|
|
108
|
-
# SDK's built-in retry policy so a hard failure surfaces
|
|
109
|
-
# in seconds, not minutes.
|
|
110
88
|
self._client = TableServiceClient.from_connection_string(
|
|
111
89
|
self.connection_string,
|
|
112
90
|
connection_timeout=_CONNECT_TIMEOUT,
|
|
@@ -155,7 +133,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
155
133
|
if v is None:
|
|
156
134
|
return None
|
|
157
135
|
|
|
158
|
-
# Treat empty containers as absent — let the row omit the property.
|
|
159
136
|
if isinstance(v, (list, tuple, dict)) and len(v) == 0:
|
|
160
137
|
return None
|
|
161
138
|
|
|
@@ -187,14 +164,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
187
164
|
def _decode_value(self, v: Any) -> Any:
|
|
188
165
|
if isinstance(v, str):
|
|
189
166
|
if v.startswith(_JSON_PREFIX):
|
|
190
|
-
payload = v[len(_JSON_PREFIX)
|
|
167
|
+
payload = v[len(_JSON_PREFIX):]
|
|
191
168
|
try:
|
|
192
169
|
return json.loads(payload)
|
|
193
170
|
except Exception:
|
|
194
171
|
return v
|
|
195
172
|
|
|
196
173
|
if v.startswith(_BYTES_PREFIX):
|
|
197
|
-
payload = v[len(_BYTES_PREFIX)
|
|
174
|
+
payload = v[len(_BYTES_PREFIX):].strip()
|
|
198
175
|
if (len(payload) % 4) == 1:
|
|
199
176
|
return v
|
|
200
177
|
if not _BASE64_RE.match(payload):
|
|
@@ -248,11 +225,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
248
225
|
# Entity pack/unpack
|
|
249
226
|
# -----------------------------
|
|
250
227
|
def _pack_entity(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
|
|
251
|
-
"""Pack data into Azure Table Storage entity format.
|
|
252
|
-
|
|
253
|
-
This version is much simpler, more readable, and fixes the fragile
|
|
254
|
-
revmap/skey logic that was likely causing normal fields to disappear.
|
|
255
|
-
"""
|
|
256
228
|
entity: JsonDict = {
|
|
257
229
|
"PartitionKey": pk,
|
|
258
230
|
"RowKey": rk,
|
|
@@ -263,27 +235,21 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
263
235
|
for orig_key, orig_val in (data or {}).items():
|
|
264
236
|
orig_key_str = str(orig_key)
|
|
265
237
|
|
|
266
|
-
# Skip Azure-reserved or special keys
|
|
267
238
|
if orig_key_str in self._RESERVED or orig_key_str in ("PartitionKey", "RowKey"):
|
|
268
239
|
continue
|
|
269
240
|
|
|
270
|
-
# Sanitize property name so it is valid for Azure Table Storage
|
|
271
241
|
skey = self._sanitize_prop_name(orig_key)
|
|
272
242
|
|
|
273
|
-
# Prevent key collisions (extremely rare but safe)
|
|
274
243
|
base = skey
|
|
275
244
|
counter = 1
|
|
276
245
|
while skey in entity:
|
|
277
246
|
skey = f"{base}_{counter}"
|
|
278
247
|
counter += 1
|
|
279
248
|
|
|
280
|
-
# Remember the original key name so _unpack_entity can restore it
|
|
281
249
|
keymap[skey] = orig_key_str
|
|
282
250
|
|
|
283
|
-
# Encode the value (must return something Azure Table accepts)
|
|
284
251
|
entity[skey] = self._encode_value(orig_val)
|
|
285
252
|
|
|
286
|
-
# Store keymap only if we actually have fields (internal use)
|
|
287
253
|
if keymap:
|
|
288
254
|
entity["__keymap__"] = json.dumps(keymap, default=json_safe)
|
|
289
255
|
|
|
@@ -319,7 +285,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
319
285
|
if k in ("PartitionKey", "RowKey"):
|
|
320
286
|
continue
|
|
321
287
|
|
|
322
|
-
# keep internal metadata fields too
|
|
323
288
|
if k.startswith("_") or k in (_MODEL_FIELD,):
|
|
324
289
|
out[k] = v
|
|
325
290
|
continue
|
|
@@ -327,14 +292,12 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
327
292
|
orig_key = keymap.get(k, k)
|
|
328
293
|
out[orig_key] = self._decode_value(v)
|
|
329
294
|
|
|
330
|
-
# ✅ guarantee id for tests & ergonomics
|
|
331
295
|
if "id" not in out and rk is not None:
|
|
332
296
|
out["id"] = rk
|
|
333
297
|
|
|
334
298
|
return out
|
|
335
299
|
|
|
336
300
|
def _sanitize_blob_part(self, value: str) -> str:
|
|
337
|
-
# Blob-safe: lowercase, alphanumeric + dash only
|
|
338
301
|
s = str(value).lower()
|
|
339
302
|
s = re.sub(r"[^a-z0-9\-]", "-", s)
|
|
340
303
|
s = re.sub(r"-+", "-", s)
|
|
@@ -346,8 +309,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
346
309
|
def _blob_key(self, pk: str, rk: str, checksum: str) -> str:
|
|
347
310
|
safe_pk = self._sanitize_blob_part(pk)
|
|
348
311
|
safe_rk = self._sanitize_blob_part(rk)
|
|
349
|
-
|
|
350
|
-
# Flat structure (avoid deep paths)
|
|
351
312
|
return f"{safe_pk}-{safe_rk}-{checksum}.json"
|
|
352
313
|
|
|
353
314
|
def _blob_upload(self, blob_key: str, data_bytes: bytes):
|
|
@@ -375,18 +336,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
375
336
|
# Required NoSQLKVAdapter hooks
|
|
376
337
|
# -----------------------------
|
|
377
338
|
def _sanitize_table_name(self, name: str) -> str:
|
|
378
|
-
"""Convert collection_name to valid Azure Table name (alphanumeric only)."""
|
|
379
339
|
if not name:
|
|
380
340
|
return "defaulttable"
|
|
381
341
|
|
|
382
|
-
# Remove all invalid characters, keep only letters and numbers
|
|
383
342
|
sanitized = re.sub(r"[^a-zA-Z0-9]", "", name)
|
|
384
343
|
|
|
385
|
-
# Must start with a letter
|
|
386
344
|
if sanitized and sanitized[0].isdigit():
|
|
387
345
|
sanitized = "t" + sanitized
|
|
388
346
|
|
|
389
|
-
# Must be 3-63 characters
|
|
390
347
|
if len(sanitized) < 3:
|
|
391
348
|
sanitized = sanitized + "table"
|
|
392
349
|
if len(sanitized) > 63:
|
|
@@ -395,10 +352,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
395
352
|
return sanitized.lower()
|
|
396
353
|
|
|
397
354
|
def _get_table_name(self, model: type) -> str:
|
|
398
|
-
"""
|
|
399
|
-
Extract collection_name from UDL model definition.
|
|
400
|
-
"""
|
|
401
|
-
# Primary source: UDL definition
|
|
402
355
|
definition = getattr(model, "__udl_definition__", None)
|
|
403
356
|
if definition:
|
|
404
357
|
metadata = getattr(definition, "x_metadata", {}) or {}
|
|
@@ -406,41 +359,22 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
406
359
|
if collection_name:
|
|
407
360
|
return self._sanitize_table_name(collection_name)
|
|
408
361
|
|
|
409
|
-
# Fallback: __polydb__ metadata
|
|
410
362
|
polydb_meta = getattr(model, "__polydb__", None)
|
|
411
363
|
if isinstance(polydb_meta, dict):
|
|
412
364
|
collection = polydb_meta.get("collection") or polydb_meta.get("collection_name")
|
|
413
365
|
if collection:
|
|
414
366
|
return self._sanitize_table_name(str(collection))
|
|
415
367
|
|
|
416
|
-
# Last resort
|
|
417
368
|
return os.getenv("AZURE_TABLE_NAME", "defaulttable") or "defaulttable"
|
|
418
369
|
|
|
419
370
|
def _get_table_client(self, model: type):
|
|
420
|
-
"""
|
|
421
|
-
Return a TableClient for the model, ensuring the table exists.
|
|
422
|
-
|
|
423
|
-
─── CHANGED ────────────────────────────────────────────────────────
|
|
424
|
-
Both the table existence check AND the TableClient handle are now
|
|
425
|
-
cached per adapter instance. The previous version ran
|
|
426
|
-
create_table_if_not_exists on every read/write/query/delete — a
|
|
427
|
-
hot-loop seed of 100 records produced ~300 unnecessary network
|
|
428
|
-
round-trips. Now:
|
|
429
|
-
|
|
430
|
-
- First call for a table: 1 HTTP call (create_table_if_not_exists)
|
|
431
|
-
+ 1 cheap local TableClient construction.
|
|
432
|
-
- All subsequent calls: O(1) dict lookup, no network.
|
|
433
|
-
"""
|
|
434
371
|
table_name = self._get_table_name(model)
|
|
435
372
|
|
|
436
|
-
# Fast path: already ensured + client cached.
|
|
437
373
|
cached = self._table_clients_cache.get(table_name)
|
|
438
374
|
if cached is not None and table_name in self._ensured_tables:
|
|
439
375
|
return cached
|
|
440
376
|
|
|
441
377
|
with self._cache_lock:
|
|
442
|
-
# Re-check inside the lock to avoid duplicate ensures under
|
|
443
|
-
# concurrent first-touch.
|
|
444
378
|
cached = self._table_clients_cache.get(table_name)
|
|
445
379
|
already_ensured = table_name in self._ensured_tables
|
|
446
380
|
|
|
@@ -449,12 +383,9 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
449
383
|
self._client.create_table_if_not_exists(table_name)
|
|
450
384
|
logger.info(f"✅ Azure Table ensured/created: {table_name}")
|
|
451
385
|
except Exception as e:
|
|
452
|
-
# TableAlreadyExists is normal and safe to ignore
|
|
453
386
|
msg = str(e)
|
|
454
387
|
if "TableAlreadyExists" not in msg and "already exists" not in msg.lower():
|
|
455
388
|
logger.warning(f"Could not create table {table_name}: {e}")
|
|
456
|
-
# Mark as ensured regardless — either it exists now or we
|
|
457
|
-
# logged the failure; we won't retry on every op.
|
|
458
389
|
self._ensured_tables.add(table_name)
|
|
459
390
|
|
|
460
391
|
if cached is None:
|
|
@@ -464,19 +395,13 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
464
395
|
return cached
|
|
465
396
|
|
|
466
397
|
def _restore_overflow_properties(self, entity_dict: JsonDict) -> JsonDict:
|
|
467
|
-
"""Detect and restore any large properties stored in Blob Storage.
|
|
468
|
-
|
|
469
|
-
Called by both _get_raw and _query_raw.
|
|
470
|
-
"""
|
|
471
398
|
restored = {}
|
|
472
399
|
|
|
473
400
|
for k, v in entity_dict.items():
|
|
474
|
-
# Internal fields (starting with _) are kept as-is
|
|
475
401
|
if k.startswith("_"):
|
|
476
402
|
restored[k] = v
|
|
477
403
|
continue
|
|
478
404
|
|
|
479
|
-
# Is this an overflow metadata JSON string?
|
|
480
405
|
if isinstance(v, str) and v.startswith("{") and '"_overflow":' in v:
|
|
481
406
|
try:
|
|
482
407
|
metadata = json.loads(v)
|
|
@@ -491,10 +416,8 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
491
416
|
restored[k] = v
|
|
492
417
|
continue
|
|
493
418
|
|
|
494
|
-
# Download real value from blob
|
|
495
419
|
blob_data = self._blob_download(blob_key)
|
|
496
420
|
|
|
497
|
-
# Optional but very safe: checksum validation
|
|
498
421
|
if checksum:
|
|
499
422
|
actual_checksum = hashlib.md5(blob_data).hexdigest()
|
|
500
423
|
if actual_checksum != checksum:
|
|
@@ -502,7 +425,6 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
502
425
|
f"Checksum mismatch for blob {blob_key} (property '{k}')"
|
|
503
426
|
)
|
|
504
427
|
|
|
505
|
-
# Restore original value
|
|
506
428
|
actual_value = json.loads(blob_data.decode("utf-8"))
|
|
507
429
|
restored[k] = actual_value
|
|
508
430
|
logger.debug(f"Restored large property '{k}' from blob: {blob_key}")
|
|
@@ -511,11 +433,9 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
511
433
|
|
|
512
434
|
except Exception as e:
|
|
513
435
|
logger.error(f"Failed to restore overflow property '{k}': {e}")
|
|
514
|
-
# Fall back to raw metadata instead of crashing
|
|
515
436
|
restored[k] = v
|
|
516
437
|
continue
|
|
517
438
|
|
|
518
|
-
# Normal (non-overflow) property
|
|
519
439
|
restored[k] = v
|
|
520
440
|
|
|
521
441
|
return restored
|
|
@@ -523,20 +443,11 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
523
443
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
524
444
|
def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
|
|
525
445
|
try:
|
|
526
|
-
# ─── CHANGED: local variable, not self._table_client ────────────
|
|
527
|
-
# The old pattern `self._table_client = self._get_table_client(...)`
|
|
528
|
-
# was a race hazard: two concurrent writes on different models
|
|
529
|
-
# could swap each other's client mid-call. Local variable is
|
|
530
|
-
# safe and identical in cost since the client is now cached.
|
|
531
446
|
table_client = self._get_table_client(model)
|
|
532
447
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
533
448
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
534
|
-
# Pack entity (encoded for Azure Table)
|
|
535
449
|
entity = self._pack_entity(model, safe_pk, safe_rk, data)
|
|
536
|
-
|
|
537
|
-
# SIZE ESTIMATION (use ORIGINAL payload, not packed)
|
|
538
|
-
# ---------------------------------------------------
|
|
539
|
-
MAX_PROPERTY_CHARS = 30 * 1024 # ~30K safe under UTF-16 32K limit
|
|
450
|
+
MAX_PROPERTY_CHARS = 30 * 1024
|
|
540
451
|
|
|
541
452
|
def _is_large_string(val: Any) -> bool:
|
|
542
453
|
return isinstance(val, str) and len(val) > MAX_PROPERTY_CHARS
|
|
@@ -573,7 +484,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
573
484
|
continue
|
|
574
485
|
if k in large_val_dict:
|
|
575
486
|
metadata = large_val_dict[k]
|
|
576
|
-
reference_entity[k] = json.dumps(metadata, default=json_safe)
|
|
487
|
+
reference_entity[k] = json.dumps(metadata, default=json_safe)
|
|
577
488
|
logger.info(f"Overflow reference stored for {k} → {metadata['_blob_key']}")
|
|
578
489
|
else:
|
|
579
490
|
reference_entity[k] = v
|
|
@@ -596,17 +507,14 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
596
507
|
try:
|
|
597
508
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
598
509
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
599
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
600
510
|
table_client = self._get_table_client(model)
|
|
601
511
|
|
|
602
512
|
entity = table_client.get_entity(safe_pk, safe_rk)
|
|
603
513
|
entity_dict = dict(entity)
|
|
604
514
|
|
|
605
|
-
# Model isolation check
|
|
606
515
|
if entity_dict.get(_MODEL_FIELD) != model.__qualname__:
|
|
607
516
|
return None
|
|
608
517
|
|
|
609
|
-
# Restore any large fields that were moved to blob
|
|
610
518
|
restored_entity = self._restore_overflow_properties(entity_dict)
|
|
611
519
|
|
|
612
520
|
out = self._unpack_entity(restored_entity)
|
|
@@ -625,10 +533,8 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
625
533
|
self, model: type, filters: Dict[str, Any], limit: Optional[int]
|
|
626
534
|
) -> List[JsonDict]:
|
|
627
535
|
try:
|
|
628
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
629
536
|
table_client = self._get_table_client(model)
|
|
630
537
|
|
|
631
|
-
# Build query filter (your original logic kept unchanged)
|
|
632
538
|
eff_filters = dict(filters or {})
|
|
633
539
|
parts: List[str] = []
|
|
634
540
|
for orig_k, orig_v in eff_filters.items():
|
|
@@ -661,7 +567,18 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
661
567
|
sval = str(ev).replace("'", "''")
|
|
662
568
|
parts.append(f"{sk} eq '{sval}'")
|
|
663
569
|
|
|
664
|
-
|
|
570
|
+
# Empty-filter contract: if the caller passed filter keys but every
|
|
571
|
+
# condition dropped out (e.g. all values None/unresolvable), the query
|
|
572
|
+
# must match NOTHING — return [] (→ None for a single-object read).
|
|
573
|
+
# Returning all entities here (query_filter=None) is how an unmatched
|
|
574
|
+
# read_one accidentally returned a stray context row. Only a genuinely
|
|
575
|
+
# empty filter dict means "list all".
|
|
576
|
+
if not parts:
|
|
577
|
+
if eff_filters:
|
|
578
|
+
return []
|
|
579
|
+
query_filter = None
|
|
580
|
+
else:
|
|
581
|
+
query_filter = " and ".join(parts)
|
|
665
582
|
|
|
666
583
|
entities = table_client.query_entities(query_filter=query_filter)
|
|
667
584
|
|
|
@@ -670,12 +587,10 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
670
587
|
for ent in entities:
|
|
671
588
|
ent_dict = dict(ent)
|
|
672
589
|
|
|
673
|
-
# Restore any large fields from blob
|
|
674
590
|
restored_entity = self._restore_overflow_properties(ent_dict)
|
|
675
591
|
|
|
676
592
|
out = self._unpack_entity(restored_entity)
|
|
677
593
|
|
|
678
|
-
# Guarantee 'id' field
|
|
679
594
|
if "id" not in out and "RowKey" in ent_dict:
|
|
680
595
|
out["id"] = ent_dict["RowKey"]
|
|
681
596
|
|
|
@@ -693,12 +608,10 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
693
608
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
694
609
|
def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
|
|
695
610
|
try:
|
|
696
|
-
# ─── CHANGED: local variable ────────────────────────────────────
|
|
697
611
|
table_client = self._get_table_client(model)
|
|
698
612
|
safe_pk = self._sanitize_pk_rk(pk)
|
|
699
613
|
safe_rk = self._sanitize_pk_rk(rk)
|
|
700
614
|
|
|
701
|
-
# read to check model + overflow
|
|
702
615
|
try:
|
|
703
616
|
entity = table_client.get_entity(safe_pk, safe_rk)
|
|
704
617
|
entity_dict = dict(entity)
|
|
@@ -719,16 +632,104 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
719
632
|
except Exception as e:
|
|
720
633
|
raise NoSQLError(f"Azure Table delete failed: {str(e)}")
|
|
721
634
|
|
|
635
|
+
# -----------------------------
|
|
636
|
+
# Pagination overrides
|
|
637
|
+
# -----------------------------
|
|
638
|
+
|
|
639
|
+
@property
|
|
640
|
+
def capabilities(self):
|
|
641
|
+
from ..models import BackendCapabilities
|
|
642
|
+
return BackendCapabilities(
|
|
643
|
+
server_order=False,
|
|
644
|
+
server_filter=True,
|
|
645
|
+
native_cursor=True,
|
|
646
|
+
supports_count=False,
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
def query_paged(self, model: type, request) -> "PageResult":
|
|
650
|
+
"""Use Azure native continuation tokens when no ORDER BY; fall back to in-memory sort."""
|
|
651
|
+
from ..models import PageResult
|
|
652
|
+
|
|
653
|
+
if request.order_by:
|
|
654
|
+
# Azure Table has no ORDER BY — fall back to in-memory sort via base class
|
|
655
|
+
return super().query_paged(model, request)
|
|
656
|
+
|
|
657
|
+
table_client = self._get_table_client(model)
|
|
658
|
+
eff_filters = dict(request.filters or {})
|
|
659
|
+
parts: List[str] = []
|
|
660
|
+
for orig_k, orig_v in eff_filters.items():
|
|
661
|
+
if orig_v is None:
|
|
662
|
+
continue
|
|
663
|
+
if orig_k in ("partition_key", "PartitionKey"):
|
|
664
|
+
sk = "PartitionKey"
|
|
665
|
+
elif orig_k in ("row_key", "RowKey"):
|
|
666
|
+
sk = "RowKey"
|
|
667
|
+
else:
|
|
668
|
+
sk = self._sanitize_prop_name(orig_k)
|
|
669
|
+
ev = self._encode_value(orig_v)
|
|
670
|
+
if ev is None:
|
|
671
|
+
parts.append(f"{sk} eq null")
|
|
672
|
+
elif isinstance(ev, bool):
|
|
673
|
+
parts.append(f"{sk} eq {str(ev).lower()}")
|
|
674
|
+
elif isinstance(ev, (int, float)):
|
|
675
|
+
parts.append(f"{sk} eq {ev}")
|
|
676
|
+
elif isinstance(ev, datetime):
|
|
677
|
+
iso = ev.isoformat()
|
|
678
|
+
if not iso.endswith("Z"):
|
|
679
|
+
iso += "Z"
|
|
680
|
+
parts.append(f"{sk} eq datetime'{iso}'")
|
|
681
|
+
else:
|
|
682
|
+
sval = str(ev).replace("'", "''")
|
|
683
|
+
parts.append(f"{sk} eq '{sval}'")
|
|
684
|
+
# Same empty-filter contract as _query_raw: filter keys that all resolved
|
|
685
|
+
# to no condition must match NOTHING, not list the whole table.
|
|
686
|
+
if not parts and eff_filters:
|
|
687
|
+
return PageResult(items=[], next_cursor=None, has_more=False)
|
|
688
|
+
query_filter = " and ".join(parts) if parts else None
|
|
689
|
+
|
|
690
|
+
azure_ct = None
|
|
691
|
+
if request.cursor:
|
|
692
|
+
cd = self._decode_cursor(request.cursor)
|
|
693
|
+
if cd.get("type") == "azure":
|
|
694
|
+
azure_ct = cd.get("token")
|
|
695
|
+
|
|
696
|
+
try:
|
|
697
|
+
paged = table_client.query_entities(
|
|
698
|
+
query_filter=query_filter,
|
|
699
|
+
results_per_page=request.limit,
|
|
700
|
+
)
|
|
701
|
+
page_iter = paged.by_page(continuation_token=azure_ct)
|
|
702
|
+
try:
|
|
703
|
+
page = next(iter(page_iter))
|
|
704
|
+
items = []
|
|
705
|
+
for ent in page:
|
|
706
|
+
ent_dict = dict(ent)
|
|
707
|
+
restored = self._restore_overflow_properties(ent_dict)
|
|
708
|
+
out = self._unpack_entity(restored)
|
|
709
|
+
if "id" not in out and "RowKey" in ent_dict:
|
|
710
|
+
out["id"] = ent_dict["RowKey"]
|
|
711
|
+
items.append(out)
|
|
712
|
+
next_azure_ct = page_iter.continuation_token
|
|
713
|
+
except StopIteration:
|
|
714
|
+
items = []
|
|
715
|
+
next_azure_ct = None
|
|
716
|
+
except Exception as e:
|
|
717
|
+
raise NoSQLError(f"Azure Table paged query failed: {str(e)}")
|
|
718
|
+
|
|
719
|
+
if request.fields:
|
|
720
|
+
field_set = set(request.fields)
|
|
721
|
+
items = [{k: v for k, v in item.items() if k in field_set} for item in items]
|
|
722
|
+
|
|
723
|
+
next_cursor = None
|
|
724
|
+
if next_azure_ct:
|
|
725
|
+
next_cursor = self._encode_cursor({"type": "azure", "token": next_azure_ct})
|
|
726
|
+
|
|
727
|
+
return PageResult(items=items, next_cursor=next_cursor, has_more=bool(next_azure_ct))
|
|
728
|
+
|
|
722
729
|
# -----------------------------
|
|
723
730
|
# Lifecycle
|
|
724
731
|
# -----------------------------
|
|
725
732
|
def close(self) -> None:
|
|
726
|
-
"""Close cached clients. Safe to call multiple times.
|
|
727
|
-
|
|
728
|
-
─── NEW ─────────────────────────────────────────────────────────────
|
|
729
|
-
Hook this into PolyDB shutdown so HTTP sessions don't leak on
|
|
730
|
-
application exit. Caches are cleared so a re-init starts clean.
|
|
731
|
-
"""
|
|
732
733
|
with self._cache_lock:
|
|
733
734
|
for tc in self._table_clients_cache.values():
|
|
734
735
|
try:
|
|
@@ -6,6 +6,7 @@ from typing import Any, Iterator, List, Optional, Tuple, Union
|
|
|
6
6
|
import hashlib
|
|
7
7
|
from contextlib import contextmanager
|
|
8
8
|
import json
|
|
9
|
+
import base64
|
|
9
10
|
from decimal import Decimal
|
|
10
11
|
from datetime import datetime, date
|
|
11
12
|
|
|
@@ -16,7 +17,7 @@ from psycopg2.extras import Json
|
|
|
16
17
|
from ..errors import DatabaseError, ConnectionError, InsufficientBalanceError
|
|
17
18
|
from ..retry import retry
|
|
18
19
|
from ..utils import validate_table_name, validate_column_name
|
|
19
|
-
from ..query import QueryBuilder
|
|
20
|
+
from ..query import QueryBuilder, Operator
|
|
20
21
|
from ..types import JsonDict, Lookup
|
|
21
22
|
|
|
22
23
|
|
|
@@ -129,8 +130,6 @@ class PostgreSQLAdapter:
|
|
|
129
130
|
try:
|
|
130
131
|
conn = self._pool.getconn() # type: ignore
|
|
131
132
|
|
|
132
|
-
# Only ping if the connection is in an error state (closed/broken),
|
|
133
|
-
# not on every call. TCP keepalives handle staleness detection.
|
|
134
133
|
if conn.closed:
|
|
135
134
|
self.logger.warning("Closed connection detected from pool, closing and retrying")
|
|
136
135
|
self._pool.putconn(conn, close=True) # type: ignore
|
|
@@ -650,6 +649,61 @@ class PostgreSQLAdapter:
|
|
|
650
649
|
if own_conn and conn:
|
|
651
650
|
self._return_connection(conn)
|
|
652
651
|
|
|
652
|
+
# ---------------------------------------------------------------------
|
|
653
|
+
# PAGED QUERY (generic PageRequest / PageResult)
|
|
654
|
+
# ---------------------------------------------------------------------
|
|
655
|
+
|
|
656
|
+
@property
|
|
657
|
+
def capabilities(self):
|
|
658
|
+
from ..models import BackendCapabilities
|
|
659
|
+
return BackendCapabilities(
|
|
660
|
+
server_order=True,
|
|
661
|
+
server_filter=True,
|
|
662
|
+
native_cursor=False,
|
|
663
|
+
supports_count=True,
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
def query_paged(self, table: str, request, tx: Optional[Any] = None):
|
|
667
|
+
"""Server-side ORDER BY + LIMIT/OFFSET with opaque offset cursor."""
|
|
668
|
+
from ..models import PageResult
|
|
669
|
+
|
|
670
|
+
offset = 0
|
|
671
|
+
if request.cursor:
|
|
672
|
+
try:
|
|
673
|
+
cd = json.loads(base64.b64decode(request.cursor.encode()).decode())
|
|
674
|
+
offset = cd.get("offset", 0)
|
|
675
|
+
except Exception:
|
|
676
|
+
offset = 0
|
|
677
|
+
|
|
678
|
+
builder = QueryBuilder()
|
|
679
|
+
for k, v in (request.filters or {}).items():
|
|
680
|
+
builder.where(k, Operator.EQ, v)
|
|
681
|
+
if request.order_by:
|
|
682
|
+
builder.order_by(request.order_by, descending=request.order_desc)
|
|
683
|
+
builder.skip(offset)
|
|
684
|
+
builder.take(request.limit + 1)
|
|
685
|
+
if request.fields:
|
|
686
|
+
builder.select_fields(request.fields)
|
|
687
|
+
|
|
688
|
+
results = self.query_linq(table, builder, tx=tx)
|
|
689
|
+
if isinstance(results, int):
|
|
690
|
+
return PageResult(items=[], has_more=False)
|
|
691
|
+
|
|
692
|
+
has_more = len(results) > request.limit
|
|
693
|
+
if has_more:
|
|
694
|
+
results = results[:request.limit]
|
|
695
|
+
|
|
696
|
+
next_cursor = None
|
|
697
|
+
if has_more:
|
|
698
|
+
cursor_data = {
|
|
699
|
+
"offset": offset + request.limit,
|
|
700
|
+
"order_by": request.order_by,
|
|
701
|
+
"desc": request.order_desc,
|
|
702
|
+
}
|
|
703
|
+
next_cursor = base64.b64encode(json.dumps(cursor_data).encode()).decode()
|
|
704
|
+
|
|
705
|
+
return PageResult(items=results, next_cursor=next_cursor, has_more=has_more)
|
|
706
|
+
|
|
653
707
|
# ---------------------------------------------------------------------
|
|
654
708
|
# EXECUTE RAW SQL
|
|
655
709
|
# ---------------------------------------------------------------------
|
|
@@ -673,7 +727,6 @@ class PostgreSQLAdapter:
|
|
|
673
727
|
cursor = None
|
|
674
728
|
try:
|
|
675
729
|
cursor = conn.cursor()
|
|
676
|
-
# Log operation shape only, never parameter values
|
|
677
730
|
self.logger.debug("Executing raw SQL (%d params)", len(params or []))
|
|
678
731
|
exec_params = self._serialize_params(params or [])
|
|
679
732
|
self._timed_execute(cursor, sql, exec_params, operation="execute", table="")
|
|
@@ -913,7 +966,6 @@ class PostgreSQLAdapter:
|
|
|
913
966
|
|
|
914
967
|
# ---------------------------------------------------------------------
|
|
915
968
|
# SAVEPOINTS
|
|
916
|
-
# Security: use pg_sql.Identifier to prevent SQL injection via savepoint names.
|
|
917
969
|
# ---------------------------------------------------------------------
|
|
918
970
|
|
|
919
971
|
def begin_savepoint(self, name: str, tx: Any) -> None:
|
polydb/base/NoSQLKVAdapter.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# src/polydb/adapters/NoSQLKVAdapter.py
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import base64
|
|
4
5
|
import hashlib
|
|
5
6
|
import json
|
|
6
7
|
import threading
|
|
@@ -317,3 +318,58 @@ class NoSQLKVAdapter:
|
|
|
317
318
|
results = unique
|
|
318
319
|
|
|
319
320
|
return results
|
|
321
|
+
|
|
322
|
+
# ---------------------------------------------------------------
|
|
323
|
+
# Generic pagination API (opaque cursor, works for all NoSQL)
|
|
324
|
+
# ---------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def capabilities(self) -> "BackendCapabilities":
|
|
328
|
+
from ..models import BackendCapabilities
|
|
329
|
+
return BackendCapabilities()
|
|
330
|
+
|
|
331
|
+
def _encode_cursor(self, data: dict) -> str:
|
|
332
|
+
return base64.b64encode(json.dumps(data, default=json_safe).encode()).decode()
|
|
333
|
+
|
|
334
|
+
def _decode_cursor(self, cursor: str) -> dict:
|
|
335
|
+
try:
|
|
336
|
+
return json.loads(base64.b64decode(cursor.encode()).decode())
|
|
337
|
+
except Exception:
|
|
338
|
+
return {}
|
|
339
|
+
|
|
340
|
+
def query_paged(self, model: type, request: "PageRequest") -> "PageResult":
|
|
341
|
+
"""In-memory pagination with opaque offset cursor. Subclasses may override."""
|
|
342
|
+
from ..models import PageResult
|
|
343
|
+
|
|
344
|
+
offset = 0
|
|
345
|
+
if request.cursor:
|
|
346
|
+
cd = self._decode_cursor(request.cursor)
|
|
347
|
+
offset = cd.get("offset", 0)
|
|
348
|
+
|
|
349
|
+
builder = QueryBuilder()
|
|
350
|
+
for k, v in (request.filters or {}).items():
|
|
351
|
+
builder.where(k, Operator.EQ, v)
|
|
352
|
+
if request.order_by:
|
|
353
|
+
builder.order_by(request.order_by, descending=request.order_desc)
|
|
354
|
+
builder.skip(offset)
|
|
355
|
+
builder.take(request.limit + 1)
|
|
356
|
+
if request.fields:
|
|
357
|
+
builder.select_fields(request.fields)
|
|
358
|
+
|
|
359
|
+
results = self.query_linq(model, builder)
|
|
360
|
+
if isinstance(results, int):
|
|
361
|
+
return PageResult(items=[], has_more=False)
|
|
362
|
+
|
|
363
|
+
has_more = len(results) > request.limit
|
|
364
|
+
if has_more:
|
|
365
|
+
results = results[:request.limit]
|
|
366
|
+
|
|
367
|
+
next_cursor = None
|
|
368
|
+
if has_more:
|
|
369
|
+
next_cursor = self._encode_cursor({
|
|
370
|
+
"offset": offset + request.limit,
|
|
371
|
+
"order_by": request.order_by,
|
|
372
|
+
"desc": request.order_desc,
|
|
373
|
+
})
|
|
374
|
+
|
|
375
|
+
return PageResult(items=results, next_cursor=next_cursor, has_more=has_more)
|
polydb/databaseFactory.py
CHANGED
|
@@ -34,6 +34,7 @@ from .audit.manager import AuditManager
|
|
|
34
34
|
from .audit.context import AuditContext
|
|
35
35
|
from .query import Operator, QueryBuilder
|
|
36
36
|
from .cloudDatabaseFactory import CloudDatabaseFactory
|
|
37
|
+
from .models import PageRequest, PageResult
|
|
37
38
|
import re as _re
|
|
38
39
|
|
|
39
40
|
logger = logging.getLogger(__name__)
|
|
@@ -60,12 +61,7 @@ def _is_unique_violation(exc: BaseException) -> bool:
|
|
|
60
61
|
return any(m in s for m in _UNIQUE_VIOLATION_MARKERS)
|
|
61
62
|
|
|
62
63
|
|
|
63
|
-
def _parse_unique_violation_columns(exc: BaseException) -> list
|
|
64
|
-
"""
|
|
65
|
-
Pull the conflicting column names out of a Postgres unique-violation error.
|
|
66
|
-
Postgres formats them as: Key (col1, col2)=(val1, val2) already exists.
|
|
67
|
-
Returns [] if the message doesn't carry that detail.
|
|
68
|
-
"""
|
|
64
|
+
def _parse_unique_violation_columns(exc: BaseException) -> list:
|
|
69
65
|
m = _UNIQUE_KEY_RE.search(str(exc))
|
|
70
66
|
if not m:
|
|
71
67
|
return []
|
|
@@ -124,12 +120,6 @@ class _ResolvedAdapters:
|
|
|
124
120
|
|
|
125
121
|
|
|
126
122
|
def _extract_meta(model: Union[type, str]) -> ModelMeta:
|
|
127
|
-
"""
|
|
128
|
-
Extract storage metadata from model class.
|
|
129
|
-
|
|
130
|
-
If model is a string, return a default NoSQL meta (UDL resolves the
|
|
131
|
-
class before calling PolyDB, so string fallback is safe).
|
|
132
|
-
"""
|
|
133
123
|
if isinstance(model, type):
|
|
134
124
|
raw = getattr(model, "__polydb__", None)
|
|
135
125
|
if raw:
|
|
@@ -143,7 +133,6 @@ def _extract_meta(model: Union[type, str]) -> ModelMeta:
|
|
|
143
133
|
cache=raw.get("cache", False),
|
|
144
134
|
cache_ttl=raw.get("cache_ttl"),
|
|
145
135
|
)
|
|
146
|
-
# Default for dynamic/string models
|
|
147
136
|
return ModelMeta(storage="nosql", table=None, collection=None)
|
|
148
137
|
|
|
149
138
|
|
|
@@ -237,14 +226,13 @@ class DatabaseFactory:
|
|
|
237
226
|
self._engine_by_name: Dict[str, EngineConfig] = {e.name: e for e in self._engines}
|
|
238
227
|
self._provider_name = self._engines[0].cloud_factory.provider.value
|
|
239
228
|
|
|
240
|
-
#
|
|
229
|
+
# ────────────────────────────────────────────────────────────────────
|
|
241
230
|
# ENGINE ROUTING
|
|
242
|
-
#
|
|
231
|
+
# ────────────────────────────────────────────────────────────────────
|
|
243
232
|
|
|
244
233
|
def _resolve_adapters(
|
|
245
234
|
self, model_name: str, storage: str, override: Optional[EngineOverride] = None
|
|
246
235
|
) -> _ResolvedAdapters:
|
|
247
|
-
# 1. Per-call override
|
|
248
236
|
if override:
|
|
249
237
|
engine = self._engine_by_name.get(override.engine_name)
|
|
250
238
|
if engine is None:
|
|
@@ -255,7 +243,6 @@ class DatabaseFactory:
|
|
|
255
243
|
sql=engine.sql(), nosql=engine.nosql(), engine_name=engine.name
|
|
256
244
|
)
|
|
257
245
|
|
|
258
|
-
# 2. Explicit allow-list
|
|
259
246
|
for engine in self._engines:
|
|
260
247
|
if storage == "sql" and engine.sql_models and model_name in engine.sql_models:
|
|
261
248
|
return _ResolvedAdapters(
|
|
@@ -266,7 +253,6 @@ class DatabaseFactory:
|
|
|
266
253
|
sql=engine.sql(), nosql=engine.nosql(), engine_name=engine.name
|
|
267
254
|
)
|
|
268
255
|
|
|
269
|
-
# 3. Default fallback
|
|
270
256
|
for engine in self._engines:
|
|
271
257
|
if storage == "sql" and engine.is_default_sql:
|
|
272
258
|
return _ResolvedAdapters(
|
|
@@ -291,9 +277,9 @@ class DatabaseFactory:
|
|
|
291
277
|
storage = "sql" if (meta.storage == "sql" and meta.table) else "nosql"
|
|
292
278
|
return self._resolve_adapters(name, storage, override)
|
|
293
279
|
|
|
294
|
-
#
|
|
280
|
+
# ────────────────────────────────────────────────────────────────────
|
|
295
281
|
# ENGINE MANAGEMENT
|
|
296
|
-
#
|
|
282
|
+
# ────────────────────────────────────────────────────────────────────
|
|
297
283
|
|
|
298
284
|
def register_engine(self, engine: EngineConfig) -> None:
|
|
299
285
|
if engine.name in self._engine_by_name:
|
|
@@ -326,9 +312,9 @@ class DatabaseFactory:
|
|
|
326
312
|
return e.nosql()
|
|
327
313
|
return self._engines[0].nosql()
|
|
328
314
|
|
|
329
|
-
#
|
|
315
|
+
# ────────────────────────────────────────────────────────────────────
|
|
330
316
|
# HELPERS
|
|
331
|
-
#
|
|
317
|
+
# ────────────────────────────────────────────────────────────────────
|
|
332
318
|
|
|
333
319
|
def _inject_audit_fields(self, data: JsonDict, is_create: bool = False) -> JsonDict:
|
|
334
320
|
data = dict(data)
|
|
@@ -360,9 +346,9 @@ class DatabaseFactory:
|
|
|
360
346
|
return False
|
|
361
347
|
return meta.storage == "sql" and bool(meta.table)
|
|
362
348
|
|
|
363
|
-
#
|
|
349
|
+
# ────────────────────────────────────────────────────────────────────
|
|
364
350
|
# CREATE
|
|
365
|
-
#
|
|
351
|
+
# ────────────────────────────────────────────────────────────────────
|
|
366
352
|
|
|
367
353
|
def create(
|
|
368
354
|
self,
|
|
@@ -393,13 +379,8 @@ class DatabaseFactory:
|
|
|
393
379
|
except Exception as exc:
|
|
394
380
|
if not _is_unique_violation(exc):
|
|
395
381
|
raise
|
|
396
|
-
# Half-ran scenario / re-activation / replay. The record already
|
|
397
|
-
# exists with these unique-key columns. Preserve idempotent
|
|
398
|
-
# "create or update" semantics by routing to UPDATE keyed on the
|
|
399
|
-
# exact columns that conflicted (parsed from the Postgres error).
|
|
400
382
|
conflict_cols = _parse_unique_violation_columns(exc)
|
|
401
383
|
if not conflict_cols or not all(c in data for c in conflict_cols):
|
|
402
|
-
# Can't determine the conflict — re-raise so the caller sees it.
|
|
403
384
|
raise
|
|
404
385
|
where = {c: data[c] for c in conflict_cols}
|
|
405
386
|
logger.warning(
|
|
@@ -407,8 +388,6 @@ class DatabaseFactory:
|
|
|
407
388
|
meta.table,
|
|
408
389
|
conflict_cols,
|
|
409
390
|
)
|
|
410
|
-
# Drop the conflict columns from the UPDATE SET clause — they're
|
|
411
|
-
# already the matching key.
|
|
412
391
|
update_data = {k: v for k, v in data.items() if k not in conflict_cols}
|
|
413
392
|
result = adapters.sql.update(meta.table, where, update_data)
|
|
414
393
|
else:
|
|
@@ -445,9 +424,9 @@ class DatabaseFactory:
|
|
|
445
424
|
except Exception:
|
|
446
425
|
raise
|
|
447
426
|
|
|
448
|
-
#
|
|
427
|
+
# ────────────────────────────────────────────────────────────────────
|
|
449
428
|
# READ
|
|
450
|
-
#
|
|
429
|
+
# ────────────────────────────────────────────────────────────────────
|
|
451
430
|
|
|
452
431
|
def read(
|
|
453
432
|
self,
|
|
@@ -493,7 +472,6 @@ class DatabaseFactory:
|
|
|
493
472
|
logger.warning("Cache set failed (non-fatal): %s", _ce)
|
|
494
473
|
return raw
|
|
495
474
|
|
|
496
|
-
# Check external cache first
|
|
497
475
|
if self._cache and use_external_cache and not no_cache:
|
|
498
476
|
cached = self._cache.get(name, query or {})
|
|
499
477
|
if cached is not None:
|
|
@@ -526,9 +504,9 @@ class DatabaseFactory:
|
|
|
526
504
|
)
|
|
527
505
|
return rows[0] if rows else None
|
|
528
506
|
|
|
529
|
-
#
|
|
507
|
+
# ────────────────────────────────────────────────────────────────────
|
|
530
508
|
# UPDATE
|
|
531
|
-
#
|
|
509
|
+
# ────────────────────────────────────────────────────────────────────
|
|
532
510
|
|
|
533
511
|
def update(
|
|
534
512
|
self,
|
|
@@ -616,9 +594,9 @@ class DatabaseFactory:
|
|
|
616
594
|
except Exception:
|
|
617
595
|
raise
|
|
618
596
|
|
|
619
|
-
#
|
|
597
|
+
# ────────────────────────────────────────────────────────────────────
|
|
620
598
|
# UPSERT
|
|
621
|
-
#
|
|
599
|
+
# ────────────────────────────────────────────────────────────────────
|
|
622
600
|
|
|
623
601
|
def upsert(
|
|
624
602
|
self,
|
|
@@ -676,9 +654,9 @@ class DatabaseFactory:
|
|
|
676
654
|
except Exception:
|
|
677
655
|
raise
|
|
678
656
|
|
|
679
|
-
#
|
|
657
|
+
# ────────────────────────────────────────────────────────────────────
|
|
680
658
|
# DELETE
|
|
681
|
-
#
|
|
659
|
+
# ────────────────────────────────────────────────────────────────────
|
|
682
660
|
|
|
683
661
|
def delete(
|
|
684
662
|
self,
|
|
@@ -746,9 +724,9 @@ class DatabaseFactory:
|
|
|
746
724
|
except Exception:
|
|
747
725
|
raise
|
|
748
726
|
|
|
749
|
-
#
|
|
727
|
+
# ────────────────────────────────────────────────────────────────────
|
|
750
728
|
# QUERY (LINQ-style)
|
|
751
|
-
#
|
|
729
|
+
# ────────────────────────────────────────────────────────────────────
|
|
752
730
|
|
|
753
731
|
def query_linq(
|
|
754
732
|
self,
|
|
@@ -780,9 +758,9 @@ class DatabaseFactory:
|
|
|
780
758
|
return result
|
|
781
759
|
return self._run(_op)
|
|
782
760
|
|
|
783
|
-
#
|
|
784
|
-
# PAGINATION
|
|
785
|
-
#
|
|
761
|
+
# ────────────────────────────────────────────────────────────────────
|
|
762
|
+
# PAGINATION (legacy simple)
|
|
763
|
+
# ────────────────────────────────────────────────────────────────────
|
|
786
764
|
|
|
787
765
|
def read_page(
|
|
788
766
|
self,
|
|
@@ -829,9 +807,40 @@ class DatabaseFactory:
|
|
|
829
807
|
return result
|
|
830
808
|
return self._run(_op)
|
|
831
809
|
|
|
832
|
-
#
|
|
810
|
+
# ────────────────────────────────────────────────────────────────────
|
|
811
|
+
# PAGINATION (generic — order_by, cursor, field projection)
|
|
812
|
+
# ────────────────────────────────────────────────────────────────────
|
|
813
|
+
|
|
814
|
+
def query_paged(
|
|
815
|
+
self,
|
|
816
|
+
model: Union[type, str],
|
|
817
|
+
request: PageRequest,
|
|
818
|
+
*,
|
|
819
|
+
engine_override: Optional[EngineOverride] = None,
|
|
820
|
+
) -> PageResult:
|
|
821
|
+
"""Unified pagination with order_by, opaque cursor, and field projection.
|
|
822
|
+
|
|
823
|
+
Routes to the best backend implementation:
|
|
824
|
+
- PostgreSQL: server-side ORDER BY + LIMIT/OFFSET
|
|
825
|
+
- Azure Table (no order_by): native continuation tokens
|
|
826
|
+
- Azure Table (with order_by) / all other NoSQL: in-memory sort + offset cursor
|
|
827
|
+
"""
|
|
828
|
+
name = _model_name(model)
|
|
829
|
+
meta = _extract_meta(model)
|
|
830
|
+
adapters = self._adapters_for(model, meta, engine_override)
|
|
831
|
+
|
|
832
|
+
if self._is_sql(meta, engine_override):
|
|
833
|
+
return adapters.sql.query_paged(meta.table, request)
|
|
834
|
+
|
|
835
|
+
cls = (
|
|
836
|
+
model if isinstance(model, type)
|
|
837
|
+
else type(name, (), {"__polydb__": meta.__dict__})
|
|
838
|
+
)
|
|
839
|
+
return adapters.nosql.query_paged(cls, request)
|
|
840
|
+
|
|
841
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
833
842
|
# BLOB STORAGE
|
|
834
|
-
#
|
|
843
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
835
844
|
|
|
836
845
|
def upload_blob(
|
|
837
846
|
self,
|
|
@@ -880,9 +889,9 @@ class DatabaseFactory:
|
|
|
880
889
|
)
|
|
881
890
|
return storage.list(prefix)
|
|
882
891
|
|
|
883
|
-
#
|
|
892
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
884
893
|
# QUEUE
|
|
885
|
-
#
|
|
894
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
886
895
|
|
|
887
896
|
def send_queue(
|
|
888
897
|
self,
|
|
@@ -929,9 +938,9 @@ class DatabaseFactory:
|
|
|
929
938
|
else queue.delete(message_id, queue_name)
|
|
930
939
|
)
|
|
931
940
|
|
|
932
|
-
#
|
|
941
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
933
942
|
# FILE STORAGE
|
|
934
|
-
#
|
|
943
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
935
944
|
|
|
936
945
|
def write_file(
|
|
937
946
|
self, path: str, data: Union[bytes, str], *, adapter_name: str = "files"
|
|
@@ -951,9 +960,9 @@ class DatabaseFactory:
|
|
|
951
960
|
files = self._engines[0].cloud_factory.get_files(adapter_name)
|
|
952
961
|
return files.list(directory)
|
|
953
962
|
|
|
954
|
-
#
|
|
963
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
955
964
|
# CACHE
|
|
956
|
-
#
|
|
965
|
+
# ═══════════════════════════════════════════════════════════════════════════════
|
|
957
966
|
|
|
958
967
|
def set_cache(self, model: str, key: Any, value: Any, ttl: int = 300) -> None:
|
|
959
968
|
if self._cache:
|
polydb/models.py
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
Data models and configurations
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
from dataclasses import dataclass
|
|
7
|
-
from typing import Optional, List, Callable, Any
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Optional, List, Callable, Any, Dict
|
|
8
8
|
from enum import Enum
|
|
9
9
|
|
|
10
10
|
|
|
@@ -43,6 +43,38 @@ class QueryOptions:
|
|
|
43
43
|
count_only: bool = False
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
@dataclass
|
|
47
|
+
class BackendCapabilities:
|
|
48
|
+
"""Describes what a storage backend can push down natively."""
|
|
49
|
+
|
|
50
|
+
server_order: bool = False
|
|
51
|
+
server_filter: bool = False
|
|
52
|
+
native_cursor: bool = False
|
|
53
|
+
supports_count: bool = True
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class PageRequest:
|
|
58
|
+
"""Unified pagination request passed to query_paged()."""
|
|
59
|
+
|
|
60
|
+
filters: Optional[Dict[str, Any]] = None
|
|
61
|
+
order_by: Optional[str] = None
|
|
62
|
+
order_desc: bool = False
|
|
63
|
+
limit: int = 50
|
|
64
|
+
cursor: Optional[str] = None
|
|
65
|
+
fields: Optional[List[str]] = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class PageResult:
|
|
70
|
+
"""Unified pagination result returned by query_paged()."""
|
|
71
|
+
|
|
72
|
+
items: List[Any] = field(default_factory=list)
|
|
73
|
+
next_cursor: Optional[str] = None
|
|
74
|
+
has_more: bool = False
|
|
75
|
+
total: Optional[int] = None
|
|
76
|
+
|
|
77
|
+
|
|
46
78
|
# ============================================================
|
|
47
79
|
# CONFIG CLASSES (TYPED)
|
|
48
80
|
# ============================================================
|
|
@@ -291,3 +323,38 @@ class MongoConfig(StorageConfig):
|
|
|
291
323
|
super().__init__(CloudProvider.MONGODB, name)
|
|
292
324
|
self.mongo_uri = mongo_uri
|
|
293
325
|
self.db_name = db_name
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
class CosmosMongoConfig(MongoConfig):
|
|
329
|
+
"""Azure Cosmos DB via the **MongoDB API**.
|
|
330
|
+
|
|
331
|
+
Cosmos-for-Mongo speaks the MongoDB wire protocol, so it needs no adapter of
|
|
332
|
+
its own — it rides the existing ``MongoDBAdapter`` (and thus the whole polydb
|
|
333
|
+
stack) unchanged. This gives the best of both: Azure-native, serverless /
|
|
334
|
+
autoscale, managed billing + monitoring **and** portability — the same app
|
|
335
|
+
runs on MongoDB Atlas by swapping the connection string.
|
|
336
|
+
|
|
337
|
+
Pass a full ``connection_string`` (recommended — works for both RU and vCore),
|
|
338
|
+
or ``account`` + ``key`` to build the standard Cosmos-for-Mongo (RU) URI.
|
|
339
|
+
"""
|
|
340
|
+
|
|
341
|
+
def __init__(
|
|
342
|
+
self,
|
|
343
|
+
name: str = "cosmos",
|
|
344
|
+
*,
|
|
345
|
+
connection_string: str = "",
|
|
346
|
+
account: str = "",
|
|
347
|
+
key: str = "",
|
|
348
|
+
db_name: str = "",
|
|
349
|
+
port: int = 10255,
|
|
350
|
+
):
|
|
351
|
+
uri = connection_string
|
|
352
|
+
if not uri and account and key:
|
|
353
|
+
uri = (
|
|
354
|
+
f"mongodb://{account}:{key}@{account}.mongo.cosmos.azure.com:{port}/"
|
|
355
|
+
f"?ssl=true&replicaSet=globaldb&retrywrites=false"
|
|
356
|
+
f"&maxIdleTimeMS=120000&appName=@{account}@"
|
|
357
|
+
)
|
|
358
|
+
super().__init__(name=name, mongo_uri=uri, db_name=db_name)
|
|
359
|
+
self.is_cosmos = True
|
|
360
|
+
self.account = account
|
|
File without changes
|
|
File without changes
|