altcodepro-polydb-python 2.2.2__py3-none-any.whl → 2.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.2.2.dist-info → altcodepro_polydb_python-2.2.3.dist-info}/METADATA +2 -1
- altcodepro_polydb_python-2.2.3.dist-info/RECORD +54 -0
- polydb/__init__.py +2 -2
- polydb/adapters/AzureBlobStorageAdapter.py +103 -39
- polydb/adapters/AzureFileStorageAdapter.py +148 -43
- polydb/adapters/AzureQueueAdapter.py +96 -34
- polydb/adapters/AzureTableStorageAdapter.py +462 -119
- polydb/adapters/DynamoDBAdapter.py +463 -176
- polydb/adapters/FirestoreAdapter.py +320 -148
- polydb/adapters/GCPPubSubAdapter.py +217 -0
- polydb/adapters/GCPStorageAdapter.py +136 -37
- polydb/adapters/MongoDBAdapter.py +159 -39
- polydb/adapters/PostgreSQLAdapter.py +285 -83
- polydb/adapters/S3Adapter.py +122 -33
- polydb/adapters/SQSAdapter.py +121 -44
- polydb/adapters/VercelBlobAdapter.py +161 -0
- polydb/adapters/VercelKVAdapter.py +275 -283
- polydb/adapters/VercelQueueAdapter.py +61 -0
- polydb/audit/AuditStorage.py +1 -1
- polydb/base/NoSQLKVAdapter.py +113 -101
- polydb/base/ObjectStorageAdapter.py +23 -2
- polydb/base/QueueAdapter.py +2 -2
- polydb/base/SharedFilesAdapter.py +2 -2
- polydb/{factory.py → cloudDatabaseFactory.py} +49 -24
- polydb/databaseFactory.py +434 -101
- polydb/query.py +111 -42
- altcodepro_polydb_python-2.2.2.dist-info/RECORD +0 -52
- polydb/adapters/PubSubAdapter.py +0 -85
- {altcodepro_polydb_python-2.2.2.dist-info → altcodepro_polydb_python-2.2.3.dist-info}/WHEEL +0 -0
- {altcodepro_polydb_python-2.2.2.dist-info → altcodepro_polydb_python-2.2.3.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.2.2.dist-info → altcodepro_polydb_python-2.2.3.dist-info}/top_level.txt +0 -0
{altcodepro_polydb_python-2.2.2.dist-info → altcodepro_polydb_python-2.2.3.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: altcodepro-polydb-python
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.3
|
|
4
4
|
Summary: Production-ready multi-cloud database abstraction layer with connection pooling, retry logic, and thread safety
|
|
5
5
|
Author: AltCodePro
|
|
6
6
|
Project-URL: Homepage, https://github.com/altcodepro/polydb-python
|
|
@@ -74,6 +74,7 @@ Requires-Dist: isort>=7.0.0; extra == "dev"
|
|
|
74
74
|
Requires-Dist: mypy>=1.19.1; extra == "dev"
|
|
75
75
|
Provides-Extra: test
|
|
76
76
|
Requires-Dist: pytest>=9.0.2; extra == "test"
|
|
77
|
+
Requires-Dist: pytest-asyncio>=1.2.0; extra == "test"
|
|
77
78
|
Requires-Dist: pytest-cov>=7.0.0; extra == "test"
|
|
78
79
|
Requires-Dist: pytest-mock>=3.15.1; extra == "test"
|
|
79
80
|
Requires-Dist: moto>=5.1.21; extra == "test"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
altcodepro_polydb_python-2.2.3.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
|
|
2
|
+
polydb/__init__.py,sha256=UhUzfSvmMgKbV2tSME1ooIyfshIBi7_WyU4xl1tWWiA,1454
|
|
3
|
+
polydb/advanced_query.py,sha256=cxMB-EB-qT3bWXJlhmjnMCUtrzogORWyoEfS50Dy7go,4280
|
|
4
|
+
polydb/batch.py,sha256=_DjWZa1ZXYSk6MLKqFe0eT7SYVRZtYNqZb9bI8Y2sao,4566
|
|
5
|
+
polydb/cache.py,sha256=FGg5ln2GPhUpj3fljdJ4cygNYhdwJmfQhDhA_2MvOFY,6290
|
|
6
|
+
polydb/cloudDatabaseFactory.py,sha256=P0n0iy5mpjowzz2Kik-WFi33kR9iWFjC7xXsR6AV0VQ,4743
|
|
7
|
+
polydb/databaseFactory.py,sha256=N1BECm02ukmnhvHGujb1wD6X3cMEzJ2zamqDECQnMU0,40261
|
|
8
|
+
polydb/decorators.py,sha256=Rzk8Bj8wHi8YFtc06HEYT5r_Vqqn7TGaCtR5qvHdY-E,420
|
|
9
|
+
polydb/errors.py,sha256=rcFeBH0cenjJ86v0cmDc2Yjj4R019pLCBcTeSC4qps4,1428
|
|
10
|
+
polydb/json_safe.py,sha256=R5PrqAGirqjYKPyy-8KH-lSXjLH0FPr2TSGozy4eheU,149
|
|
11
|
+
polydb/models.py,sha256=HZYKB67ayoS1D4qxYfNLrUplbk7W-SIpexUnz4foyuQ,923
|
|
12
|
+
polydb/monitoring.py,sha256=pwR2p-sSlt6nA29lCAJFmdT0ODIyVQ9gSxB51hgaAbQ,10137
|
|
13
|
+
polydb/multitenancy.py,sha256=9kyY98RpKg8xDy9ejB_MyV_YzF7eZd4uxashw5S8vlg,6408
|
|
14
|
+
polydb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
polydb/query.py,sha256=L13FL8A06HxT0F_LGlY9IBh-62j1VWZrv2Tu_wS-Ed0,6348
|
|
16
|
+
polydb/registry.py,sha256=g-jLKq5jzDvdZ24weAqZHSxUIUaBsu1TZghxRMqbUXQ,1926
|
|
17
|
+
polydb/retry.py,sha256=etsj8MGo1WMvlcZMzWmFELAsWCRs-XPEuJe6K76QgbM,2548
|
|
18
|
+
polydb/schema.py,sha256=VrOayX6V6AD2Qh3-lm4ZVPTpI24e4V52IYheZf2rNQ4,5812
|
|
19
|
+
polydb/security.py,sha256=-bXdRjFmvq4X6ie6FrZMcO9ZbgjWFkNySSbRwFt1X1Q,16281
|
|
20
|
+
polydb/types.py,sha256=XB_85Un8_aWt4dSfpjIGotHbK3KBY2WurQGXr9EOxWY,2992
|
|
21
|
+
polydb/utils.py,sha256=G_ki5zKr5rGPgpFQM1CTq6twQd5OytaHKfet267MftM,1662
|
|
22
|
+
polydb/validation.py,sha256=a1o1d02k3c6PWQwkBbw_0nEmIgrdB5RR8OcpNQMn4cA,4810
|
|
23
|
+
polydb/adapters/AzureBlobStorageAdapter.py,sha256=U-wOph2p17F7r9OznjXh-JVMKZAWwaj8aamhevYyO78,4736
|
|
24
|
+
polydb/adapters/AzureFileStorageAdapter.py,sha256=OuZY5P-FTQ36954obJN65oSMqmW3d-7QBmXxVGX0lds,6086
|
|
25
|
+
polydb/adapters/AzureQueueAdapter.py,sha256=FR8uj8f6gtGVB92AeNvI-p-4PeyVvM1eezVbBp88TOE,4022
|
|
26
|
+
polydb/adapters/AzureTableStorageAdapter.py,sha256=dqbCCTxXsPvaOf-z9hJ0BfgE51DomyMJOdTN8tyyDsI,18498
|
|
27
|
+
polydb/adapters/DynamoDBAdapter.py,sha256=MSgIk43WT5z0fbBpZJ_5uLq0CfZS2YRfwX06-b_sl3o,18235
|
|
28
|
+
polydb/adapters/EFSAdapter.py,sha256=GFHXn2fjohXxVJaM4ptbisEs5bwiwkHeQ1Av_5ILiCA,1688
|
|
29
|
+
polydb/adapters/FirestoreAdapter.py,sha256=cvDOlxrEI8u81kWkjhazqb2rG4Q4I8Ftoho_cbx9nZQ,13947
|
|
30
|
+
polydb/adapters/GCPPubSubAdapter.py,sha256=0xb-6nzOokaIov-Kq3Dx8i_04GZYbL0XQsXqcSMnwnI,8143
|
|
31
|
+
polydb/adapters/GCPStorageAdapter.py,sha256=pIwBkEt_JNJG3P9nXXI_cLz49K2S7GXkxiFxc_8Dr5U,5733
|
|
32
|
+
polydb/adapters/MongoDBAdapter.py,sha256=vX3SAHDLbTnHABGesES9N-gYSQqPqdqFLJgd7pYWZzw,7471
|
|
33
|
+
polydb/adapters/PostgreSQLAdapter.py,sha256=qYrtDPxaIk_Q0W9DyXYvhbqbSLtheWuSjL1CKpVR8og,24852
|
|
34
|
+
polydb/adapters/S3Adapter.py,sha256=6ZgMvRfwqqk1g0aUCxEjmuVgL6rWEis2Gmf9kVRmVTM,5415
|
|
35
|
+
polydb/adapters/S3CompatibleAdapter.py,sha256=3hiOVEqyGbRX1rnpeldea8aWljSlWgCN3iE-g_Bks0I,3513
|
|
36
|
+
polydb/adapters/SQSAdapter.py,sha256=Ao9IOVLuDafgnnCD7DulGTeobw4tucxuswkx1dwyvSw,5200
|
|
37
|
+
polydb/adapters/VercelBlobAdapter.py,sha256=_azPYw-XBOENsdFIUiNJEaZpR4lmpfsyT7Jtr5iFZxA,5071
|
|
38
|
+
polydb/adapters/VercelKVAdapter.py,sha256=QZxRkuYzVNWFCEFaJPSph8YEAut-YtlXPqbCt0JlROI,8647
|
|
39
|
+
polydb/adapters/VercelQueueAdapter.py,sha256=s_Kpc6m9nIuvA3XbyEW8UH6PWm0-ckhth-taKc50LdQ,1732
|
|
40
|
+
polydb/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
+
polydb/audit/AuditStorage.py,sha256=A5HLhkWG8kef_caUuWakJf4fSK_r03NN4JLbbK7N55c,4949
|
|
42
|
+
polydb/audit/__init__.py,sha256=m_GE7gjLw00zfHX-1SpkF7QZpRE72HO699ZzKzqD3kc,244
|
|
43
|
+
polydb/audit/context.py,sha256=-A1FMtmr-2snVAHpTrVT80u-D_MCaqX6AoV4Ku2bz_o,1955
|
|
44
|
+
polydb/audit/manager.py,sha256=KzaaOf5bDfr4M-CkCAZBG_U_4xIBCKDLRAf3hsm-DAk,1236
|
|
45
|
+
polydb/audit/models.py,sha256=BgkSEQRbjbourxyGcEeJYIYzozwTM-pqTiSOM_BhWHs,2256
|
|
46
|
+
polydb/base/NoSQLKVAdapter.py,sha256=hiL0QyCV-5ChOyedkOk0ydH5jbtAtKir8IpJyIikPi4,11515
|
|
47
|
+
polydb/base/ObjectStorageAdapter.py,sha256=B4a_em2O5_dJ3DYrP04CvD-92Qgmfa4PVc7kGgeYDwg,1881
|
|
48
|
+
polydb/base/QueueAdapter.py,sha256=u-vi8t8On1NmCyJlgOE1NJ0T3-DpK_8DjRjGwr9Kme0,770
|
|
49
|
+
polydb/base/SharedFilesAdapter.py,sha256=hvmdNNhNxpN46Ob9RLAi8l46GB6JolYyZWnAMuaJ86g,708
|
|
50
|
+
polydb/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
+
altcodepro_polydb_python-2.2.3.dist-info/METADATA,sha256=zEJKZXzcgWztRjKd_kUyIvtHuSWcrtC58OELzu2u62c,10920
|
|
52
|
+
altcodepro_polydb_python-2.2.3.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
|
|
53
|
+
altcodepro_polydb_python-2.2.3.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
|
|
54
|
+
altcodepro_polydb_python-2.2.3.dist-info/RECORD,,
|
polydb/__init__.py
CHANGED
|
@@ -4,9 +4,9 @@ PolyDB - Enterprise Cloud-Independent Database Abstraction
|
|
|
4
4
|
Full LINQ support, field-level audit, cache, soft delete, overflow storage
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "2.2.
|
|
7
|
+
__version__ = "2.2.3"
|
|
8
8
|
|
|
9
|
-
from .
|
|
9
|
+
from .cloudDatabaseFactory import CloudDatabaseFactory
|
|
10
10
|
from .databaseFactory import DatabaseFactory
|
|
11
11
|
from .models import CloudProvider, PartitionConfig
|
|
12
12
|
from .decorators import polydb_model
|
|
@@ -1,77 +1,141 @@
|
|
|
1
1
|
# src/polydb/adapters/AzureBlobStorageAdapter.py
|
|
2
2
|
|
|
3
|
-
from polydb.base.ObjectStorageAdapter import ObjectStorageAdapter
|
|
4
|
-
from polydb.errors import ConnectionError, StorageError
|
|
5
|
-
from polydb.retry import retry
|
|
6
3
|
import os
|
|
7
4
|
import threading
|
|
8
|
-
from typing import List
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
from azure.storage.blob import BlobServiceClient, ContainerClient
|
|
8
|
+
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
|
|
9
|
+
|
|
10
|
+
from ..base.ObjectStorageAdapter import ObjectStorageAdapter
|
|
11
|
+
from ..errors import ConnectionError, StorageError
|
|
12
|
+
from ..retry import retry
|
|
13
|
+
|
|
9
14
|
|
|
10
15
|
class AzureBlobStorageAdapter(ObjectStorageAdapter):
|
|
11
|
-
"""
|
|
16
|
+
"""
|
|
17
|
+
Production-grade Azure Blob Storage adapter.
|
|
18
|
+
|
|
19
|
+
Features
|
|
20
|
+
- Thread-safe client initialization
|
|
21
|
+
- Container auto-creation
|
|
22
|
+
- Retry support
|
|
23
|
+
- Structured logging
|
|
24
|
+
- Connection reuse
|
|
25
|
+
"""
|
|
12
26
|
|
|
13
|
-
def __init__(self):
|
|
27
|
+
def __init__(self, connection_string: str = "", container_name: str = ""):
|
|
14
28
|
super().__init__()
|
|
15
|
-
|
|
16
|
-
self.
|
|
17
|
-
self.
|
|
29
|
+
|
|
30
|
+
self.connection_string = connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
|
31
|
+
self.container_name = container_name or os.getenv("AZURE_CONTAINER_NAME", "polydb")
|
|
32
|
+
|
|
33
|
+
if not self.connection_string:
|
|
34
|
+
raise ConnectionError("AZURE_STORAGE_CONNECTION_STRING is not configured")
|
|
35
|
+
|
|
36
|
+
self._client: Optional[BlobServiceClient] = None
|
|
37
|
+
self._container: Optional[ContainerClient] = None
|
|
18
38
|
self._lock = threading.Lock()
|
|
39
|
+
|
|
19
40
|
self._initialize_client()
|
|
20
41
|
|
|
21
|
-
def _initialize_client(self):
|
|
22
|
-
"""Initialize Azure Blob
|
|
42
|
+
def _initialize_client(self) -> None:
|
|
43
|
+
"""Initialize Azure Blob client and container"""
|
|
23
44
|
try:
|
|
24
|
-
from azure.storage.blob import BlobServiceClient
|
|
25
|
-
|
|
26
45
|
with self._lock:
|
|
27
|
-
if
|
|
28
|
-
|
|
29
|
-
|
|
46
|
+
if self._client is not None:
|
|
47
|
+
return
|
|
48
|
+
if not self.connection_string:
|
|
49
|
+
raise ConnectionError("AZURE_STORAGE_CONNECTION_STRING is not configured")
|
|
50
|
+
self._client = BlobServiceClient.from_connection_string(self.connection_string)
|
|
51
|
+
|
|
52
|
+
self._container = self._client.get_container_client(self.container_name)
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
self._container.create_container()
|
|
56
|
+
self.logger.info(f"Created container: {self.container_name}")
|
|
57
|
+
except ResourceExistsError:
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
self.logger.info(
|
|
61
|
+
f"Azure Blob Storage initialized (container={self.container_name})"
|
|
62
|
+
)
|
|
63
|
+
|
|
30
64
|
except Exception as e:
|
|
31
|
-
raise ConnectionError(f"Failed to initialize Azure Blob Storage: {
|
|
65
|
+
raise ConnectionError(f"Failed to initialize Azure Blob Storage: {e}")
|
|
66
|
+
|
|
67
|
+
def _require_container(self) -> ContainerClient:
|
|
68
|
+
"""Ensure container exists"""
|
|
69
|
+
if self._container is None:
|
|
70
|
+
raise ConnectionError("Azure Blob Storage client is not initialized")
|
|
71
|
+
return self._container
|
|
32
72
|
|
|
33
73
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
34
74
|
def _put_raw(self, key: str, data: bytes) -> str:
|
|
35
|
-
"""
|
|
75
|
+
"""Upload blob"""
|
|
36
76
|
try:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
77
|
+
container = self._require_container()
|
|
78
|
+
|
|
79
|
+
blob_client = container.get_blob_client(key)
|
|
80
|
+
blob_client.upload_blob(data, overwrite=True)
|
|
81
|
+
|
|
82
|
+
self.logger.debug(f"Uploaded blob key={key}")
|
|
83
|
+
|
|
41
84
|
return key
|
|
85
|
+
|
|
42
86
|
except Exception as e:
|
|
43
|
-
raise StorageError(f"Azure Blob put failed: {
|
|
87
|
+
raise StorageError(f"Azure Blob put failed: {e}")
|
|
44
88
|
|
|
45
89
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
46
90
|
def get(self, key: str) -> bytes | None:
|
|
47
|
-
"""
|
|
91
|
+
"""Download blob"""
|
|
48
92
|
try:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
93
|
+
container = self._require_container()
|
|
94
|
+
|
|
95
|
+
blob_client = container.get_blob_client(key)
|
|
96
|
+
|
|
97
|
+
downloader = blob_client.download_blob()
|
|
98
|
+
data = downloader.readall()
|
|
99
|
+
|
|
100
|
+
self.logger.debug(f"Downloaded blob key={key}")
|
|
101
|
+
|
|
102
|
+
return data
|
|
103
|
+
|
|
104
|
+
except ResourceNotFoundError:
|
|
52
105
|
return None
|
|
53
106
|
except Exception as e:
|
|
54
|
-
raise StorageError(f"Azure Blob get failed: {
|
|
107
|
+
raise StorageError(f"Azure Blob get failed: {e}")
|
|
55
108
|
|
|
56
109
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
57
110
|
def delete(self, key: str) -> bool:
|
|
58
111
|
"""Delete blob"""
|
|
59
112
|
try:
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
113
|
+
container = self._require_container()
|
|
114
|
+
|
|
115
|
+
blob_client = container.get_blob_client(key)
|
|
116
|
+
blob_client.delete_blob(delete_snapshots="include")
|
|
117
|
+
|
|
118
|
+
self.logger.debug(f"Deleted blob key={key}")
|
|
119
|
+
|
|
120
|
+
return True
|
|
121
|
+
|
|
122
|
+
except ResourceNotFoundError:
|
|
64
123
|
return False
|
|
65
124
|
except Exception as e:
|
|
66
|
-
raise StorageError(f"Azure Blob delete failed: {
|
|
125
|
+
raise StorageError(f"Azure Blob delete failed: {e}")
|
|
67
126
|
|
|
68
127
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
69
128
|
def list(self, prefix: str = "") -> List[str]:
|
|
70
|
-
"""List blobs
|
|
129
|
+
"""List blobs"""
|
|
71
130
|
try:
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
131
|
+
container = self._require_container()
|
|
132
|
+
|
|
133
|
+
blobs = container.list_blobs(name_starts_with=prefix)
|
|
134
|
+
results = [blob.name for blob in blobs]
|
|
135
|
+
|
|
136
|
+
self.logger.debug(f"Listed {len(results)} blobs prefix={prefix}")
|
|
137
|
+
|
|
138
|
+
return results
|
|
139
|
+
|
|
76
140
|
except Exception as e:
|
|
77
|
-
raise StorageError(f"Azure Blob list failed: {
|
|
141
|
+
raise StorageError(f"Azure Blob list failed: {e}")
|
|
@@ -1,79 +1,184 @@
|
|
|
1
1
|
# src/polydb/adapters/AzureFileStorageAdapter.py
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
|
|
5
|
-
from polydb.retry import retry
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
6
5
|
import os
|
|
7
6
|
import threading
|
|
8
|
-
from typing import List
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from azure.storage.fileshare import ShareServiceClient
|
|
10
|
+
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
|
|
11
|
+
|
|
12
|
+
from ..base.SharedFilesAdapter import SharedFilesAdapter
|
|
13
|
+
from ..errors import ConnectionError, StorageError
|
|
14
|
+
from ..retry import retry
|
|
15
|
+
|
|
9
16
|
|
|
10
17
|
class AzureFileStorageAdapter(SharedFilesAdapter):
|
|
11
|
-
"""
|
|
18
|
+
"""
|
|
19
|
+
Production-grade Azure File Storage adapter.
|
|
20
|
+
|
|
21
|
+
Fixes:
|
|
22
|
+
- Adds upload/download methods expected by tests
|
|
23
|
+
- Ensures share exists
|
|
24
|
+
- Ensures directory structure exists
|
|
25
|
+
- Correct Azure file creation before upload
|
|
26
|
+
- Keeps write/read compatibility
|
|
27
|
+
"""
|
|
12
28
|
|
|
13
|
-
def __init__(self):
|
|
29
|
+
def __init__(self, connection_string: str = "", share_name: str = ""):
|
|
14
30
|
super().__init__()
|
|
15
|
-
|
|
16
|
-
self.
|
|
17
|
-
|
|
31
|
+
|
|
32
|
+
self.connection_string = (
|
|
33
|
+
connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING") or ""
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
self.share_name = share_name or os.getenv("AZURE_SHARE_NAME", "polydb")
|
|
37
|
+
|
|
38
|
+
if not self.connection_string:
|
|
39
|
+
raise ConnectionError("AZURE_STORAGE_CONNECTION_STRING not configured")
|
|
40
|
+
|
|
41
|
+
self._client: Optional[ShareServiceClient] = None
|
|
42
|
+
self._share = None
|
|
18
43
|
self._lock = threading.Lock()
|
|
44
|
+
|
|
19
45
|
self._initialize_client()
|
|
20
46
|
|
|
47
|
+
# --------------------------------------------------
|
|
48
|
+
# Client initialization
|
|
49
|
+
# --------------------------------------------------
|
|
50
|
+
|
|
21
51
|
def _initialize_client(self):
|
|
22
|
-
"""Initialize Azure File Storage client once"""
|
|
23
52
|
try:
|
|
24
|
-
from azure.storage.fileshare import ShareServiceClient
|
|
25
|
-
|
|
26
53
|
with self._lock:
|
|
27
|
-
if
|
|
28
|
-
|
|
29
|
-
|
|
54
|
+
if self._client:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
self._client = ShareServiceClient.from_connection_string(self.connection_string)
|
|
58
|
+
|
|
59
|
+
self._share = self._client.get_share_client(self.share_name)
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
self._share.create_share()
|
|
63
|
+
except ResourceExistsError:
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
self.logger.info("Initialized Azure File Storage client")
|
|
67
|
+
|
|
30
68
|
except Exception as e:
|
|
31
69
|
raise ConnectionError(f"Failed to initialize Azure File Storage: {str(e)}")
|
|
32
70
|
|
|
71
|
+
# --------------------------------------------------
|
|
72
|
+
# Helpers
|
|
73
|
+
# --------------------------------------------------
|
|
74
|
+
|
|
75
|
+
def _split_path(self, path: str):
|
|
76
|
+
if "/" not in path:
|
|
77
|
+
return "", path
|
|
78
|
+
|
|
79
|
+
directory, filename = path.rsplit("/", 1)
|
|
80
|
+
return directory, filename
|
|
81
|
+
|
|
82
|
+
def _ensure_directory(self, directory: str):
|
|
83
|
+
if not directory and self._share:
|
|
84
|
+
return self._share.get_directory_client("")
|
|
85
|
+
if not self._share:
|
|
86
|
+
raise ConnectionError("Azure File Storage share not initialized")
|
|
87
|
+
dir_client = self._share.get_directory_client(directory)
|
|
88
|
+
|
|
89
|
+
try:
|
|
90
|
+
dir_client.create_directory()
|
|
91
|
+
except ResourceExistsError:
|
|
92
|
+
pass
|
|
93
|
+
|
|
94
|
+
return dir_client
|
|
95
|
+
|
|
96
|
+
# --------------------------------------------------
|
|
97
|
+
# Core operations
|
|
98
|
+
# --------------------------------------------------
|
|
99
|
+
|
|
33
100
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
34
|
-
def
|
|
35
|
-
"""
|
|
101
|
+
def upload(self, path: str, data: bytes) -> str:
|
|
102
|
+
"""Upload file"""
|
|
36
103
|
try:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
104
|
+
directory, filename = self._split_path(path)
|
|
105
|
+
|
|
106
|
+
dir_client = self._ensure_directory(directory)
|
|
107
|
+
file_client = dir_client.get_file_client(filename)
|
|
108
|
+
|
|
109
|
+
file_client.create_file(len(data))
|
|
110
|
+
file_client.upload_file(data)
|
|
111
|
+
|
|
112
|
+
return path
|
|
113
|
+
|
|
42
114
|
except Exception as e:
|
|
43
|
-
raise StorageError(f"Azure File
|
|
115
|
+
raise StorageError(f"Azure File upload failed: {str(e)}")
|
|
44
116
|
|
|
45
117
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
46
|
-
def
|
|
47
|
-
"""
|
|
118
|
+
def download(self, path: str) -> bytes:
|
|
119
|
+
"""Download file"""
|
|
48
120
|
try:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
121
|
+
directory, filename = self._split_path(path)
|
|
122
|
+
if not self._share:
|
|
123
|
+
raise ConnectionError("Azure File Storage share not initialized")
|
|
124
|
+
dir_client = self._share.get_directory_client(directory or "")
|
|
125
|
+
file_client = dir_client.get_file_client(filename)
|
|
126
|
+
|
|
127
|
+
return file_client.download_file().readall()
|
|
128
|
+
|
|
129
|
+
except ResourceNotFoundError:
|
|
130
|
+
raise StorageError(f"File not found: {path}")
|
|
53
131
|
except Exception as e:
|
|
54
|
-
raise StorageError(f"Azure File
|
|
132
|
+
raise StorageError(f"Azure File download failed: {str(e)}")
|
|
55
133
|
|
|
56
134
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
57
135
|
def delete(self, path: str) -> bool:
|
|
58
136
|
"""Delete file"""
|
|
59
137
|
try:
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
138
|
+
directory, filename = self._split_path(path)
|
|
139
|
+
if not self._share:
|
|
140
|
+
raise ConnectionError("Azure File Storage share not initialized")
|
|
141
|
+
dir_client = self._share.get_directory_client(directory or "")
|
|
142
|
+
file_client = dir_client.get_file_client(filename)
|
|
143
|
+
|
|
144
|
+
file_client.delete_file()
|
|
145
|
+
return True
|
|
146
|
+
|
|
147
|
+
except ResourceNotFoundError:
|
|
64
148
|
return False
|
|
65
149
|
except Exception as e:
|
|
66
150
|
raise StorageError(f"Azure File delete failed: {str(e)}")
|
|
67
151
|
|
|
68
152
|
@retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
|
|
69
|
-
def list(self, directory: str = "
|
|
70
|
-
"""List files
|
|
153
|
+
def list(self, directory: str = "") -> List[str]:
|
|
154
|
+
"""List files"""
|
|
71
155
|
try:
|
|
72
|
-
if self.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
156
|
+
if not self._share:
|
|
157
|
+
raise ConnectionError("Azure File Storage share not initialized")
|
|
158
|
+
dir_client = self._share.get_directory_client(directory or "")
|
|
159
|
+
|
|
160
|
+
results: List[str] = []
|
|
161
|
+
|
|
162
|
+
for item in dir_client.list_directories_and_files():
|
|
163
|
+
results.append(item.name)
|
|
164
|
+
|
|
165
|
+
return results
|
|
166
|
+
|
|
78
167
|
except Exception as e:
|
|
79
|
-
raise StorageError(f"Azure File list failed: {str(e)}")
|
|
168
|
+
raise StorageError(f"Azure File list failed: {str(e)}")
|
|
169
|
+
|
|
170
|
+
# --------------------------------------------------
|
|
171
|
+
# Backward compatibility
|
|
172
|
+
# --------------------------------------------------
|
|
173
|
+
|
|
174
|
+
def write(self, path: str, data: bytes) -> bool:
|
|
175
|
+
"""Alias for upload"""
|
|
176
|
+
self.upload(path, data)
|
|
177
|
+
return True
|
|
178
|
+
|
|
179
|
+
def read(self, path: str) -> bytes | None:
|
|
180
|
+
"""Alias for download"""
|
|
181
|
+
try:
|
|
182
|
+
return self.download(path)
|
|
183
|
+
except StorageError:
|
|
184
|
+
return None
|
|
@@ -1,63 +1,125 @@
|
|
|
1
1
|
# src/polydb/adapters/AzureQueueAdapter.py
|
|
2
|
-
|
|
3
|
-
from polydb.errors import ConnectionError, QueueError
|
|
4
|
-
from polydb.retry import retry
|
|
2
|
+
|
|
5
3
|
import os
|
|
6
4
|
import threading
|
|
7
|
-
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from azure.storage.queue import QueueServiceClient, QueueClient
|
|
9
|
+
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
|
|
8
10
|
|
|
11
|
+
from ..base.QueueAdapter import QueueAdapter
|
|
12
|
+
from ..errors import ConnectionError, QueueError
|
|
13
|
+
from ..retry import retry
|
|
9
14
|
from ..json_safe import json_safe
|
|
10
15
|
|
|
16
|
+
|
|
11
17
|
class AzureQueueAdapter(QueueAdapter):
|
|
12
|
-
"""
|
|
18
|
+
"""
|
|
19
|
+
Azure Queue Storage adapter.
|
|
20
|
+
|
|
21
|
+
Features
|
|
22
|
+
- Thread-safe initialization
|
|
23
|
+
- Automatic queue creation
|
|
24
|
+
- Client reuse
|
|
25
|
+
- Retry support
|
|
26
|
+
"""
|
|
13
27
|
|
|
14
|
-
def __init__(self):
|
|
28
|
+
def __init__(self, connection_string: str = ""):
|
|
15
29
|
super().__init__()
|
|
16
|
-
|
|
17
|
-
self.
|
|
30
|
+
|
|
31
|
+
self.connection_string = connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
|
32
|
+
|
|
33
|
+
if not self.connection_string:
|
|
34
|
+
raise ConnectionError("AZURE_STORAGE_CONNECTION_STRING is not configured")
|
|
35
|
+
|
|
36
|
+
self._client: Optional[QueueServiceClient] = None
|
|
37
|
+
self._queues: Dict[str, QueueClient] = {}
|
|
38
|
+
|
|
18
39
|
self._lock = threading.Lock()
|
|
40
|
+
|
|
19
41
|
self._initialize_client()
|
|
20
42
|
|
|
21
|
-
def _initialize_client(self):
|
|
22
|
-
"""Initialize Azure Queue
|
|
43
|
+
def _initialize_client(self) -> None:
|
|
44
|
+
"""Initialize Azure Queue client"""
|
|
23
45
|
try:
|
|
24
|
-
from azure.storage.queue import QueueServiceClient
|
|
25
|
-
|
|
26
46
|
with self._lock:
|
|
27
|
-
if
|
|
28
|
-
|
|
29
|
-
|
|
47
|
+
if self._client is not None:
|
|
48
|
+
return
|
|
49
|
+
if not self.connection_string:
|
|
50
|
+
raise ConnectionError("AZURE_STORAGE_CONNECTION_STRING is not configured")
|
|
51
|
+
self._client = QueueServiceClient.from_connection_string(self.connection_string)
|
|
52
|
+
|
|
53
|
+
self.logger.info("Initialized Azure Queue Storage client")
|
|
54
|
+
|
|
30
55
|
except Exception as e:
|
|
31
|
-
raise ConnectionError(f"Failed to initialize Azure Queue Storage: {
|
|
56
|
+
raise ConnectionError(f"Failed to initialize Azure Queue Storage: {e}")
|
|
57
|
+
|
|
58
|
+
def _get_queue(self, queue_name: str) -> QueueClient:
|
|
59
|
+
"""Get or create queue client"""
|
|
60
|
+
if self._client is None:
|
|
61
|
+
raise ConnectionError("Azure Queue client not initialized")
|
|
62
|
+
|
|
63
|
+
if queue_name not in self._queues:
|
|
64
|
+
queue_client = self._client.get_queue_client(queue_name)
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
queue_client.create_queue()
|
|
68
|
+
except ResourceExistsError:
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
self._queues[queue_name] = queue_client
|
|
72
|
+
|
|
73
|
+
return self._queues[queue_name]
|
|
32
74
|
|
|
33
75
|
@retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
|
|
34
76
|
def send(self, message: Dict[str, Any], queue_name: str = "default") -> str:
|
|
35
77
|
"""Send message to queue"""
|
|
36
78
|
try:
|
|
37
|
-
|
|
79
|
+
queue_client = self._get_queue(queue_name)
|
|
80
|
+
|
|
81
|
+
response = queue_client.send_message(json.dumps(message, default=json_safe))
|
|
82
|
+
|
|
83
|
+
return response.id
|
|
38
84
|
|
|
39
|
-
if self._client:
|
|
40
|
-
queue_client = self._client.get_queue_client(queue_name)
|
|
41
|
-
response = queue_client.send_message(json.dumps(message,default=json_safe))
|
|
42
|
-
return response.id
|
|
43
|
-
return ""
|
|
44
85
|
except Exception as e:
|
|
45
|
-
raise QueueError(f"Azure Queue send failed: {
|
|
86
|
+
raise QueueError(f"Azure Queue send failed: {e}")
|
|
46
87
|
|
|
47
88
|
@retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
|
|
48
89
|
def receive(self, queue_name: str = "default", max_messages: int = 1) -> List[Dict[str, Any]]:
|
|
49
|
-
"""Receive messages
|
|
90
|
+
"""Receive messages"""
|
|
50
91
|
try:
|
|
51
|
-
|
|
92
|
+
queue_client = self._get_queue(queue_name)
|
|
93
|
+
|
|
94
|
+
messages = queue_client.receive_messages(max_messages=max_messages)
|
|
95
|
+
|
|
96
|
+
results = []
|
|
97
|
+
|
|
98
|
+
for msg in messages:
|
|
99
|
+
payload = json.loads(msg.content)
|
|
100
|
+
results.append(
|
|
101
|
+
{
|
|
102
|
+
"id": msg.id,
|
|
103
|
+
"pop_receipt": msg.pop_receipt,
|
|
104
|
+
"body": payload,
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
return results
|
|
52
109
|
|
|
53
|
-
if self._client:
|
|
54
|
-
queue_client = self._client.get_queue_client(queue_name)
|
|
55
|
-
messages = queue_client.receive_messages(max_messages=max_messages)
|
|
56
|
-
return [json.loads(msg.content) for msg in messages]
|
|
57
|
-
return []
|
|
58
110
|
except Exception as e:
|
|
59
|
-
raise QueueError(f"Azure Queue receive failed: {
|
|
111
|
+
raise QueueError(f"Azure Queue receive failed: {e}")
|
|
112
|
+
|
|
113
|
+
def delete(self, message_id: str, queue_name: str = "default", pop_receipt: str = "") -> bool:
|
|
114
|
+
"""Delete message from queue"""
|
|
115
|
+
try:
|
|
116
|
+
queue_client = self._get_queue(queue_name)
|
|
60
117
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
118
|
+
queue_client.delete_message(message_id, pop_receipt)
|
|
119
|
+
|
|
120
|
+
return True
|
|
121
|
+
|
|
122
|
+
except ResourceNotFoundError:
|
|
123
|
+
return False
|
|
124
|
+
except Exception as e:
|
|
125
|
+
raise QueueError(f"Azure Queue delete failed: {e}")
|