altcodepro-polydb-python 2.3.8__py3-none-any.whl → 2.3.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.3.8.dist-info → altcodepro_polydb_python-2.3.10.dist-info}/METADATA +11 -1
- altcodepro_polydb_python-2.3.10.dist-info/RECORD +61 -0
- polydb/adapters/AzureBlobStorageAdapter.py +92 -90
- polydb/adapters/AzureFileStorageAdapter.py +74 -74
- polydb/adapters/AzureQueueAdapter.py +9 -5
- polydb/adapters/AzureTableStorageAdapter.py +5 -5
- polydb/adapters/BlockchainBlobAdapter.py +1 -1
- polydb/adapters/BlockchainFileAdapter.py +217 -0
- polydb/adapters/BlockchainKVAdapter.py +4 -3
- polydb/adapters/BlockchainQueueAdapter.py +3 -2
- polydb/adapters/DynamoDBAdapter.py +12 -3
- polydb/adapters/EFSAdapter.py +45 -19
- polydb/adapters/FirestoreAdapter.py +15 -13
- polydb/adapters/GCPFilestoreAdapter.py +77 -0
- polydb/adapters/GCPPubSubAdapter.py +4 -4
- polydb/adapters/GCPStorageAdapter.py +78 -117
- polydb/adapters/PostgreSQLAdapter.py +3 -2
- polydb/adapters/S3Adapter.py +5 -2
- polydb/adapters/S3CompatibleAdapter.py +87 -57
- polydb/adapters/SQSAdapter.py +5 -2
- polydb/adapters/VercelFileAdapter.py +29 -0
- polydb/audit/__init__.py +1 -1
- polydb/base/SharedFilesAdapter.py +5 -5
- polydb/cloudDatabaseFactory.py +37 -66
- polydb/databaseFactory.py +23 -7
- altcodepro_polydb_python-2.3.8.dist-info/RECORD +0 -58
- {altcodepro_polydb_python-2.3.8.dist-info → altcodepro_polydb_python-2.3.10.dist-info}/WHEEL +0 -0
- {altcodepro_polydb_python-2.3.8.dist-info → altcodepro_polydb_python-2.3.10.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.3.8.dist-info → altcodepro_polydb_python-2.3.10.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,8 @@ import json
|
|
|
8
8
|
import base64
|
|
9
9
|
import hashlib
|
|
10
10
|
import threading
|
|
11
|
+
import logging
|
|
12
|
+
|
|
11
13
|
from datetime import datetime, date
|
|
12
14
|
from decimal import Decimal
|
|
13
15
|
from typing import Any, Dict, List, Optional
|
|
@@ -19,7 +21,6 @@ from ..errors import NoSQLError, ConnectionError
|
|
|
19
21
|
from ..retry import retry
|
|
20
22
|
from ..types import JsonDict
|
|
21
23
|
from ..models import PartitionConfig
|
|
22
|
-
import logging
|
|
23
24
|
|
|
24
25
|
logger = logging.getLogger(__name__)
|
|
25
26
|
|
|
@@ -540,7 +541,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
540
541
|
if "ResourceNotFound" in str(e):
|
|
541
542
|
return None
|
|
542
543
|
raise NoSQLError(f"Azure Table get failed: {str(e)}")
|
|
543
|
-
|
|
544
|
+
|
|
544
545
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
545
546
|
def _query_raw(
|
|
546
547
|
self, model: type, filters: Dict[str, Any], limit: Optional[int]
|
|
@@ -561,9 +562,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
561
562
|
sk = "RowKey"
|
|
562
563
|
else:
|
|
563
564
|
sk = (
|
|
564
|
-
self._sanitize_prop_name(orig_k)
|
|
565
|
-
if orig_k != _MODEL_FIELD
|
|
566
|
-
else _MODEL_FIELD
|
|
565
|
+
self._sanitize_prop_name(orig_k) if orig_k != _MODEL_FIELD else _MODEL_FIELD
|
|
567
566
|
)
|
|
568
567
|
|
|
569
568
|
ev = self._encode_value(orig_v)
|
|
@@ -611,6 +610,7 @@ class AzureTableStorageAdapter(NoSQLKVAdapter):
|
|
|
611
610
|
|
|
612
611
|
except Exception as e:
|
|
613
612
|
raise NoSQLError(f"Azure Table query failed: {str(e)}")
|
|
613
|
+
|
|
614
614
|
@retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
|
|
615
615
|
def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
|
|
616
616
|
try:
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# src/polydb/adapters/BlockchainFileAdapter.py
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import List, Optional
|
|
7
|
+
|
|
8
|
+
from dotenv import load_dotenv
|
|
9
|
+
|
|
10
|
+
from ..base.SharedFilesAdapter import SharedFilesAdapter
|
|
11
|
+
from ..errors import StorageError, ConnectionError
|
|
12
|
+
|
|
13
|
+
SUPPORTED_CHAINS = {"ethereum", "polygon", "avalanche", "bnb", "arbitrum"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BlockchainFileAdapter(SharedFilesAdapter):
|
|
17
|
+
"""
|
|
18
|
+
Blockchain-backed shared file storage.
|
|
19
|
+
|
|
20
|
+
Strategy:
|
|
21
|
+
- file BYTES are stored on IPFS (content-addressed -> CID)
|
|
22
|
+
- the PATH -> CID mapping is kept in an on-chain registry contract
|
|
23
|
+
- per-call `share_name` namespaces the path on-chain
|
|
24
|
+
|
|
25
|
+
Fully supported: write / read / delete.
|
|
26
|
+
Partially supported: list (on-chain listFiles(prefix), with an event-replay
|
|
27
|
+
fallback — true cheap enumeration needs an off-chain indexer like TheGraph).
|
|
28
|
+
|
|
29
|
+
EVM chains: ethereum, polygon, avalanche, bnb, arbitrum.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
chain: Optional[str] = None,
|
|
35
|
+
rpc_url: Optional[str] = None,
|
|
36
|
+
private_key: Optional[str] = None,
|
|
37
|
+
contract_address: Optional[str] = None,
|
|
38
|
+
contract_abi: Optional[list] = None,
|
|
39
|
+
ipfs_url: Optional[str] = None,
|
|
40
|
+
):
|
|
41
|
+
super().__init__()
|
|
42
|
+
load_dotenv()
|
|
43
|
+
|
|
44
|
+
self.chain = (chain or os.getenv("BLOCKCHAIN_CHAIN", "ethereum")).lower()
|
|
45
|
+
if self.chain not in SUPPORTED_CHAINS:
|
|
46
|
+
raise ValueError(f"Unsupported blockchain: {self.chain}")
|
|
47
|
+
|
|
48
|
+
self.rpc_url = rpc_url or os.getenv("BLOCKCHAIN_RPC_URL")
|
|
49
|
+
self.private_key = private_key or os.getenv("BLOCKCHAIN_PRIVATE_KEY")
|
|
50
|
+
self.contract_address = (
|
|
51
|
+
contract_address
|
|
52
|
+
or os.getenv("BLOCKCHAIN_FILE_CONTRACT")
|
|
53
|
+
or os.getenv("BLOCKCHAIN_CONTRACT")
|
|
54
|
+
)
|
|
55
|
+
self.ipfs_url = (ipfs_url or os.getenv("IPFS_API_URL", "http://localhost:5001")).rstrip("/")
|
|
56
|
+
|
|
57
|
+
if not self.rpc_url:
|
|
58
|
+
raise ConnectionError("BLOCKCHAIN_RPC_URL not configured")
|
|
59
|
+
if not self.private_key:
|
|
60
|
+
raise ConnectionError("BLOCKCHAIN_PRIVATE_KEY not configured")
|
|
61
|
+
if not self.contract_address:
|
|
62
|
+
raise ConnectionError("BLOCKCHAIN_FILE_CONTRACT not configured")
|
|
63
|
+
|
|
64
|
+
from web3 import Web3
|
|
65
|
+
from web3.middleware import ExtraDataToPOAMiddleware
|
|
66
|
+
|
|
67
|
+
self.w3 = Web3(Web3.HTTPProvider(self.rpc_url))
|
|
68
|
+
if self.chain in {"polygon", "avalanche", "bnb"}:
|
|
69
|
+
self.w3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)
|
|
70
|
+
|
|
71
|
+
self.account = self.w3.eth.account.from_key(self.private_key)
|
|
72
|
+
self.contract = self.w3.eth.contract(
|
|
73
|
+
address=Web3.to_checksum_address(self.contract_address),
|
|
74
|
+
abi=contract_abi or self._default_abi(),
|
|
75
|
+
)
|
|
76
|
+
self.logger.info(
|
|
77
|
+
f"Blockchain file adapter ready (chain={self.chain}, ipfs={self.ipfs_url})"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# ------------------------------------------------------------------
|
|
81
|
+
# Contract ABI + tx helpers
|
|
82
|
+
# ------------------------------------------------------------------
|
|
83
|
+
def _default_abi(self):
|
|
84
|
+
return [
|
|
85
|
+
{
|
|
86
|
+
"inputs": [{"name": "path", "type": "string"}, {"name": "cid", "type": "string"}],
|
|
87
|
+
"name": "putFile",
|
|
88
|
+
"outputs": [],
|
|
89
|
+
"stateMutability": "nonpayable",
|
|
90
|
+
"type": "function",
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"inputs": [{"name": "path", "type": "string"}],
|
|
94
|
+
"name": "getFile",
|
|
95
|
+
"outputs": [{"name": "", "type": "string"}],
|
|
96
|
+
"stateMutability": "view",
|
|
97
|
+
"type": "function",
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"inputs": [{"name": "path", "type": "string"}],
|
|
101
|
+
"name": "deleteFile",
|
|
102
|
+
"outputs": [],
|
|
103
|
+
"stateMutability": "nonpayable",
|
|
104
|
+
"type": "function",
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"inputs": [{"name": "prefix", "type": "string"}],
|
|
108
|
+
"name": "listFiles",
|
|
109
|
+
"outputs": [{"name": "", "type": "string[]"}],
|
|
110
|
+
"stateMutability": "view",
|
|
111
|
+
"type": "function",
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
"anonymous": False,
|
|
115
|
+
"inputs": [
|
|
116
|
+
{"indexed": False, "name": "path", "type": "string"},
|
|
117
|
+
{"indexed": False, "name": "cid", "type": "string"},
|
|
118
|
+
],
|
|
119
|
+
"name": "FileStored",
|
|
120
|
+
"type": "event",
|
|
121
|
+
},
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
def _send_tx(self, fn):
|
|
125
|
+
nonce = self.w3.eth.get_transaction_count(self.account.address)
|
|
126
|
+
try:
|
|
127
|
+
gas = int(fn.estimate_gas({"from": self.account.address}) * 1.2)
|
|
128
|
+
except Exception:
|
|
129
|
+
gas = 500000
|
|
130
|
+
tx = fn.build_transaction(
|
|
131
|
+
{
|
|
132
|
+
"from": self.account.address,
|
|
133
|
+
"nonce": nonce,
|
|
134
|
+
"gas": gas,
|
|
135
|
+
"gasPrice": self.w3.eth.gas_price,
|
|
136
|
+
}
|
|
137
|
+
)
|
|
138
|
+
signed = self.account.sign_transaction(tx)
|
|
139
|
+
tx_hash = self.w3.eth.send_raw_transaction(signed.raw_transaction)
|
|
140
|
+
return self.w3.eth.wait_for_transaction_receipt(tx_hash)
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def _key(path: str, share_name: Optional[str] = None) -> str:
|
|
144
|
+
path = (path or "").lstrip("/")
|
|
145
|
+
if share_name:
|
|
146
|
+
return f"{share_name.strip('/')}/{path}".rstrip("/")
|
|
147
|
+
return path
|
|
148
|
+
|
|
149
|
+
# ------------------------------------------------------------------
|
|
150
|
+
# IPFS bytes layer
|
|
151
|
+
# ------------------------------------------------------------------
|
|
152
|
+
def _ipfs_add(self, data: bytes) -> str:
|
|
153
|
+
import requests
|
|
154
|
+
|
|
155
|
+
resp = requests.post(f"{self.ipfs_url}/api/v0/add", files={"file": data}, timeout=60)
|
|
156
|
+
resp.raise_for_status()
|
|
157
|
+
return resp.json()["Hash"]
|
|
158
|
+
|
|
159
|
+
def _ipfs_cat(self, cid: str) -> bytes:
|
|
160
|
+
import requests
|
|
161
|
+
|
|
162
|
+
resp = requests.post(f"{self.ipfs_url}/api/v0/cat", params={"arg": cid}, timeout=60)
|
|
163
|
+
resp.raise_for_status()
|
|
164
|
+
return resp.content
|
|
165
|
+
|
|
166
|
+
# ------------------------------------------------------------------
|
|
167
|
+
# SharedFilesAdapter interface
|
|
168
|
+
# ------------------------------------------------------------------
|
|
169
|
+
def write(self, path: str, data: bytes, share_name: Optional[str] = None) -> bool:
|
|
170
|
+
try:
|
|
171
|
+
cid = self._ipfs_add(data)
|
|
172
|
+
self._send_tx(self.contract.functions.putFile(self._key(path, share_name), cid))
|
|
173
|
+
self.logger.debug(f"Blockchain file stored path={path} cid={cid}")
|
|
174
|
+
return True
|
|
175
|
+
except Exception as e:
|
|
176
|
+
raise StorageError(f"Blockchain file write failed: {str(e)}")
|
|
177
|
+
|
|
178
|
+
def read(self, path: str, share_name: Optional[str] = None) -> bytes | None:
|
|
179
|
+
try:
|
|
180
|
+
cid = self.contract.functions.getFile(self._key(path, share_name)).call()
|
|
181
|
+
if not cid:
|
|
182
|
+
return None
|
|
183
|
+
return self._ipfs_cat(cid)
|
|
184
|
+
except Exception as e:
|
|
185
|
+
raise StorageError(f"Blockchain file read failed: {str(e)}")
|
|
186
|
+
|
|
187
|
+
def delete(self, path: str, share_name: Optional[str] = None) -> bool:
|
|
188
|
+
try:
|
|
189
|
+
key = self._key(path, share_name)
|
|
190
|
+
if not self.contract.functions.getFile(key).call():
|
|
191
|
+
return False # nothing mapped
|
|
192
|
+
self._send_tx(self.contract.functions.deleteFile(key))
|
|
193
|
+
return True
|
|
194
|
+
except Exception as e:
|
|
195
|
+
raise StorageError(f"Blockchain file delete failed: {str(e)}")
|
|
196
|
+
|
|
197
|
+
def list(self, directory: str = "", share_name: Optional[str] = None) -> List[str]:
|
|
198
|
+
prefix = self._key(directory, share_name)
|
|
199
|
+
|
|
200
|
+
# 1) native on-chain enumeration if the contract supports it
|
|
201
|
+
try:
|
|
202
|
+
keys = self.contract.functions.listFiles(prefix).call()
|
|
203
|
+
return list(keys)
|
|
204
|
+
except Exception:
|
|
205
|
+
pass
|
|
206
|
+
|
|
207
|
+
# 2) fallback: replay FileStored events and reconstruct live keys
|
|
208
|
+
try:
|
|
209
|
+
evt = self.contract.events.FileStored.create_filter(fromBlock=0)
|
|
210
|
+
keys = {}
|
|
211
|
+
for e in evt.get_all_entries():
|
|
212
|
+
p = e["args"]["path"]
|
|
213
|
+
if p.startswith(prefix):
|
|
214
|
+
keys[p] = e["args"]["cid"]
|
|
215
|
+
return list(keys.keys())
|
|
216
|
+
except Exception as e:
|
|
217
|
+
raise StorageError(f"Blockchain file list failed: {str(e)}")
|
|
@@ -5,8 +5,7 @@ import logging
|
|
|
5
5
|
import os
|
|
6
6
|
from typing import Any, Dict, Optional
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
from web3.middleware import ExtraDataToPOAMiddleware
|
|
8
|
+
|
|
10
9
|
from dotenv import load_dotenv
|
|
11
10
|
|
|
12
11
|
logger = logging.getLogger(__name__)
|
|
@@ -61,6 +60,8 @@ class BlockchainKVAdapter:
|
|
|
61
60
|
|
|
62
61
|
if not self.contract_address:
|
|
63
62
|
raise RuntimeError("BLOCKCHAIN_CONTRACT not configured")
|
|
63
|
+
from web3 import Web3
|
|
64
|
+
from web3.middleware import ExtraDataToPOAMiddleware
|
|
64
65
|
|
|
65
66
|
self.w3 = Web3(Web3.HTTPProvider(self.rpc_url))
|
|
66
67
|
|
|
@@ -149,4 +150,4 @@ class BlockchainKVAdapter:
|
|
|
149
150
|
def query(self, model, query: Dict[str, Any], limit: Optional[int] = None):
|
|
150
151
|
raise NotImplementedError(
|
|
151
152
|
"Blockchain query requires an off-chain indexer (TheGraph / Elastic)"
|
|
152
|
-
)
|
|
153
|
+
)
|
|
@@ -5,8 +5,7 @@ import logging
|
|
|
5
5
|
import os
|
|
6
6
|
from typing import Any, Dict, List, Optional
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
from web3.middleware import ExtraDataToPOAMiddleware
|
|
8
|
+
|
|
10
9
|
from dotenv import load_dotenv
|
|
11
10
|
|
|
12
11
|
from ..errors import QueueError
|
|
@@ -36,6 +35,8 @@ class BlockchainQueueAdapter:
|
|
|
36
35
|
|
|
37
36
|
if not self.rpc_url:
|
|
38
37
|
raise RuntimeError("BLOCKCHAIN_RPC_URL not configured")
|
|
38
|
+
from web3 import Web3
|
|
39
|
+
from web3.middleware import ExtraDataToPOAMiddleware
|
|
39
40
|
|
|
40
41
|
self.w3 = Web3(Web3.HTTPProvider(self.rpc_url))
|
|
41
42
|
self.w3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)
|
|
@@ -9,9 +9,7 @@ import threading
|
|
|
9
9
|
from polydb.errors import DatabaseError
|
|
10
10
|
from typing import Any, Dict, List, Optional, Tuple
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
from botocore.exceptions import ClientError
|
|
14
|
-
from boto3.session import Session
|
|
12
|
+
|
|
15
13
|
from ..base.NoSQLKVAdapter import NoSQLKVAdapter
|
|
16
14
|
from ..errors import ConnectionError, NoSQLError
|
|
17
15
|
from ..json_safe import json_safe
|
|
@@ -78,6 +76,9 @@ class DynamoDBAdapter(NoSQLKVAdapter):
|
|
|
78
76
|
# ---------------------------------------------------------------------
|
|
79
77
|
|
|
80
78
|
def _initialize_clients(self) -> None:
|
|
79
|
+
|
|
80
|
+
from boto3.session import Session
|
|
81
|
+
|
|
81
82
|
try:
|
|
82
83
|
with self._lock:
|
|
83
84
|
if self._dynamodb and self._s3:
|
|
@@ -120,6 +121,9 @@ class DynamoDBAdapter(NoSQLKVAdapter):
|
|
|
120
121
|
Ensure PK/SK table exists. Safe in prod (no-op if exists).
|
|
121
122
|
Required for LocalStack integration tests.
|
|
122
123
|
"""
|
|
124
|
+
from boto3.dynamodb.conditions import Attr, Key
|
|
125
|
+
from botocore.exceptions import ClientError
|
|
126
|
+
|
|
123
127
|
if not self._dynamodb:
|
|
124
128
|
return
|
|
125
129
|
|
|
@@ -307,6 +311,9 @@ class DynamoDBAdapter(NoSQLKVAdapter):
|
|
|
307
311
|
Prefer Query when PK can be derived. Otherwise Scan.
|
|
308
312
|
Supports filters like {"id": "..."} and arbitrary Attr equality.
|
|
309
313
|
"""
|
|
314
|
+
from boto3.dynamodb.conditions import Attr, Key
|
|
315
|
+
from botocore.exceptions import ClientError
|
|
316
|
+
|
|
310
317
|
try:
|
|
311
318
|
table = self._get_table(model)
|
|
312
319
|
filters = filters or {}
|
|
@@ -412,6 +419,8 @@ class DynamoDBAdapter(NoSQLKVAdapter):
|
|
|
412
419
|
results until page_size items are returned
|
|
413
420
|
- continuation_token = base64(json(LastEvaluatedKey))
|
|
414
421
|
"""
|
|
422
|
+
from boto3.dynamodb.conditions import Attr, Key
|
|
423
|
+
|
|
415
424
|
try:
|
|
416
425
|
table = self._get_table(model)
|
|
417
426
|
query = query or {}
|
polydb/adapters/EFSAdapter.py
CHANGED
|
@@ -1,51 +1,77 @@
|
|
|
1
1
|
# src/polydb/adapters/EFSAdapter.py
|
|
2
2
|
|
|
3
|
-
from polydb.base.SharedFilesAdapter import SharedFilesAdapter
|
|
4
|
-
from polydb.errors import StorageError
|
|
5
3
|
import os
|
|
6
|
-
from typing import List
|
|
4
|
+
from typing import List, Optional
|
|
5
|
+
|
|
6
|
+
from ..base.SharedFilesAdapter import SharedFilesAdapter
|
|
7
|
+
from ..errors import StorageError, ConnectionError
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class EFSAdapter(SharedFilesAdapter):
|
|
10
|
-
"""
|
|
11
|
+
"""
|
|
12
|
+
AWS EFS — managed NFS mounted onto the host at a path (e.g. /mnt/efs).
|
|
13
|
+
Once mounted it's an ordinary POSIX directory, so ops are plain os/open calls.
|
|
14
|
+
|
|
15
|
+
- per-call `share_name` selects a sub-root directory under the mount point
|
|
16
|
+
- paths are confined to the resolved root (no traversal escapes)
|
|
17
|
+
"""
|
|
11
18
|
|
|
12
|
-
def __init__(self, mount_point: str):
|
|
19
|
+
def __init__(self, mount_point: str = ""):
|
|
13
20
|
super().__init__()
|
|
14
21
|
self.mount_point = mount_point or os.getenv("EFS_MOUNT_POINT", "/mnt/efs")
|
|
22
|
+
if not self.mount_point:
|
|
23
|
+
raise ConnectionError("EFS mount_point is required")
|
|
24
|
+
|
|
25
|
+
def _resolve(self, path: str, share_name: Optional[str] = None) -> str:
|
|
26
|
+
base = os.path.join(self.mount_point, share_name) if share_name else self.mount_point
|
|
27
|
+
base_abs = os.path.abspath(base)
|
|
28
|
+
rel = (path or "").lstrip("/\\")
|
|
29
|
+
full_abs = os.path.abspath(os.path.join(base_abs, rel))
|
|
30
|
+
if full_abs != base_abs and not full_abs.startswith(base_abs + os.sep):
|
|
31
|
+
raise StorageError(f"Path escapes storage root: {path}")
|
|
32
|
+
return full_abs
|
|
15
33
|
|
|
16
|
-
def write(self, path: str, data: bytes) -> bool:
|
|
17
|
-
"""Write file"""
|
|
34
|
+
def write(self, path: str, data: bytes, share_name: Optional[str] = None) -> bool:
|
|
18
35
|
try:
|
|
19
|
-
full_path =
|
|
36
|
+
full_path = self._resolve(path, share_name)
|
|
20
37
|
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
|
21
38
|
with open(full_path, "wb") as f:
|
|
22
39
|
f.write(data)
|
|
23
40
|
return True
|
|
41
|
+
except StorageError:
|
|
42
|
+
raise
|
|
24
43
|
except Exception as e:
|
|
25
44
|
raise StorageError(f"EFS write failed: {str(e)}")
|
|
26
45
|
|
|
27
|
-
def read(self, path: str) -> bytes:
|
|
28
|
-
"""Read file"""
|
|
46
|
+
def read(self, path: str, share_name: Optional[str] = None) -> bytes | None:
|
|
29
47
|
try:
|
|
30
|
-
|
|
31
|
-
with open(full_path, "rb") as f:
|
|
48
|
+
with open(self._resolve(path, share_name), "rb") as f:
|
|
32
49
|
return f.read()
|
|
50
|
+
except FileNotFoundError:
|
|
51
|
+
return None
|
|
52
|
+
except StorageError:
|
|
53
|
+
raise
|
|
33
54
|
except Exception as e:
|
|
34
55
|
raise StorageError(f"EFS read failed: {str(e)}")
|
|
35
56
|
|
|
36
|
-
def delete(self, path: str) -> bool:
|
|
37
|
-
"""Delete file"""
|
|
57
|
+
def delete(self, path: str, share_name: Optional[str] = None) -> bool:
|
|
38
58
|
try:
|
|
39
|
-
|
|
40
|
-
os.remove(full_path)
|
|
59
|
+
os.remove(self._resolve(path, share_name))
|
|
41
60
|
return True
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
return False
|
|
63
|
+
except StorageError:
|
|
64
|
+
raise
|
|
42
65
|
except Exception as e:
|
|
43
66
|
raise StorageError(f"EFS delete failed: {str(e)}")
|
|
44
67
|
|
|
45
|
-
def list(self, directory: str = "
|
|
46
|
-
"""List files in directory"""
|
|
68
|
+
def list(self, directory: str = "", share_name: Optional[str] = None) -> List[str]:
|
|
47
69
|
try:
|
|
48
|
-
full_path =
|
|
70
|
+
full_path = self._resolve(directory, share_name)
|
|
71
|
+
if not os.path.isdir(full_path):
|
|
72
|
+
return []
|
|
49
73
|
return os.listdir(full_path)
|
|
74
|
+
except StorageError:
|
|
75
|
+
raise
|
|
50
76
|
except Exception as e:
|
|
51
77
|
raise StorageError(f"EFS list failed: {str(e)}")
|
|
@@ -6,12 +6,7 @@ import json
|
|
|
6
6
|
import os
|
|
7
7
|
import threading
|
|
8
8
|
from sqlite3 import DatabaseError
|
|
9
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
10
|
-
|
|
11
|
-
from google.cloud import firestore
|
|
12
|
-
from google.cloud import storage
|
|
13
|
-
from google.cloud.firestore import Client
|
|
14
|
-
from google.cloud.firestore_v1.base_query import FieldFilter
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
|
15
10
|
|
|
16
11
|
from ..base.NoSQLKVAdapter import NoSQLKVAdapter
|
|
17
12
|
from ..errors import ConnectionError, NoSQLError
|
|
@@ -20,6 +15,10 @@ from ..models import PartitionConfig
|
|
|
20
15
|
from ..retry import retry
|
|
21
16
|
from ..types import JsonDict
|
|
22
17
|
|
|
18
|
+
if TYPE_CHECKING: # type-checker only — not imported at runtime
|
|
19
|
+
from google.cloud import storage
|
|
20
|
+
from google.cloud.firestore import Client
|
|
21
|
+
|
|
23
22
|
|
|
24
23
|
class FirestoreAdapter(NoSQLKVAdapter):
|
|
25
24
|
"""
|
|
@@ -31,6 +30,10 @@ class FirestoreAdapter(NoSQLKVAdapter):
|
|
|
31
30
|
- delete() returns {"id": <pk>} and raises DatabaseError on missing
|
|
32
31
|
- query_page() returns (rows, token) with stable pagination
|
|
33
32
|
- Emulator support via FIRESTORE_EMULATOR_HOST
|
|
33
|
+
|
|
34
|
+
NOTE: google-cloud-firestore / google-cloud-storage are imported lazily
|
|
35
|
+
inside the methods that use them, so installing them is only required when
|
|
36
|
+
this adapter is actually used.
|
|
34
37
|
"""
|
|
35
38
|
|
|
36
39
|
FIRESTORE_MAX_SIZE = 1024 * 1024 # 1MB doc limit (practical)
|
|
@@ -66,6 +69,8 @@ class FirestoreAdapter(NoSQLKVAdapter):
|
|
|
66
69
|
# ---------------------------------------------------------------------
|
|
67
70
|
|
|
68
71
|
def _initialize_clients(self) -> None:
|
|
72
|
+
from google.cloud import firestore, storage # lazy: GCP only
|
|
73
|
+
|
|
69
74
|
try:
|
|
70
75
|
with self._lock:
|
|
71
76
|
if self._client:
|
|
@@ -242,6 +247,8 @@ class FirestoreAdapter(NoSQLKVAdapter):
|
|
|
242
247
|
Note: Firestore requires indexes for some compound queries in real GCP.
|
|
243
248
|
Emulator usually allows most.
|
|
244
249
|
"""
|
|
250
|
+
from google.cloud.firestore_v1.base_query import FieldFilter # lazy: GCP only
|
|
251
|
+
|
|
245
252
|
try:
|
|
246
253
|
collection = self._get_collection(model)
|
|
247
254
|
query = collection
|
|
@@ -312,9 +319,7 @@ class FirestoreAdapter(NoSQLKVAdapter):
|
|
|
312
319
|
raise NoSQLError(f"Firestore delete failed: {e}")
|
|
313
320
|
|
|
314
321
|
# ---------------------------------------------------------------------
|
|
315
|
-
# Pagination
|
|
316
|
-
# If your base calls only _query_raw, you can still add a public query_page method
|
|
317
|
-
# in NoSQLKVAdapter; but since your tests call gcp_nosql.query_page(...) we provide it.
|
|
322
|
+
# Pagination
|
|
318
323
|
# ---------------------------------------------------------------------
|
|
319
324
|
|
|
320
325
|
def query_page(
|
|
@@ -327,11 +332,8 @@ class FirestoreAdapter(NoSQLKVAdapter):
|
|
|
327
332
|
) -> Tuple[List[JsonDict], Optional[str]]:
|
|
328
333
|
"""
|
|
329
334
|
Returns (rows, next_token). Token is last document id from the page.
|
|
330
|
-
|
|
331
|
-
Works with your tests:
|
|
332
|
-
page1, tok = gcp_nosql.query_page(GcpItem, {"tenant_id": tag}, 3)
|
|
333
|
-
page2, _ = gcp_nosql.query_page(GcpItem, {"tenant_id": tag}, 3, tok)
|
|
334
335
|
"""
|
|
336
|
+
from google.cloud.firestore_v1.base_query import FieldFilter # lazy: GCP only
|
|
335
337
|
|
|
336
338
|
try:
|
|
337
339
|
collection = self._get_collection(model)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# src/polydb/adapters/GCPFilestoreAdapter.py
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import List, Optional
|
|
5
|
+
|
|
6
|
+
from ..base.SharedFilesAdapter import SharedFilesAdapter
|
|
7
|
+
from ..errors import StorageError, ConnectionError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class FilestoreAdapter(SharedFilesAdapter):
|
|
11
|
+
"""
|
|
12
|
+
GCP Filestore — managed NFS mounted onto the host at a path (e.g. /mnt/filestore).
|
|
13
|
+
Like EFS, it's a POSIX directory once mounted, so ops are plain os/open calls.
|
|
14
|
+
|
|
15
|
+
- per-call `share_name` selects a sub-root directory under the mount point
|
|
16
|
+
- paths are confined to the resolved root (no traversal escapes)
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, mount_point: str = ""):
|
|
20
|
+
super().__init__()
|
|
21
|
+
self.mount_point = mount_point or os.getenv("FILESTORE_MOUNT_POINT", "/mnt/filestore")
|
|
22
|
+
if not self.mount_point:
|
|
23
|
+
raise ConnectionError("Filestore mount_point is required")
|
|
24
|
+
|
|
25
|
+
def _resolve(self, path: str, share_name: Optional[str] = None) -> str:
|
|
26
|
+
base = os.path.join(self.mount_point, share_name) if share_name else self.mount_point
|
|
27
|
+
base_abs = os.path.abspath(base)
|
|
28
|
+
rel = (path or "").lstrip("/\\")
|
|
29
|
+
full_abs = os.path.abspath(os.path.join(base_abs, rel))
|
|
30
|
+
if full_abs != base_abs and not full_abs.startswith(base_abs + os.sep):
|
|
31
|
+
raise StorageError(f"Path escapes storage root: {path}")
|
|
32
|
+
return full_abs
|
|
33
|
+
|
|
34
|
+
def write(self, path: str, data: bytes, share_name: Optional[str] = None) -> bool:
|
|
35
|
+
try:
|
|
36
|
+
full_path = self._resolve(path, share_name)
|
|
37
|
+
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
|
38
|
+
with open(full_path, "wb") as f:
|
|
39
|
+
f.write(data)
|
|
40
|
+
return True
|
|
41
|
+
except StorageError:
|
|
42
|
+
raise
|
|
43
|
+
except Exception as e:
|
|
44
|
+
raise StorageError(f"Filestore write failed: {str(e)}")
|
|
45
|
+
|
|
46
|
+
def read(self, path: str, share_name: Optional[str] = None) -> bytes | None:
|
|
47
|
+
try:
|
|
48
|
+
with open(self._resolve(path, share_name), "rb") as f:
|
|
49
|
+
return f.read()
|
|
50
|
+
except FileNotFoundError:
|
|
51
|
+
return None
|
|
52
|
+
except StorageError:
|
|
53
|
+
raise
|
|
54
|
+
except Exception as e:
|
|
55
|
+
raise StorageError(f"Filestore read failed: {str(e)}")
|
|
56
|
+
|
|
57
|
+
def delete(self, path: str, share_name: Optional[str] = None) -> bool:
|
|
58
|
+
try:
|
|
59
|
+
os.remove(self._resolve(path, share_name))
|
|
60
|
+
return True
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
return False
|
|
63
|
+
except StorageError:
|
|
64
|
+
raise
|
|
65
|
+
except Exception as e:
|
|
66
|
+
raise StorageError(f"Filestore delete failed: {str(e)}")
|
|
67
|
+
|
|
68
|
+
def list(self, directory: str = "", share_name: Optional[str] = None) -> List[str]:
|
|
69
|
+
try:
|
|
70
|
+
full_path = self._resolve(directory, share_name)
|
|
71
|
+
if not os.path.isdir(full_path):
|
|
72
|
+
return []
|
|
73
|
+
return os.listdir(full_path)
|
|
74
|
+
except StorageError:
|
|
75
|
+
raise
|
|
76
|
+
except Exception as e:
|
|
77
|
+
raise StorageError(f"Filestore list failed: {str(e)}")
|
|
@@ -3,7 +3,6 @@ import json
|
|
|
3
3
|
import threading
|
|
4
4
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
5
5
|
|
|
6
|
-
from google.cloud import pubsub_v1
|
|
7
6
|
from google.api_core.exceptions import AlreadyExists, NotFound
|
|
8
7
|
|
|
9
8
|
from ..base.QueueAdapter import QueueAdapter
|
|
@@ -11,7 +10,6 @@ from ..errors import ConnectionError, QueueError
|
|
|
11
10
|
from ..retry import retry
|
|
12
11
|
from ..json_safe import json_safe
|
|
13
12
|
|
|
14
|
-
|
|
15
13
|
JsonLike = Union[Dict[str, Any], List[Any], str, int, float, bool, None]
|
|
16
14
|
|
|
17
15
|
|
|
@@ -40,13 +38,15 @@ class GCPPubSubAdapter(QueueAdapter):
|
|
|
40
38
|
"PUBSUB_SUBSCRIPTION", "polydb-sub"
|
|
41
39
|
)
|
|
42
40
|
|
|
43
|
-
self._publisher
|
|
44
|
-
self._subscriber
|
|
41
|
+
self._publisher = None
|
|
42
|
+
self._subscriber = None
|
|
45
43
|
self._lock = threading.Lock()
|
|
46
44
|
|
|
47
45
|
self._initialize_clients()
|
|
48
46
|
|
|
49
47
|
def _initialize_clients(self) -> None:
|
|
48
|
+
from google.cloud import pubsub_v1
|
|
49
|
+
|
|
50
50
|
try:
|
|
51
51
|
with self._lock:
|
|
52
52
|
if self._publisher and self._subscriber:
|