altcodepro-polydb-python 2.3.31__py3-none-any.whl → 2.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/METADATA +4 -1
- {altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/RECORD +14 -14
- {altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/WHEEL +1 -1
- polydb/adapters/MongoDBAdapter.py +106 -18
- polydb/advanced_query.py +63 -1
- polydb/audit/AuditStorage.py +110 -12
- polydb/audit/models.py +95 -3
- polydb/errors.py +6 -0
- polydb/multitenancy.py +44 -4
- polydb/query.py +46 -7
- polydb/schema.py +36 -9
- polydb/security.py +108 -29
- {altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/top_level.txt +0 -0
{altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: altcodepro-polydb-python
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.2
|
|
4
4
|
Summary: Production-ready multi-cloud database abstraction layer with connection pooling, retry logic, and thread safety
|
|
5
5
|
Author: AltCodePro
|
|
6
6
|
Project-URL: Homepage, https://github.com/altcodepro/polydb-python
|
|
@@ -69,6 +69,8 @@ Provides-Extra: generic
|
|
|
69
69
|
Requires-Dist: pymongo>=4.16.0; extra == "generic"
|
|
70
70
|
Requires-Dist: pika>=1.3.2; extra == "generic"
|
|
71
71
|
Requires-Dist: boto3>=1.42.47; extra == "generic"
|
|
72
|
+
Provides-Extra: security
|
|
73
|
+
Requires-Dist: cryptography>=42.0.0; extra == "security"
|
|
72
74
|
Provides-Extra: all
|
|
73
75
|
Requires-Dist: boto3>=1.42.47; extra == "all"
|
|
74
76
|
Requires-Dist: botocore>=1.42.47; extra == "all"
|
|
@@ -83,6 +85,7 @@ Requires-Dist: google-cloud-storage>=3.9.0; extra == "all"
|
|
|
83
85
|
Requires-Dist: pymongo>=4.16.0; extra == "all"
|
|
84
86
|
Requires-Dist: pika>=1.3.2; extra == "all"
|
|
85
87
|
Requires-Dist: requests>=2.32.5; extra == "all"
|
|
88
|
+
Requires-Dist: cryptography>=42.0.0; extra == "all"
|
|
86
89
|
Provides-Extra: dev
|
|
87
90
|
Requires-Dist: black>=26.1.0; extra == "dev"
|
|
88
91
|
Requires-Dist: flake8>=7.3.0; extra == "dev"
|
{altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/RECORD
RENAMED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
altcodepro_polydb_python-2.
|
|
1
|
+
altcodepro_polydb_python-2.5.2.dist-info/licenses/LICENSE,sha256=9X8GLocsBwy-5aR5JGOt2SAMDDPs9Qv-YnqmHBHOXrw,1067
|
|
2
2
|
polydb/PolyDB.py,sha256=MG7-nV59zDvrUQwXNEgV4eetHCeW9TYY9DQqtxu_r7k,23543
|
|
3
3
|
polydb/__init__.py,sha256=d9KSz888A336904TPEjWPYBtyJLPB0lTdcaOnYp8byY,1535
|
|
4
|
-
polydb/advanced_query.py,sha256=
|
|
4
|
+
polydb/advanced_query.py,sha256=JZUQr4wyC9ydrcO9VbZiH2ueiQiC4dxOX9hzG_JaFhs,6931
|
|
5
5
|
polydb/batch.py,sha256=_DjWZa1ZXYSk6MLKqFe0eT7SYVRZtYNqZb9bI8Y2sao,4566
|
|
6
6
|
polydb/cache.py,sha256=JBXF1XEK-fY80ar8SDE893Z1Z116YtXAEG0PaJ0Nkcw,7658
|
|
7
7
|
polydb/cloudDatabaseFactory.py,sha256=Gp6L__YtgrkGahD8B7ItzXMHCoj2ZUGDjXLS9w0TujY,17780
|
|
8
8
|
polydb/databaseFactory.py,sha256=AdEMRtWcwf0elJ9JhrWS5krre-3RVFfIgt0mA93eJJQ,44068
|
|
9
9
|
polydb/decorators.py,sha256=L_WP2uXP_k8Ac49SUm-mthbM4jWI-XYfHXEyKzumOww,43062
|
|
10
|
-
polydb/errors.py,sha256=
|
|
10
|
+
polydb/errors.py,sha256=ZM3XEefhDMo82v3MMRQ7CRLdKI4N_lYLZN6O0ajBM7M,1712
|
|
11
11
|
polydb/json_safe.py,sha256=R5PrqAGirqjYKPyy-8KH-lSXjLH0FPr2TSGozy4eheU,149
|
|
12
12
|
polydb/models.py,sha256=pkGpYnEEJmXdoWVpWS2K3CnuO9JZ_91DKp1bpVNQCns,10259
|
|
13
13
|
polydb/monitoring.py,sha256=UMm3ybyRJjAQi-prXXMLl9zuHhnhMnYBzMD3XWK66y8,9571
|
|
14
|
-
polydb/multitenancy.py,sha256=
|
|
14
|
+
polydb/multitenancy.py,sha256=BfnSJRRyRl4ObM4noXDJhsGJHLESXzvYRsdKwic7TQc,8135
|
|
15
15
|
polydb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
polydb/query.py,sha256=
|
|
16
|
+
polydb/query.py,sha256=72QbnQaljNGvIWktSD29BII0476MMjrJ-7dQmOcWuZc,8509
|
|
17
17
|
polydb/registry.py,sha256=RD_elvFXcmhTdCyZDm2f3ej0elxqhArnSJ2aO9k5VCU,2352
|
|
18
18
|
polydb/retry.py,sha256=QduWwDyiU-dt1fHFQmm6ltbnaJY2460cDfQ7ZE8wjXs,4466
|
|
19
|
-
polydb/schema.py,sha256=
|
|
20
|
-
polydb/security.py,sha256=
|
|
19
|
+
polydb/schema.py,sha256=hI7uJM4WvH31A-h0kGZhCD5mCYcnrNvEega--_kvafE,6889
|
|
20
|
+
polydb/security.py,sha256=KqV5cD19hLyIvykW2DkKCXuj1hBDLt3xQ1tFtz_sCQE,20202
|
|
21
21
|
polydb/types.py,sha256=XB_85Un8_aWt4dSfpjIGotHbK3KBY2WurQGXr9EOxWY,2992
|
|
22
22
|
polydb/utils.py,sha256=tS_KI1vt-VZ6JftbFxbnVxC0xpie1-zYikTGf1nSoFw,3183
|
|
23
23
|
polydb/validation.py,sha256=a1o1d02k3c6PWQwkBbw_0nEmIgrdB5RR8OcpNQMn4cA,4810
|
|
@@ -35,7 +35,7 @@ polydb/adapters/FirestoreAdapter.py,sha256=e3DEODOZ-UAWULLtGTIED-Ym1JrxOKaOLPQVK
|
|
|
35
35
|
polydb/adapters/GCPFilestoreAdapter.py,sha256=yjFQQwsWYWc8mo8XwMViVTWb5_D--xAyTMvE-4AOpNM,3006
|
|
36
36
|
polydb/adapters/GCPPubSubAdapter.py,sha256=7XNots2VA0ReEDku-rjg-OTYmftIpx5UgnXYDdXNkOo,8692
|
|
37
37
|
polydb/adapters/GCPStorageAdapter.py,sha256=9yS1Jhcn5_rCRdZ5uOqcRW6Ba-UNb6VOYpwENP-C6Qk,7133
|
|
38
|
-
polydb/adapters/MongoDBAdapter.py,sha256=
|
|
38
|
+
polydb/adapters/MongoDBAdapter.py,sha256=qB7RwQb3tvSLLPBiKV_BoAHCvl8Me3C02ieunj01Iew,11788
|
|
39
39
|
polydb/adapters/PostgreSQLAdapter.py,sha256=9VI4-JOdDPeCDba8lRar4RkVMuF0bAymkvSTE3HILjs,38239
|
|
40
40
|
polydb/adapters/S3Adapter.py,sha256=5R0zHAL2SkGFjp1L3bp-IU468bXYdSf6nKx974MN104,7586
|
|
41
41
|
polydb/adapters/S3CompatibleAdapter.py,sha256=jpafqbAjA8-irdXBrfXa1QJySIzrcUQ6UrFt5h5FAEc,7006
|
|
@@ -45,11 +45,11 @@ polydb/adapters/VercelFileAdapter.py,sha256=-fLRCi0AUbyXAR4nkHCV-wXkociHF2hzzEDq
|
|
|
45
45
|
polydb/adapters/VercelKVAdapter.py,sha256=QZxRkuYzVNWFCEFaJPSph8YEAut-YtlXPqbCt0JlROI,8647
|
|
46
46
|
polydb/adapters/VercelQueueAdapter.py,sha256=cWtPaMIWCako0HHr_rzAE6vMLugSR6zBXqp3VP9MXwY,2375
|
|
47
47
|
polydb/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
|
-
polydb/audit/AuditStorage.py,sha256=
|
|
48
|
+
polydb/audit/AuditStorage.py,sha256=6luCS7hTNL5MH2kkn6QlBlBsH-3FLuEFwP-cmkugRss,10469
|
|
49
49
|
polydb/audit/__init__.py,sha256=Z7-y5djq3glQ2Yun6nj-13Efpj3oGz9Qc0veS2g06Y4,245
|
|
50
50
|
polydb/audit/context.py,sha256=-A1FMtmr-2snVAHpTrVT80u-D_MCaqX6AoV4Ku2bz_o,1955
|
|
51
51
|
polydb/audit/manager.py,sha256=KzaaOf5bDfr4M-CkCAZBG_U_4xIBCKDLRAf3hsm-DAk,1236
|
|
52
|
-
polydb/audit/models.py,sha256=
|
|
52
|
+
polydb/audit/models.py,sha256=cSDkzW22FnsMvjQFvLQACQUm9lYFcnJ2Ew5mngBF9yQ,7043
|
|
53
53
|
polydb/base/NoSQLKVAdapter.py,sha256=U64nXaeHd40GCZV4DQnogZRUxvI8HHLEhNBMv4oddq4,14491
|
|
54
54
|
polydb/base/ObjectStorageAdapter.py,sha256=VeJ3qXET6H0xd3lJpE8-WSsKs8EyK9S0-9VNR0EJzME,2288
|
|
55
55
|
polydb/base/QueueAdapter.py,sha256=jFgyG-SUK4nhRNxm2NbzUbwnA9b_5iAC-ikLSUpXRwk,799
|
|
@@ -60,7 +60,7 @@ polydb/observability/logging.py,sha256=gv1SshNjaSkmR98Uv7Qbwq3gkeQUzVCA4ichqHKL6
|
|
|
60
60
|
polydb/services/__init__.py,sha256=nMolMXzmhejRSm7f42kU9fCHF0GIpqpejN2kywuBsUQ,241
|
|
61
61
|
polydb/services/compliance_service.py,sha256=-0ncX7PrXy4-NuyWXIv65cLUsqz0jXzb3LtsD8rlrXE,4983
|
|
62
62
|
polydb/services/security_service.py,sha256=q7GzK6sbMnBBsc4fuL4oXTd-ui2b-MYQyrWTBvdVNSk,5062
|
|
63
|
-
altcodepro_polydb_python-2.
|
|
64
|
-
altcodepro_polydb_python-2.
|
|
65
|
-
altcodepro_polydb_python-2.
|
|
66
|
-
altcodepro_polydb_python-2.
|
|
63
|
+
altcodepro_polydb_python-2.5.2.dist-info/METADATA,sha256=sto5LPHuXLzUHq-h5cuH6g6IaF1DhTo0s_y1o0xidOs,12492
|
|
64
|
+
altcodepro_polydb_python-2.5.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
65
|
+
altcodepro_polydb_python-2.5.2.dist-info/top_level.txt,sha256=WgLFWJoYjUhwvyPxJFl6jYLrVFuBJDX3OABf4ocwk_E,7
|
|
66
|
+
altcodepro_polydb_python-2.5.2.dist-info/RECORD,,
|
|
@@ -14,6 +14,25 @@ from ..types import JsonDict
|
|
|
14
14
|
from ..models import PartitionConfig
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
def _as_literal(value: Any) -> Any:
|
|
18
|
+
"""Wrap a dict value so Mongo compares it instead of executing it.
|
|
19
|
+
|
|
20
|
+
A bare dict on the right-hand side of a field is an *operator expression*
|
|
21
|
+
to Mongo, so a filter value that came from request input can turn
|
|
22
|
+
``{"password": <value>}`` into ``{"password": {"$gt": ""}}`` - a match on
|
|
23
|
+
every document, i.e. filter/auth bypass and full-collection extraction.
|
|
24
|
+
``{"$eq": {...}}`` is exactly the same comparison Mongo performs for a
|
|
25
|
+
literal subdocument, so legitimate exact-match-on-subdocument queries keep
|
|
26
|
+
working while ``$``-operators can no longer be smuggled in through a value.
|
|
27
|
+
"""
|
|
28
|
+
return {"$eq": value} if isinstance(value, dict) else value
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _sanitize_query(query: Dict[str, Any]) -> Dict[str, Any]:
|
|
32
|
+
"""Apply _as_literal to every value of a caller-supplied Mongo query."""
|
|
33
|
+
return {k: _as_literal(v) for k, v in (query or {}).items()}
|
|
34
|
+
|
|
35
|
+
|
|
17
36
|
class MongoDBAdapter(NoSQLKVAdapter):
|
|
18
37
|
"""MongoDB adapter compatible with PolyDB contract"""
|
|
19
38
|
|
|
@@ -31,6 +50,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
31
50
|
|
|
32
51
|
self._client = None
|
|
33
52
|
self._lock = threading.Lock()
|
|
53
|
+
self._indexed_collections: set = set()
|
|
34
54
|
|
|
35
55
|
self._initialize_client()
|
|
36
56
|
|
|
@@ -74,7 +94,73 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
74
94
|
|
|
75
95
|
collection_name = meta.get("collection") or meta.get("table") or model.__name__.lower()
|
|
76
96
|
|
|
77
|
-
|
|
97
|
+
collection = self._client[self.db_name][collection_name] # type: ignore
|
|
98
|
+
|
|
99
|
+
self._ensure_pk_rk_index(collection, collection_name)
|
|
100
|
+
|
|
101
|
+
return collection
|
|
102
|
+
|
|
103
|
+
def _ensure_pk_rk_index(self, collection, collection_name: str) -> None:
|
|
104
|
+
"""
|
|
105
|
+
Create the compound (_pk, _rk) index once per collection.
|
|
106
|
+
|
|
107
|
+
Without it, every _get_raw/_put_raw lookup and query_page's
|
|
108
|
+
`.sort("_pk")` scan the whole collection -- on Cosmos DB's Mongo
|
|
109
|
+
API that's a cross-partition fan-out, billed and latent
|
|
110
|
+
accordingly. Best-effort: a provider that rejects index creation
|
|
111
|
+
(e.g. restricted tier) shouldn't take the adapter down, so a
|
|
112
|
+
failure here is logged once and not retried on every call.
|
|
113
|
+
Note: this is a logical uniqueness/query index only -- it does
|
|
114
|
+
NOT retroactively set Cosmos's physical shard key, which can only
|
|
115
|
+
be chosen at container-creation time.
|
|
116
|
+
"""
|
|
117
|
+
if collection_name in self._indexed_collections:
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
with self._lock:
|
|
121
|
+
if collection_name in self._indexed_collections:
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
collection.create_index(
|
|
126
|
+
[("_pk", 1), ("_rk", 1)], unique=True, name="pk_rk_unique"
|
|
127
|
+
)
|
|
128
|
+
except Exception:
|
|
129
|
+
self.logger.warning(
|
|
130
|
+
"Could not ensure (_pk, _rk) index on %s",
|
|
131
|
+
collection_name,
|
|
132
|
+
exc_info=True,
|
|
133
|
+
)
|
|
134
|
+
finally:
|
|
135
|
+
self._indexed_collections.add(collection_name)
|
|
136
|
+
|
|
137
|
+
# -----------------------------------------------------
|
|
138
|
+
# Internal <-> domain field mapping
|
|
139
|
+
# -----------------------------------------------------
|
|
140
|
+
|
|
141
|
+
def _finalize_doc(self, model: type, doc: JsonDict) -> JsonDict:
|
|
142
|
+
"""
|
|
143
|
+
Strip PolyDB's internal _pk/_rk storage keys before returning a
|
|
144
|
+
record, mapping each back to the domain field it was derived from
|
|
145
|
+
(the model's declared partition_key/sort_key) -- mirrors
|
|
146
|
+
AzureTableStorageAdapter._unpack_entity, which already does this
|
|
147
|
+
for PartitionKey/RowKey. Without it, callers see raw storage-
|
|
148
|
+
internal column names instead of their own schema's field names.
|
|
149
|
+
Guarded with setdefault so a real stored property never gets
|
|
150
|
+
clobbered by the derived value.
|
|
151
|
+
"""
|
|
152
|
+
pk_field, rk_field = self._pk_rk_field_names(model)
|
|
153
|
+
pk_val = doc.pop("_pk", None)
|
|
154
|
+
rk_val = doc.pop("_rk", None)
|
|
155
|
+
|
|
156
|
+
if pk_val is not None:
|
|
157
|
+
doc.setdefault(pk_field, pk_val)
|
|
158
|
+
if rk_val is not None:
|
|
159
|
+
doc.setdefault(rk_field, rk_val)
|
|
160
|
+
|
|
161
|
+
doc.setdefault("id", rk_val)
|
|
162
|
+
|
|
163
|
+
return doc
|
|
78
164
|
|
|
79
165
|
# -----------------------------------------------------
|
|
80
166
|
# PUT
|
|
@@ -87,7 +173,11 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
87
173
|
payload = dict(data or {})
|
|
88
174
|
payload["_pk"] = pk
|
|
89
175
|
payload["_rk"] = rk
|
|
90
|
-
|
|
176
|
+
# _rk is the record's natural identifier (the base class
|
|
177
|
+
# already falls back to `id` when a model has no explicit
|
|
178
|
+
# sort_key); _pk is just the tenant/partition grouping, so it
|
|
179
|
+
# must never overwrite a caller-supplied `id`.
|
|
180
|
+
payload.setdefault("id", rk)
|
|
91
181
|
|
|
92
182
|
collection.update_one(
|
|
93
183
|
{"_pk": pk, "_rk": rk},
|
|
@@ -96,11 +186,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
96
186
|
)
|
|
97
187
|
|
|
98
188
|
# return full stored row (tests expect this)
|
|
99
|
-
|
|
100
|
-
result.pop("_pk", None)
|
|
101
|
-
result.pop("_rk", None)
|
|
102
|
-
|
|
103
|
-
return result
|
|
189
|
+
return self._finalize_doc(model, dict(payload))
|
|
104
190
|
|
|
105
191
|
except Exception as e:
|
|
106
192
|
raise NoSQLError(f"MongoDB put failed: {e}")
|
|
@@ -120,9 +206,8 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
120
206
|
return None
|
|
121
207
|
|
|
122
208
|
doc.pop("_id", None)
|
|
123
|
-
doc.setdefault("id", pk)
|
|
124
209
|
|
|
125
|
-
return doc
|
|
210
|
+
return self._finalize_doc(model, doc)
|
|
126
211
|
|
|
127
212
|
except Exception as e:
|
|
128
213
|
raise NoSQLError(f"MongoDB get failed: {e}")
|
|
@@ -140,7 +225,9 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
140
225
|
try:
|
|
141
226
|
collection = self._get_collection(model)
|
|
142
227
|
|
|
143
|
-
|
|
228
|
+
# Sanitize before adding our own operator clause below, so the
|
|
229
|
+
# pagination cursor isn't stripped along with injected operators.
|
|
230
|
+
query = _sanitize_query(query)
|
|
144
231
|
|
|
145
232
|
if continuation_token:
|
|
146
233
|
query["_pk"] = {"$gt": continuation_token}
|
|
@@ -151,8 +238,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
151
238
|
|
|
152
239
|
for doc in cursor:
|
|
153
240
|
doc.pop("_id", None)
|
|
154
|
-
|
|
155
|
-
rows.append(doc)
|
|
241
|
+
rows.append(self._finalize_doc(model, doc))
|
|
156
242
|
|
|
157
243
|
if not rows:
|
|
158
244
|
return [], None
|
|
@@ -179,7 +265,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
179
265
|
for k, v in (filters or {}).items():
|
|
180
266
|
|
|
181
267
|
if k == "id":
|
|
182
|
-
query["
|
|
268
|
+
query["_rk"] = _as_literal(v)
|
|
183
269
|
continue
|
|
184
270
|
|
|
185
271
|
if k.endswith("__gt"):
|
|
@@ -195,7 +281,10 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
195
281
|
query[k[:-5]] = {"$lte": v}
|
|
196
282
|
|
|
197
283
|
elif k.endswith("__in"):
|
|
198
|
-
|
|
284
|
+
# $in operands are matched literally by Mongo, but the list
|
|
285
|
+
# itself has to be a list - a dict here would be read as an
|
|
286
|
+
# operator document.
|
|
287
|
+
query[k[:-4]] = {"$in": list(v) if isinstance(v, (list, tuple, set)) else [v]}
|
|
199
288
|
|
|
200
289
|
elif k.endswith("__contains"):
|
|
201
290
|
safe_pattern = re.escape(str(v))
|
|
@@ -205,7 +294,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
205
294
|
}
|
|
206
295
|
|
|
207
296
|
else:
|
|
208
|
-
query[k] = v
|
|
297
|
+
query[k] = _as_literal(v)
|
|
209
298
|
|
|
210
299
|
cursor = collection.find(query)
|
|
211
300
|
|
|
@@ -216,8 +305,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
216
305
|
|
|
217
306
|
for doc in cursor:
|
|
218
307
|
doc.pop("_id", None)
|
|
219
|
-
|
|
220
|
-
results.append(doc)
|
|
308
|
+
results.append(self._finalize_doc(model, doc))
|
|
221
309
|
|
|
222
310
|
return results
|
|
223
311
|
|
|
@@ -238,7 +326,7 @@ class MongoDBAdapter(NoSQLKVAdapter):
|
|
|
238
326
|
if result.deleted_count == 0:
|
|
239
327
|
raise DatabaseError(f"Item {pk}/{rk} does not exist")
|
|
240
328
|
|
|
241
|
-
return {"id":
|
|
329
|
+
return {"id": rk}
|
|
242
330
|
|
|
243
331
|
except DatabaseError:
|
|
244
332
|
raise
|
polydb/advanced_query.py
CHANGED
|
@@ -5,6 +5,45 @@ Advanced query capabilities: JOIN, subqueries, aggregates
|
|
|
5
5
|
from typing import List, Optional, Any, Dict
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
from enum import Enum
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
from .errors import ValidationError
|
|
11
|
+
from .utils import validate_table_name, validate_column_name
|
|
12
|
+
|
|
13
|
+
# table.column / alias.column - each half validated with the same allowlists
|
|
14
|
+
# used elsewhere for identifiers, joined by a single literal dot.
|
|
15
|
+
_QUALIFIED_COLUMN_RE = re.compile(r"^[a-zA-Z0-9_-]+\.[a-zA-Z0-9_]+$")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _validate_qualified_column(value: str) -> str:
|
|
19
|
+
"""Validate a column reference that may be table/alias-qualified (a.b) or bare (b)."""
|
|
20
|
+
if not isinstance(value, str) or not value:
|
|
21
|
+
raise ValidationError(f"Invalid column reference: {value!r}")
|
|
22
|
+
if _QUALIFIED_COLUMN_RE.match(value):
|
|
23
|
+
return value
|
|
24
|
+
return validate_column_name(value)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Safe HAVING grammar: one or more `FUNC(col) OP operand` comparisons joined by
|
|
28
|
+
# AND/OR. No string literals, no semicolons/comments, no subqueries - this is
|
|
29
|
+
# an allowlist, not a sanitizer, so anything outside this shape is rejected.
|
|
30
|
+
_IDENT = r"[A-Za-z_][A-Za-z0-9_]*"
|
|
31
|
+
_QUALIFIED = rf"{_IDENT}(?:\.{_IDENT})?"
|
|
32
|
+
_FUNC_CALL = rf"{_IDENT}\(\s*(?:\*|{_QUALIFIED})\s*\)"
|
|
33
|
+
_OPERAND = rf"(?:{_FUNC_CALL}|{_QUALIFIED}|-?\d+(?:\.\d+)?)"
|
|
34
|
+
_COMPARISON_OP = r"(?:!=|<>|>=|<=|=|<|>)"
|
|
35
|
+
_CONDITION = rf"{_OPERAND}\s*{_COMPARISON_OP}\s*{_OPERAND}"
|
|
36
|
+
_HAVING_RE = re.compile(rf"^{_CONDITION}(?:\s+(?:AND|OR)\s+{_CONDITION})*$", re.IGNORECASE)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _validate_having_condition(condition: str) -> str:
|
|
40
|
+
if not isinstance(condition, str) or not _HAVING_RE.match(condition.strip()):
|
|
41
|
+
raise ValidationError(
|
|
42
|
+
f"Invalid HAVING condition: {condition!r}. Only simple aggregate "
|
|
43
|
+
'comparisons such as "COUNT(id) > 5" (optionally chained with AND/OR) '
|
|
44
|
+
"are allowed - no string literals, subqueries, or statement separators."
|
|
45
|
+
)
|
|
46
|
+
return condition
|
|
8
47
|
|
|
9
48
|
|
|
10
49
|
class JoinType(Enum):
|
|
@@ -49,6 +88,13 @@ class AdvancedQueryBuilder:
|
|
|
49
88
|
having_conditions: List[str] = field(default_factory=list)
|
|
50
89
|
subqueries: Dict[str, "AdvancedQueryBuilder"] = field(default_factory=dict)
|
|
51
90
|
|
|
91
|
+
def __post_init__(self) -> None:
|
|
92
|
+
validate_table_name(self.table)
|
|
93
|
+
for gf in self.group_by_fields:
|
|
94
|
+
_validate_qualified_column(gf)
|
|
95
|
+
for cond in self.having_conditions:
|
|
96
|
+
_validate_having_condition(cond)
|
|
97
|
+
|
|
52
98
|
def join(
|
|
53
99
|
self,
|
|
54
100
|
table: str,
|
|
@@ -58,6 +104,11 @@ class AdvancedQueryBuilder:
|
|
|
58
104
|
alias: Optional[str] = None,
|
|
59
105
|
) -> "AdvancedQueryBuilder":
|
|
60
106
|
"""Add JOIN clause"""
|
|
107
|
+
validate_table_name(table)
|
|
108
|
+
_validate_qualified_column(on_left)
|
|
109
|
+
_validate_qualified_column(on_right)
|
|
110
|
+
if alias is not None:
|
|
111
|
+
validate_column_name(alias)
|
|
61
112
|
self.joins.append(
|
|
62
113
|
Join(table=table, join_type=join_type, on_left=on_left, on_right=on_right, alias=alias)
|
|
63
114
|
)
|
|
@@ -67,16 +118,27 @@ class AdvancedQueryBuilder:
|
|
|
67
118
|
self, function: AggregateFunction, field: str, alias: str
|
|
68
119
|
) -> "AdvancedQueryBuilder":
|
|
69
120
|
"""Add aggregate function"""
|
|
121
|
+
if field != "*":
|
|
122
|
+
_validate_qualified_column(field)
|
|
123
|
+
validate_column_name(alias)
|
|
70
124
|
self.aggregates.append(Aggregate(function=function, field=field, alias=alias))
|
|
71
125
|
return self
|
|
72
126
|
|
|
73
127
|
def group_by(self, *fields: str) -> "AdvancedQueryBuilder":
|
|
74
128
|
"""Add GROUP BY"""
|
|
129
|
+
for f in fields:
|
|
130
|
+
_validate_qualified_column(f)
|
|
75
131
|
self.group_by_fields.extend(fields)
|
|
76
132
|
return self
|
|
77
133
|
|
|
78
134
|
def having(self, condition: str) -> "AdvancedQueryBuilder":
|
|
79
|
-
"""Add HAVING clause
|
|
135
|
+
"""Add HAVING clause.
|
|
136
|
+
|
|
137
|
+
`condition` must match the safe aggregate-comparison grammar enforced
|
|
138
|
+
by `_validate_having_condition` (e.g. "COUNT(id) > 5") - free-form SQL
|
|
139
|
+
text is rejected to prevent HAVING-clause injection.
|
|
140
|
+
"""
|
|
141
|
+
_validate_having_condition(condition)
|
|
80
142
|
self.having_conditions.append(condition)
|
|
81
143
|
return self
|
|
82
144
|
|
polydb/audit/AuditStorage.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# src/polydb/audit/AuditStorage.py
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import hmac
|
|
4
5
|
import threading
|
|
6
|
+
from dataclasses import dataclass
|
|
5
7
|
from typing import Optional, Dict, Any
|
|
6
8
|
|
|
7
9
|
from .models import AuditRecord
|
|
@@ -97,7 +99,20 @@ class AuditStorage:
|
|
|
97
99
|
and re-chains instead of forking. (threading.Lock alone was only
|
|
98
100
|
process-local — the old "distributed-safe" claim was false.)"""
|
|
99
101
|
from dataclasses import asdict
|
|
100
|
-
from .models import
|
|
102
|
+
from .models import (
|
|
103
|
+
AuditKeyMissingError,
|
|
104
|
+
AUDIT_HMAC_KEY_ENV,
|
|
105
|
+
audit_hmac_key,
|
|
106
|
+
compute_audit_hash,
|
|
107
|
+
require_hmac,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
if require_hmac() and audit_hmac_key() is None:
|
|
111
|
+
raise AuditKeyMissingError(
|
|
112
|
+
f"{AUDIT_HMAC_KEY_ENV} is unset. An unkeyed audit hash can be "
|
|
113
|
+
"recomputed by anyone who can write the log, so this "
|
|
114
|
+
"deployment has asked to refuse writing one."
|
|
115
|
+
)
|
|
101
116
|
|
|
102
117
|
last_err: Optional[Exception] = None
|
|
103
118
|
for _ in range(8):
|
|
@@ -139,11 +154,38 @@ class AuditStorage:
|
|
|
139
154
|
raise last_err or RuntimeError("audit persist failed after retries")
|
|
140
155
|
|
|
141
156
|
def verify_chain(self, tenant_id: Optional[str] = None) -> bool:
|
|
142
|
-
"""Verify BOTH chain linkage AND per-record content integrity.
|
|
143
|
-
|
|
144
|
-
|
|
157
|
+
"""Verify BOTH chain linkage AND per-record content integrity."""
|
|
158
|
+
return self.verify_chain_detailed(tenant_id).ok
|
|
159
|
+
|
|
160
|
+
def verify_chain_detailed(
|
|
161
|
+
self, tenant_id: Optional[str] = None
|
|
162
|
+
) -> "ChainVerification":
|
|
163
|
+
"""Verify the chain, reporting *why* rather than only whether.
|
|
164
|
+
|
|
165
|
+
Two things are checked per record: that it links to its predecessor,
|
|
166
|
+
and that its own digest still matches its content. The second is what
|
|
167
|
+
catches an editor who changed ``before``/``after``/``action`` and left
|
|
168
|
+
``hash`` alone.
|
|
169
|
+
|
|
170
|
+
Records written before ``POLYDB_AUDIT_HMAC_KEY`` was configured carry
|
|
171
|
+
an unkeyed SHA-256 and cannot be re-verified with the key. They are
|
|
172
|
+
accepted, but only *before* the first keyed record in the chain -- and
|
|
173
|
+
that restriction is what keeps it from being a loophole rather than a
|
|
174
|
+
migration path:
|
|
175
|
+
|
|
176
|
+
* An attacker cannot append unkeyed records after the cutover, because
|
|
177
|
+
once a keyed record has been seen every later record must be keyed.
|
|
178
|
+
* An attacker cannot rewrite the legacy prefix either. Rewriting record
|
|
179
|
+
N means recomputing ``hash(N)``, which is ``previous_hash`` inside
|
|
180
|
+
record N+1's payload -- and if N+1 is keyed, its digest cannot be
|
|
181
|
+
recomputed without the key.
|
|
182
|
+
|
|
183
|
+
So configuring a key seals everything written up to that moment as well
|
|
184
|
+
as everything after it. ``legacy_records`` reports how much of the
|
|
185
|
+
chain is still resting on that seal.
|
|
186
|
+
"""
|
|
145
187
|
from ..query import QueryBuilder, Operator
|
|
146
|
-
from .models import compute_audit_hash
|
|
188
|
+
from .models import audit_hmac_key, compute_audit_hash
|
|
147
189
|
|
|
148
190
|
builder = QueryBuilder()
|
|
149
191
|
if tenant_id is not None:
|
|
@@ -152,13 +194,69 @@ class AuditStorage:
|
|
|
152
194
|
|
|
153
195
|
records = self.sql.query_linq("polydb_audit_log", builder)
|
|
154
196
|
if not records:
|
|
155
|
-
return True
|
|
197
|
+
return ChainVerification(ok=True)
|
|
156
198
|
|
|
199
|
+
key = audit_hmac_key()
|
|
157
200
|
prev = ""
|
|
158
|
-
|
|
201
|
+
legacy = 0
|
|
202
|
+
seen_keyed = False
|
|
203
|
+
|
|
204
|
+
for index, r in enumerate(records):
|
|
205
|
+
audit_id = r.get("audit_id")
|
|
206
|
+
|
|
159
207
|
if (r.get("previous_hash") or "") != prev:
|
|
160
|
-
return
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
208
|
+
return ChainVerification(
|
|
209
|
+
ok=False,
|
|
210
|
+
broken_at=index,
|
|
211
|
+
audit_id=audit_id,
|
|
212
|
+
reason="link",
|
|
213
|
+
legacy_records=legacy,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
stored = r.get("hash") or ""
|
|
217
|
+
|
|
218
|
+
if key is not None and hmac.compare_digest(
|
|
219
|
+
stored, compute_audit_hash(r, key=key)
|
|
220
|
+
):
|
|
221
|
+
seen_keyed = True
|
|
222
|
+
elif not seen_keyed and hmac.compare_digest(
|
|
223
|
+
stored, compute_audit_hash(r, key=None)
|
|
224
|
+
):
|
|
225
|
+
# Predates the key. Sealed by the first keyed record that
|
|
226
|
+
# follows it, so it is only trusted while none has appeared.
|
|
227
|
+
legacy += 1
|
|
228
|
+
else:
|
|
229
|
+
# Distinguish the two ways this arrives. If the *unkeyed*
|
|
230
|
+
# digest matches the record as stored, the content is intact
|
|
231
|
+
# and the record simply was not keyed - which after the
|
|
232
|
+
# cutover is what stripping the key looks like. Otherwise the
|
|
233
|
+
# record itself was edited.
|
|
234
|
+
downgraded = seen_keyed and hmac.compare_digest(
|
|
235
|
+
stored, compute_audit_hash(r, key=None)
|
|
236
|
+
)
|
|
237
|
+
return ChainVerification(
|
|
238
|
+
ok=False,
|
|
239
|
+
broken_at=index,
|
|
240
|
+
audit_id=audit_id,
|
|
241
|
+
reason="unkeyed_after_cutover" if downgraded else "content",
|
|
242
|
+
legacy_records=legacy,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
prev = stored
|
|
246
|
+
|
|
247
|
+
return ChainVerification(ok=True, legacy_records=legacy)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@dataclass(frozen=True)
|
|
251
|
+
class ChainVerification:
|
|
252
|
+
"""The outcome of :meth:`AuditStorage.verify_chain_detailed`."""
|
|
253
|
+
|
|
254
|
+
ok: bool
|
|
255
|
+
broken_at: Optional[int] = None
|
|
256
|
+
audit_id: Optional[str] = None
|
|
257
|
+
#: "link" (previous_hash mismatch), "content" (record edited), or
|
|
258
|
+
#: "unkeyed_after_cutover" (an unkeyed record appended after keying began,
|
|
259
|
+
#: which is what stripping the key would look like).
|
|
260
|
+
reason: Optional[str] = None
|
|
261
|
+
#: How many leading records still carry a pre-key unkeyed digest.
|
|
262
|
+
legacy_records: int = 0
|
polydb/audit/models.py
CHANGED
|
@@ -1,13 +1,71 @@
|
|
|
1
1
|
# src/polydb/audit/models.py
|
|
2
|
+
"""The audit record and its integrity hash.
|
|
3
|
+
|
|
4
|
+
The chain used a plain SHA-256 over the record's canonical payload. That
|
|
5
|
+
detects accidental corruption and it detects an editor who forgets to
|
|
6
|
+
recompute the hash - but it does not detect the adversary an audit log exists
|
|
7
|
+
for. Anyone who can write the table can also recompute SHA-256, so they can
|
|
8
|
+
rewrite a record, re-hash it, and re-hash every record after it: the chain
|
|
9
|
+
still verifies, and the tampering is invisible. The digest was unkeyed, so
|
|
10
|
+
"can write the log" implied "can forge the log".
|
|
11
|
+
|
|
12
|
+
``POLYDB_AUDIT_HMAC_KEY`` turns the digest into HMAC-SHA256. The output is
|
|
13
|
+
still 64 hex characters, so it fits the existing ``VARCHAR(64)`` columns and
|
|
14
|
+
needs no migration - only the way the value is derived changes.
|
|
15
|
+
|
|
16
|
+
Records written before a key was configured stay verifiable. See
|
|
17
|
+
``AuditStorage.verify_chain`` for the cutover rule that makes that safe rather
|
|
18
|
+
than a loophole.
|
|
19
|
+
"""
|
|
2
20
|
|
|
3
21
|
from dataclasses import dataclass, asdict
|
|
4
22
|
from typing import Any, Dict, List, Optional
|
|
5
|
-
import uuid
|
|
6
23
|
import hashlib
|
|
24
|
+
import hmac
|
|
7
25
|
import json
|
|
26
|
+
import logging
|
|
27
|
+
import os
|
|
28
|
+
import uuid
|
|
8
29
|
from datetime import datetime, timezone
|
|
9
30
|
from ..json_safe import json_safe
|
|
10
31
|
|
|
32
|
+
logger = logging.getLogger("polydb.audit")
|
|
33
|
+
|
|
34
|
+
#: Env var holding the audit HMAC key. Absent means legacy unkeyed hashing.
|
|
35
|
+
AUDIT_HMAC_KEY_ENV = "POLYDB_AUDIT_HMAC_KEY"
|
|
36
|
+
|
|
37
|
+
#: Set to 1/true/yes to refuse to write audit records without a key, for
|
|
38
|
+
#: deployments that would rather fail than keep a forgeable log.
|
|
39
|
+
AUDIT_REQUIRE_HMAC_ENV = "POLYDB_AUDIT_REQUIRE_HMAC"
|
|
40
|
+
|
|
41
|
+
_warned_no_key = False
|
|
42
|
+
|
|
43
|
+
#: Distinguishes "caller did not specify a key" from "caller specified None",
|
|
44
|
+
#: which verification needs in order to check the legacy variant deliberately.
|
|
45
|
+
_UNSET = object()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class AuditKeyMissingError(RuntimeError):
|
|
49
|
+
"""No audit HMAC key, and the deployment asked to require one."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def audit_hmac_key() -> Optional[bytes]:
|
|
53
|
+
"""The configured audit key, or ``None``.
|
|
54
|
+
|
|
55
|
+
Read per call rather than cached so a key rotated into the environment
|
|
56
|
+
takes effect without a restart, and so tests can set it per case.
|
|
57
|
+
"""
|
|
58
|
+
raw = os.getenv(AUDIT_HMAC_KEY_ENV, "").strip()
|
|
59
|
+
return raw.encode("utf-8") if raw else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def require_hmac() -> bool:
|
|
63
|
+
return os.getenv(AUDIT_REQUIRE_HMAC_ENV, "").strip().lower() in (
|
|
64
|
+
"1",
|
|
65
|
+
"true",
|
|
66
|
+
"yes",
|
|
67
|
+
)
|
|
68
|
+
|
|
11
69
|
|
|
12
70
|
def _iso(ts: Any) -> str:
|
|
13
71
|
return ts.isoformat() if hasattr(ts, "isoformat") else str(ts)
|
|
@@ -43,8 +101,42 @@ def canonical_audit_payload(src: Dict[str, Any]) -> str:
|
|
|
43
101
|
return json.dumps(payload, sort_keys=True, default=json_safe)
|
|
44
102
|
|
|
45
103
|
|
|
46
|
-
def compute_audit_hash(
|
|
47
|
-
|
|
104
|
+
def compute_audit_hash(
|
|
105
|
+
src: Dict[str, Any], *, key: Optional[bytes] = _UNSET # type: ignore[assignment]
|
|
106
|
+
) -> str:
|
|
107
|
+
"""The record's integrity digest, keyed when a key is available.
|
|
108
|
+
|
|
109
|
+
``key`` defaults to whatever the environment provides; pass it explicitly
|
|
110
|
+
(including ``None``) to compute a specific variant, which is what
|
|
111
|
+
verification does when it has to check a record written under the other
|
|
112
|
+
scheme.
|
|
113
|
+
"""
|
|
114
|
+
if key is _UNSET:
|
|
115
|
+
key = audit_hmac_key()
|
|
116
|
+
if key is None:
|
|
117
|
+
_warn_unkeyed_once()
|
|
118
|
+
|
|
119
|
+
payload = canonical_audit_payload(src).encode()
|
|
120
|
+
if key is None:
|
|
121
|
+
return hashlib.sha256(payload).hexdigest()
|
|
122
|
+
return hmac.new(key, payload, hashlib.sha256).hexdigest()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _warn_unkeyed_once() -> None:
|
|
126
|
+
global _warned_no_key
|
|
127
|
+
if _warned_no_key:
|
|
128
|
+
return
|
|
129
|
+
_warned_no_key = True
|
|
130
|
+
logger.warning(
|
|
131
|
+
"Audit records are being hashed without a key (%s is unset). The "
|
|
132
|
+
"chain still detects accidental corruption, but anyone who can write "
|
|
133
|
+
"polydb_audit_log can recompute it, so it does not detect deliberate "
|
|
134
|
+
"tampering. Set %s to seal the log; set %s=1 to refuse to write "
|
|
135
|
+
"without one.",
|
|
136
|
+
AUDIT_HMAC_KEY_ENV,
|
|
137
|
+
AUDIT_HMAC_KEY_ENV,
|
|
138
|
+
AUDIT_REQUIRE_HMAC_ENV,
|
|
139
|
+
)
|
|
48
140
|
|
|
49
141
|
|
|
50
142
|
@dataclass
|
polydb/errors.py
CHANGED
|
@@ -86,3 +86,9 @@ class InsufficientBalanceError(PolyDBError):
|
|
|
86
86
|
"""Raised when an atomic decrement is attempted but the balance is insufficient."""
|
|
87
87
|
|
|
88
88
|
pass
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class EncryptionConfigError(PolyDBError):
|
|
92
|
+
"""Raised when field encryption is requested but no usable key is configured."""
|
|
93
|
+
|
|
94
|
+
pass
|
polydb/multitenancy.py
CHANGED
|
@@ -2,11 +2,43 @@
|
|
|
2
2
|
"""
|
|
3
3
|
Multi-tenancy enforcement and isolation
|
|
4
4
|
"""
|
|
5
|
+
import re
|
|
5
6
|
from typing import Dict, Any, List, Optional, Callable
|
|
6
7
|
from contextvars import ContextVar
|
|
7
8
|
from dataclasses import dataclass, field
|
|
8
9
|
from enum import Enum
|
|
9
10
|
|
|
11
|
+
from .errors import ValidationError
|
|
12
|
+
|
|
13
|
+
# Schema/database names get interpolated into DDL, which cannot be
|
|
14
|
+
# parameterised - a bind parameter is a *value*, and these are identifiers.
|
|
15
|
+
# So they need the same allowlist the SQL adapter already applies to every
|
|
16
|
+
# table and column name via utils.validate_table_name. These DDL paths were
|
|
17
|
+
# the one place that skipped it.
|
|
18
|
+
_SQL_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
19
|
+
#: Longest identifier PostgreSQL accepts.
|
|
20
|
+
_MAX_IDENTIFIER_LENGTH = 63
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _validate_sql_identifier(value: str, *, kind: str) -> str:
|
|
24
|
+
"""Allowlist a schema/database name before it is spliced into DDL.
|
|
25
|
+
|
|
26
|
+
Rejects anything that is not a bare identifier, so a tenant-derived name
|
|
27
|
+
such as ``x; DROP SCHEMA public CASCADE; --`` can never reach the database.
|
|
28
|
+
"""
|
|
29
|
+
if not isinstance(value, str) or not value:
|
|
30
|
+
raise ValidationError(f"Invalid {kind}: {value!r} (must be a non-empty string)")
|
|
31
|
+
if len(value) > _MAX_IDENTIFIER_LENGTH:
|
|
32
|
+
raise ValidationError(
|
|
33
|
+
f"Invalid {kind}: {value!r} exceeds {_MAX_IDENTIFIER_LENGTH} characters"
|
|
34
|
+
)
|
|
35
|
+
if not _SQL_IDENTIFIER_RE.match(value):
|
|
36
|
+
raise ValidationError(
|
|
37
|
+
f"Invalid {kind}: {value!r}. Only letters, digits and underscores are "
|
|
38
|
+
"allowed, and it may not start with a digit."
|
|
39
|
+
)
|
|
40
|
+
return value
|
|
41
|
+
|
|
10
42
|
|
|
11
43
|
class IsolationLevel(Enum):
|
|
12
44
|
"""Tenant isolation levels"""
|
|
@@ -172,12 +204,16 @@ class TenantMigrationManager:
|
|
|
172
204
|
|
|
173
205
|
if config.isolation_level == IsolationLevel.SEPARATE_SCHEMA:
|
|
174
206
|
# Create schema
|
|
175
|
-
|
|
207
|
+
schema = _validate_sql_identifier(config.schema_name, kind="schema_name")
|
|
208
|
+
schema_sql = f"CREATE SCHEMA IF NOT EXISTS {schema};"
|
|
176
209
|
self.factory._sql.execute(schema_sql)
|
|
177
210
|
|
|
178
211
|
elif config.isolation_level == IsolationLevel.SEPARATE_DATABASE:
|
|
179
212
|
# Create database (requires superuser)
|
|
180
|
-
|
|
213
|
+
database = _validate_sql_identifier(
|
|
214
|
+
config.database_name, kind="database_name"
|
|
215
|
+
)
|
|
216
|
+
db_sql = f"CREATE DATABASE {database};"
|
|
181
217
|
self.factory._sql.execute(db_sql)
|
|
182
218
|
|
|
183
219
|
def deprovision_tenant(self, tenant_id: str):
|
|
@@ -188,10 +224,14 @@ class TenantMigrationManager:
|
|
|
188
224
|
|
|
189
225
|
if config.isolation_level == IsolationLevel.SEPARATE_SCHEMA:
|
|
190
226
|
# Drop schema
|
|
191
|
-
|
|
227
|
+
schema = _validate_sql_identifier(config.schema_name, kind="schema_name")
|
|
228
|
+
schema_sql = f"DROP SCHEMA IF EXISTS {schema} CASCADE;"
|
|
192
229
|
self.factory._sql.execute(schema_sql)
|
|
193
230
|
|
|
194
231
|
elif config.isolation_level == IsolationLevel.SEPARATE_DATABASE:
|
|
195
232
|
# Drop database
|
|
196
|
-
|
|
233
|
+
database = _validate_sql_identifier(
|
|
234
|
+
config.database_name, kind="database_name"
|
|
235
|
+
)
|
|
236
|
+
db_sql = f"DROP DATABASE IF EXISTS {database};"
|
|
197
237
|
self.factory._sql.execute(db_sql)
|
polydb/query.py
CHANGED
|
@@ -2,10 +2,48 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
from dataclasses import dataclass, field
|
|
5
|
-
from typing import Any, Dict, List, Optional, Union
|
|
5
|
+
from typing import Any, Dict, List, Mapping, Optional, Union
|
|
6
6
|
from enum import Enum
|
|
7
|
+
from .errors import ValidationError
|
|
7
8
|
from .utils import validate_column_name
|
|
8
9
|
|
|
10
|
+
#: Escape character used for LIKE patterns built from user values.
|
|
11
|
+
LIKE_ESCAPE_CHAR = "\\"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _escape_like(value: Any) -> str:
|
|
15
|
+
"""Neutralise LIKE metacharacters in a value before it is wrapped in wildcards.
|
|
16
|
+
|
|
17
|
+
``CONTAINS``/``STARTS_WITH``/``ENDS_WITH`` mean "this literal substring".
|
|
18
|
+
The value is bound as a parameter (so there is no SQL injection here), but
|
|
19
|
+
an unescaped ``%`` or ``_`` in it is still interpreted by the pattern
|
|
20
|
+
matcher: it silently widens the match, and a leading ``%`` turns an
|
|
21
|
+
index-friendly prefix scan into a full scan.
|
|
22
|
+
"""
|
|
23
|
+
text = str(value)
|
|
24
|
+
for char in (LIKE_ESCAPE_CHAR, "%", "_"):
|
|
25
|
+
text = text.replace(char, LIKE_ESCAPE_CHAR + char)
|
|
26
|
+
return text
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _reject_mapping_value(f: "QueryFilter") -> None:
|
|
30
|
+
"""Refuse dict filter values on the NoSQL path.
|
|
31
|
+
|
|
32
|
+
A document store reads a dict on the right-hand side of a field as an
|
|
33
|
+
operator expression, so a filter value that reaches here straight from
|
|
34
|
+
request input - ``where("password", EQ, {"$gt": ""})`` - stops being a
|
|
35
|
+
comparison and becomes "any document with a password", i.e. auth bypass
|
|
36
|
+
plus full-collection extraction. Nothing goes through QueryBuilder with a
|
|
37
|
+
legitimate dict value; lists stay allowed because ``IN`` needs them (and a
|
|
38
|
+
list can never be read as an operator).
|
|
39
|
+
"""
|
|
40
|
+
values = f.value if isinstance(f.value, (list, tuple, set)) else [f.value]
|
|
41
|
+
if isinstance(f.value, Mapping) or any(isinstance(v, Mapping) for v in values):
|
|
42
|
+
raise ValidationError(
|
|
43
|
+
f"Invalid filter value for '{f.field}': dict values are not allowed "
|
|
44
|
+
"in NoSQL filters (they would be interpreted as query operators)."
|
|
45
|
+
)
|
|
46
|
+
|
|
9
47
|
|
|
10
48
|
class Operator(Enum):
|
|
11
49
|
EQ = "=="
|
|
@@ -175,16 +213,16 @@ class QueryBuilder:
|
|
|
175
213
|
params.extend(f.value)
|
|
176
214
|
|
|
177
215
|
elif f.operator == Operator.CONTAINS:
|
|
178
|
-
clauses.append(f"{f.field} LIKE %s")
|
|
179
|
-
params.append(f"%{f.value}%")
|
|
216
|
+
clauses.append(f"{f.field} LIKE %s ESCAPE '{LIKE_ESCAPE_CHAR}'")
|
|
217
|
+
params.append(f"%{_escape_like(f.value)}%")
|
|
180
218
|
|
|
181
219
|
elif f.operator == Operator.STARTS_WITH:
|
|
182
|
-
clauses.append(f"{f.field} LIKE %s")
|
|
183
|
-
params.append(f"{f.value}%")
|
|
220
|
+
clauses.append(f"{f.field} LIKE %s ESCAPE '{LIKE_ESCAPE_CHAR}'")
|
|
221
|
+
params.append(f"{_escape_like(f.value)}%")
|
|
184
222
|
|
|
185
223
|
elif f.operator == Operator.ENDS_WITH:
|
|
186
|
-
clauses.append(f"{f.field} LIKE %s")
|
|
187
|
-
params.append(f"%{f.value}")
|
|
224
|
+
clauses.append(f"{f.field} LIKE %s ESCAPE '{LIKE_ESCAPE_CHAR}'")
|
|
225
|
+
params.append(f"%{_escape_like(f.value)}")
|
|
188
226
|
|
|
189
227
|
return " AND ".join(clauses), params
|
|
190
228
|
|
|
@@ -197,6 +235,7 @@ class QueryBuilder:
|
|
|
197
235
|
result = {}
|
|
198
236
|
|
|
199
237
|
for f in self.filters:
|
|
238
|
+
_reject_mapping_value(f)
|
|
200
239
|
|
|
201
240
|
if f.operator == Operator.EQ:
|
|
202
241
|
result[f.field] = f.value
|
polydb/schema.py
CHANGED
|
@@ -4,10 +4,33 @@ Schema management and migrations
|
|
|
4
4
|
"""
|
|
5
5
|
from typing import Dict, List, Optional, Any
|
|
6
6
|
from dataclasses import dataclass
|
|
7
|
+
from decimal import Decimal
|
|
7
8
|
from enum import Enum
|
|
8
9
|
import json
|
|
9
10
|
from datetime import datetime
|
|
10
11
|
|
|
12
|
+
from .errors import ValidationError
|
|
13
|
+
from .utils import validate_column_name, validate_table_name
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _render_default(value: Any) -> str:
|
|
17
|
+
"""Render a column DEFAULT as a literal that cannot escape its context.
|
|
18
|
+
|
|
19
|
+
A string default used to be interpolated as ``DEFAULT '{value}'``, so a
|
|
20
|
+
single quote in it closed the literal and the rest of the value was parsed
|
|
21
|
+
as SQL - the whole statement is DDL, which cannot be parameterised, so the
|
|
22
|
+
literal has to be made safe here. Quotes are doubled (the SQL-standard
|
|
23
|
+
escape); numbers and booleans keep rendering bare as before.
|
|
24
|
+
"""
|
|
25
|
+
if isinstance(value, bool) or isinstance(value, (int, float, Decimal)):
|
|
26
|
+
return f"DEFAULT {value}"
|
|
27
|
+
|
|
28
|
+
text = str(value)
|
|
29
|
+
if "\x00" in text:
|
|
30
|
+
raise ValidationError("Invalid column default: NUL characters are not allowed")
|
|
31
|
+
escaped = text.replace("'", "''")
|
|
32
|
+
return f"DEFAULT '{escaped}'"
|
|
33
|
+
|
|
11
34
|
|
|
12
35
|
class ColumnType(Enum):
|
|
13
36
|
INTEGER = "INTEGER"
|
|
@@ -61,10 +84,11 @@ class SchemaBuilder:
|
|
|
61
84
|
|
|
62
85
|
def to_create_table(self, table_name: str) -> str:
|
|
63
86
|
"""Generate CREATE TABLE statement"""
|
|
87
|
+
validate_table_name(table_name)
|
|
64
88
|
col_defs = []
|
|
65
|
-
|
|
89
|
+
|
|
66
90
|
for col in self.columns:
|
|
67
|
-
parts = [col.name]
|
|
91
|
+
parts = [validate_column_name(col.name)]
|
|
68
92
|
|
|
69
93
|
# Type
|
|
70
94
|
if col.type == ColumnType.VARCHAR and col.max_length:
|
|
@@ -78,10 +102,7 @@ class SchemaBuilder:
|
|
|
78
102
|
|
|
79
103
|
# Default
|
|
80
104
|
if col.default is not None:
|
|
81
|
-
|
|
82
|
-
parts.append(f"DEFAULT '{col.default}'")
|
|
83
|
-
else:
|
|
84
|
-
parts.append(f"DEFAULT {col.default}")
|
|
105
|
+
parts.append(_render_default(col.default))
|
|
85
106
|
|
|
86
107
|
# Unique
|
|
87
108
|
if col.unique:
|
|
@@ -91,7 +112,8 @@ class SchemaBuilder:
|
|
|
91
112
|
|
|
92
113
|
# Primary key
|
|
93
114
|
if self.primary_keys:
|
|
94
|
-
|
|
115
|
+
pk_cols = [validate_column_name(c) for c in self.primary_keys]
|
|
116
|
+
col_defs.append(f"PRIMARY KEY ({', '.join(pk_cols)})")
|
|
95
117
|
|
|
96
118
|
sql = f"CREATE TABLE IF NOT EXISTS {table_name} (\n"
|
|
97
119
|
sql += ",\n".join(f" {col}" for col in col_defs)
|
|
@@ -103,10 +125,15 @@ class SchemaBuilder:
|
|
|
103
125
|
"""Generate CREATE INDEX statements"""
|
|
104
126
|
statements = []
|
|
105
127
|
|
|
128
|
+
validate_table_name(table_name)
|
|
129
|
+
|
|
106
130
|
for idx in self.indexes:
|
|
107
131
|
unique = "UNIQUE " if idx.unique else ""
|
|
108
|
-
cols = ", ".join(idx.columns)
|
|
109
|
-
sql =
|
|
132
|
+
cols = ", ".join(validate_column_name(c) for c in idx.columns)
|
|
133
|
+
sql = (
|
|
134
|
+
f"CREATE {unique}INDEX IF NOT EXISTS {validate_table_name(idx.name)} "
|
|
135
|
+
f"ON {table_name}({cols});"
|
|
136
|
+
)
|
|
110
137
|
statements.append(sql)
|
|
111
138
|
|
|
112
139
|
return statements
|
polydb/security.py
CHANGED
|
@@ -13,9 +13,12 @@ from functools import wraps
|
|
|
13
13
|
import logging
|
|
14
14
|
|
|
15
15
|
from .json_safe import json_safe
|
|
16
|
+
from .errors import EncryptionConfigError
|
|
16
17
|
|
|
17
18
|
logger = logging.getLogger(__name__)
|
|
18
19
|
|
|
20
|
+
DEFAULT_KEY_ID = "v1"
|
|
21
|
+
|
|
19
22
|
|
|
20
23
|
@dataclass
|
|
21
24
|
class EncryptionConfig:
|
|
@@ -27,60 +30,138 @@ class EncryptionConfig:
|
|
|
27
30
|
|
|
28
31
|
|
|
29
32
|
class FieldEncryption:
|
|
30
|
-
"""Field-level encryption for sensitive data
|
|
33
|
+
"""Field-level encryption for sensitive data (AES-256-GCM).
|
|
34
|
+
|
|
35
|
+
Keys are loaded from the environment - never generated in memory - so a
|
|
36
|
+
process restart can never silently orphan previously-encrypted data:
|
|
37
|
+
|
|
38
|
+
- ``POLYDB_ENCRYPTION_KEY``: base64-encoded 32-byte key used to encrypt
|
|
39
|
+
new values. Its key id is ``POLYDB_ENCRYPTION_KEY_ID`` (default "v1").
|
|
40
|
+
- ``POLYDB_ENCRYPTION_KEYS``: optional JSON object ``{key_id: base64key}``
|
|
41
|
+
of retired keys kept around only to decrypt values written before a
|
|
42
|
+
rotation; not used for new encryption.
|
|
43
|
+
|
|
44
|
+
Constructing this class raises :class:`EncryptionConfigError` if no
|
|
45
|
+
active key is configured - callers only instantiate it when encryption
|
|
46
|
+
is explicitly enabled, so failing fast here means a misconfigured
|
|
47
|
+
deployment refuses to start rather than persisting plaintext.
|
|
48
|
+
"""
|
|
31
49
|
|
|
32
50
|
def __init__(self, encryption_key: Optional[bytes] = None):
|
|
33
|
-
|
|
51
|
+
if encryption_key is not None:
|
|
52
|
+
self._keys: Dict[str, bytes] = {DEFAULT_KEY_ID: encryption_key}
|
|
53
|
+
self._active_key_id = DEFAULT_KEY_ID
|
|
54
|
+
else:
|
|
55
|
+
self._keys = self._load_keys()
|
|
56
|
+
self._active_key_id = os.getenv("POLYDB_ENCRYPTION_KEY_ID", DEFAULT_KEY_ID).strip()
|
|
57
|
+
if self._active_key_id not in self._keys:
|
|
58
|
+
raise EncryptionConfigError(
|
|
59
|
+
f"POLYDB_ENCRYPTION_KEY_ID={self._active_key_id!r} has no matching key. "
|
|
60
|
+
"Set POLYDB_ENCRYPTION_KEY (for the active key id) or include it in "
|
|
61
|
+
"POLYDB_ENCRYPTION_KEYS."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def encryption_key(self) -> bytes:
|
|
66
|
+
"""The active key's raw bytes (kept for backward-compatible access)."""
|
|
67
|
+
return self._keys[self._active_key_id]
|
|
34
68
|
|
|
35
69
|
@staticmethod
|
|
36
|
-
def
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
logger.warning(
|
|
46
|
-
"Generated new encryption key. Store it securely in POLYDB_ENCRYPTION_KEY env var."
|
|
47
|
-
)
|
|
70
|
+
def _decode_key(raw: str, source: str) -> bytes:
|
|
71
|
+
try:
|
|
72
|
+
key = base64.b64decode(raw, validate=True)
|
|
73
|
+
except Exception as exc:
|
|
74
|
+
raise EncryptionConfigError(f"{source} is not valid base64: {exc}") from exc
|
|
75
|
+
if len(key) != 32:
|
|
76
|
+
raise EncryptionConfigError(
|
|
77
|
+
f"{source} decodes to {len(key)} bytes; AES-256-GCM requires exactly 32 bytes."
|
|
78
|
+
)
|
|
48
79
|
return key
|
|
49
80
|
|
|
81
|
+
@classmethod
|
|
82
|
+
def _load_keys(cls) -> Dict[str, bytes]:
|
|
83
|
+
"""Load the active key plus any retired keys from the environment."""
|
|
84
|
+
keys: Dict[str, bytes] = {}
|
|
85
|
+
|
|
86
|
+
retired_raw = os.getenv("POLYDB_ENCRYPTION_KEYS")
|
|
87
|
+
if retired_raw:
|
|
88
|
+
try:
|
|
89
|
+
retired = json.loads(retired_raw)
|
|
90
|
+
except json.JSONDecodeError as exc:
|
|
91
|
+
raise EncryptionConfigError(
|
|
92
|
+
f"POLYDB_ENCRYPTION_KEYS is not valid JSON: {exc}"
|
|
93
|
+
) from exc
|
|
94
|
+
for key_id, raw in retired.items():
|
|
95
|
+
keys[key_id] = cls._decode_key(raw, f"POLYDB_ENCRYPTION_KEYS[{key_id!r}]")
|
|
96
|
+
|
|
97
|
+
active_raw = os.getenv("POLYDB_ENCRYPTION_KEY")
|
|
98
|
+
if not active_raw:
|
|
99
|
+
raise EncryptionConfigError(
|
|
100
|
+
"Field encryption is enabled but POLYDB_ENCRYPTION_KEY is not set. "
|
|
101
|
+
"Generate one with: python -c \"import os,base64; "
|
|
102
|
+
"print(base64.b64encode(os.urandom(32)).decode())\" and store it securely "
|
|
103
|
+
"(e.g. Azure Key Vault / AWS Secrets Manager / GCP Secret Manager) - do not "
|
|
104
|
+
"commit it or let it be generated on the fly."
|
|
105
|
+
)
|
|
106
|
+
active_key_id = os.getenv("POLYDB_ENCRYPTION_KEY_ID", DEFAULT_KEY_ID).strip()
|
|
107
|
+
keys[active_key_id] = cls._decode_key(active_raw, "POLYDB_ENCRYPTION_KEY")
|
|
108
|
+
return keys
|
|
109
|
+
|
|
50
110
|
def _encrypt_value(self, value: Any) -> str:
|
|
51
111
|
"""Encrypt arbitrary value (serialize if non-str)"""
|
|
52
112
|
if value is None:
|
|
53
113
|
return ""
|
|
54
114
|
data = json.dumps(value, default=json_safe) if not isinstance(value, str) else value
|
|
115
|
+
|
|
55
116
|
try:
|
|
56
117
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
118
|
+
except ImportError as exc:
|
|
119
|
+
raise ImportError(
|
|
120
|
+
"cryptography not installed. Install with: pip install 'altcodepro-polydb-python[security]'"
|
|
121
|
+
) from exc
|
|
57
122
|
|
|
58
|
-
|
|
59
|
-
|
|
123
|
+
aesgcm = AESGCM(self._keys[self._active_key_id])
|
|
124
|
+
nonce = os.urandom(12)
|
|
60
125
|
|
|
61
|
-
|
|
126
|
+
ciphertext = aesgcm.encrypt(nonce, data.encode("utf-8"), None)
|
|
62
127
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
raise ImportError("cryptography not installed. Install with: pip install cryptography")
|
|
128
|
+
# Combine nonce and ciphertext; tag the payload with the key id used so a
|
|
129
|
+
# future key rotation can still decrypt values written under an older key.
|
|
130
|
+
encrypted = base64.b64encode(nonce + ciphertext).decode("utf-8")
|
|
131
|
+
return f"encrypted:{self._active_key_id}:{encrypted}"
|
|
68
132
|
|
|
69
133
|
def _decrypt_value(self, encrypted_data: Any) -> Any:
|
|
70
134
|
"""Decrypt field data (deserialize if needed)"""
|
|
71
135
|
if not isinstance(encrypted_data, str) or not encrypted_data.startswith("encrypted:"):
|
|
72
136
|
return encrypted_data
|
|
73
137
|
|
|
74
|
-
|
|
75
|
-
|
|
138
|
+
payload = encrypted_data[len("encrypted:"):]
|
|
139
|
+
# Legacy format (pre key-versioning) has no key id segment - decrypt
|
|
140
|
+
# with the active key for backward compatibility with existing data.
|
|
141
|
+
key_id, _, b64_blob = payload.partition(":")
|
|
142
|
+
if not b64_blob:
|
|
143
|
+
key_id, b64_blob = self._active_key_id, key_id
|
|
76
144
|
|
|
77
|
-
|
|
78
|
-
|
|
145
|
+
key = self._keys.get(key_id)
|
|
146
|
+
if key is None:
|
|
147
|
+
raise EncryptionConfigError(
|
|
148
|
+
f"No encryption key available for key id {key_id!r}. Add it to "
|
|
149
|
+
"POLYDB_ENCRYPTION_KEYS to decrypt values written before a key rotation."
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
try:
|
|
154
|
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
155
|
+
except ImportError as exc:
|
|
156
|
+
raise ImportError(
|
|
157
|
+
"cryptography not installed. Install with: pip install 'altcodepro-polydb-python[security]'"
|
|
158
|
+
) from exc
|
|
79
159
|
|
|
160
|
+
combined = base64.b64decode(b64_blob)
|
|
80
161
|
nonce = combined[:12]
|
|
81
162
|
ciphertext = combined[12:]
|
|
82
163
|
|
|
83
|
-
aesgcm = AESGCM(
|
|
164
|
+
aesgcm = AESGCM(key)
|
|
84
165
|
plaintext_bytes = aesgcm.decrypt(nonce, ciphertext, None)
|
|
85
166
|
plaintext = plaintext_bytes.decode("utf-8")
|
|
86
167
|
|
|
@@ -89,8 +170,6 @@ class FieldEncryption:
|
|
|
89
170
|
return json.loads(plaintext)
|
|
90
171
|
except json.JSONDecodeError:
|
|
91
172
|
return plaintext
|
|
92
|
-
except ImportError:
|
|
93
|
-
raise ImportError("cryptography not installed")
|
|
94
173
|
except Exception as e:
|
|
95
174
|
# Fail loud. Returning ciphertext as if it were plaintext masks
|
|
96
175
|
# key-rotation errors / corruption and leaks the 'encrypted:' blob
|
|
File without changes
|
{altcodepro_polydb_python-2.3.31.dist-info → altcodepro_polydb_python-2.5.2.dist-info}/top_level.txt
RENAMED
|
File without changes
|