altcodepro-polydb-python 2.3.20__py3-none-any.whl → 2.3.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altcodepro_polydb_python-2.3.20.dist-info → altcodepro_polydb_python-2.3.22.dist-info}/METADATA +1 -1
- {altcodepro_polydb_python-2.3.20.dist-info → altcodepro_polydb_python-2.3.22.dist-info}/RECORD +13 -8
- polydb/__init__.py +2 -0
- polydb/adapters/PostgreSQLAdapter.py +300 -191
- polydb/errors.py +6 -0
- polydb/observability/__init__.py +3 -0
- polydb/observability/logging.py +124 -0
- polydb/services/__init__.py +9 -0
- polydb/services/compliance_service.py +141 -0
- polydb/services/security_service.py +133 -0
- {altcodepro_polydb_python-2.3.20.dist-info → altcodepro_polydb_python-2.3.22.dist-info}/WHEEL +0 -0
- {altcodepro_polydb_python-2.3.20.dist-info → altcodepro_polydb_python-2.3.22.dist-info}/licenses/LICENSE +0 -0
- {altcodepro_polydb_python-2.3.20.dist-info → altcodepro_polydb_python-2.3.22.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# src/polydb/adapters/postgres.py
|
|
2
2
|
import os
|
|
3
3
|
import threading
|
|
4
|
+
import time
|
|
4
5
|
from typing import Any, Iterator, List, Optional, Tuple, Union
|
|
5
6
|
import hashlib
|
|
6
7
|
from contextlib import contextmanager
|
|
@@ -9,9 +10,10 @@ from decimal import Decimal
|
|
|
9
10
|
from datetime import datetime, date
|
|
10
11
|
|
|
11
12
|
import psycopg2.extensions
|
|
13
|
+
from psycopg2 import sql as pg_sql
|
|
12
14
|
from psycopg2.extras import Json
|
|
13
15
|
|
|
14
|
-
from ..errors import DatabaseError, ConnectionError
|
|
16
|
+
from ..errors import DatabaseError, ConnectionError, InsufficientBalanceError
|
|
15
17
|
from ..retry import retry
|
|
16
18
|
from ..utils import validate_table_name, validate_column_name
|
|
17
19
|
from ..query import QueryBuilder
|
|
@@ -25,6 +27,7 @@ class PostgreSQLAdapter:
|
|
|
25
27
|
from ..utils import setup_logger
|
|
26
28
|
|
|
27
29
|
self.logger = setup_logger(__name__)
|
|
30
|
+
self._slow_query_ms: float = float(os.getenv("POLYDB_SLOW_QUERY_MS", "1000"))
|
|
28
31
|
self.connection_string = connection_string or os.getenv(
|
|
29
32
|
"POSTGRES_CONNECTION_STRING",
|
|
30
33
|
os.getenv("POSTGRES_URL", ""),
|
|
@@ -40,45 +43,29 @@ class PostgreSQLAdapter:
|
|
|
40
43
|
# ---------------------------------------------------------------------
|
|
41
44
|
|
|
42
45
|
def _is_idle(self, conn) -> bool:
|
|
43
|
-
"""True iff the connection is not inside a (possibly aborted) transaction."""
|
|
44
46
|
try:
|
|
45
47
|
return conn.info.transaction_status == psycopg2.extensions.TRANSACTION_STATUS_IDLE
|
|
46
48
|
except Exception:
|
|
47
49
|
return False
|
|
48
50
|
|
|
49
51
|
def _drain_transaction(self, conn) -> None:
|
|
50
|
-
"""Force the connection back to IDLE so it's safe to change session
|
|
51
|
-
settings (e.g. autocommit). Safe to call when already idle."""
|
|
52
52
|
if self._is_idle(conn):
|
|
53
53
|
return
|
|
54
54
|
try:
|
|
55
55
|
conn.rollback()
|
|
56
56
|
except Exception:
|
|
57
|
-
# Last resort: caller will surface a real error on next use.
|
|
58
57
|
pass
|
|
59
58
|
|
|
60
59
|
def _ping_connection(self, conn) -> bool:
|
|
61
|
-
"""Test if connection is still alive.
|
|
62
|
-
|
|
63
|
-
psycopg2 auto-starts a transaction on the first statement after
|
|
64
|
-
commit/rollback, so we MUST rollback after the SELECT 1 — otherwise
|
|
65
|
-
the next caller inherits an active transaction and any attempt to
|
|
66
|
-
toggle autocommit raises ``set_session cannot be used inside a
|
|
67
|
-
transaction``.
|
|
68
|
-
"""
|
|
69
60
|
try:
|
|
70
61
|
with conn.cursor() as cur:
|
|
71
62
|
cur.execute("SELECT 1")
|
|
72
|
-
# Clean up the auto-started transaction so the connection
|
|
73
|
-
# leaves this method in IDLE state.
|
|
74
63
|
try:
|
|
75
64
|
conn.rollback()
|
|
76
65
|
except Exception:
|
|
77
66
|
pass
|
|
78
67
|
return True
|
|
79
68
|
except Exception:
|
|
80
|
-
# Best-effort cleanup on a failed ping. The connection is
|
|
81
|
-
# almost certainly broken; the caller will close it.
|
|
82
69
|
try:
|
|
83
70
|
conn.rollback()
|
|
84
71
|
except Exception:
|
|
@@ -90,7 +77,6 @@ class PostgreSQLAdapter:
|
|
|
90
77
|
import psycopg2.pool
|
|
91
78
|
from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
|
|
92
79
|
|
|
93
|
-
# Enhance connection string with better Azure settings
|
|
94
80
|
dsn = self.connection_string
|
|
95
81
|
if "postgresql://" in dsn:
|
|
96
82
|
parsed = urlparse(dsn)
|
|
@@ -105,17 +91,37 @@ class PostgreSQLAdapter:
|
|
|
105
91
|
parsed = parsed._replace(query=new_query)
|
|
106
92
|
dsn = urlunparse(parsed)
|
|
107
93
|
|
|
94
|
+
_maxconn = int(os.getenv("POSTGRES_MAX_CONNECTIONS", "100"))
|
|
108
95
|
with self._lock:
|
|
109
96
|
if not self._pool:
|
|
110
97
|
self._pool = psycopg2.pool.ThreadedConnectionPool(
|
|
111
98
|
minconn=int(os.getenv("POSTGRES_MIN_CONNECTIONS", "5")),
|
|
112
|
-
maxconn=
|
|
99
|
+
maxconn=_maxconn,
|
|
113
100
|
dsn=dsn,
|
|
114
101
|
)
|
|
115
|
-
self.logger.info(
|
|
102
|
+
self.logger.info(
|
|
103
|
+
"PostgreSQL pool initialized: min=%s max=%s",
|
|
104
|
+
os.getenv("POSTGRES_MIN_CONNECTIONS", "5"),
|
|
105
|
+
_maxconn,
|
|
106
|
+
)
|
|
116
107
|
except Exception as e:
|
|
117
108
|
raise ConnectionError(f"Failed to initialize PostgreSQL pool: {str(e)}")
|
|
118
109
|
|
|
110
|
+
def _log_pool_utilization(self) -> None:
|
|
111
|
+
if not self._pool:
|
|
112
|
+
return
|
|
113
|
+
try:
|
|
114
|
+
maxconn = int(os.getenv("POSTGRES_MAX_CONNECTIONS", "100"))
|
|
115
|
+
used_count = len(getattr(self._pool, "_used", {}))
|
|
116
|
+
pct = (used_count / maxconn * 100) if maxconn else 0
|
|
117
|
+
if pct > 80:
|
|
118
|
+
self.logger.warning(
|
|
119
|
+
"PostgreSQL pool utilization: %d/%d connections in use (%.0f%%)",
|
|
120
|
+
used_count, maxconn, pct,
|
|
121
|
+
)
|
|
122
|
+
except Exception:
|
|
123
|
+
pass
|
|
124
|
+
|
|
119
125
|
def _get_connection(self) -> Any:
|
|
120
126
|
if not self._pool:
|
|
121
127
|
self._initialize_pool()
|
|
@@ -123,16 +129,15 @@ class PostgreSQLAdapter:
|
|
|
123
129
|
try:
|
|
124
130
|
conn = self._pool.getconn() # type: ignore
|
|
125
131
|
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
self.logger.warning("Stale connection detected from pool, closing and retrying")
|
|
132
|
+
# Only ping if the connection is in an error state (closed/broken),
|
|
133
|
+
# not on every call. TCP keepalives handle staleness detection.
|
|
134
|
+
if conn.closed:
|
|
135
|
+
self.logger.warning("Closed connection detected from pool, closing and retrying")
|
|
131
136
|
self._pool.putconn(conn, close=True) # type: ignore
|
|
132
|
-
conn = self._pool.getconn() # type: ignore
|
|
133
|
-
# New connection might still have been used before — make sure it's idle.
|
|
137
|
+
conn = self._pool.getconn() # type: ignore
|
|
134
138
|
self._drain_transaction(conn)
|
|
135
139
|
|
|
140
|
+
self._log_pool_utilization()
|
|
136
141
|
return conn
|
|
137
142
|
|
|
138
143
|
except Exception as e:
|
|
@@ -140,20 +145,45 @@ class PostgreSQLAdapter:
|
|
|
140
145
|
raise ConnectionError(f"Could not obtain database connection: {e}") from e
|
|
141
146
|
|
|
142
147
|
def _return_connection(self, conn: Any):
|
|
143
|
-
"""Return a connection to the pool, defensively draining any
|
|
144
|
-
leftover transaction state. Belt-and-suspenders: every operation
|
|
145
|
-
in this adapter already commits/rolls-back before returning, but
|
|
146
|
-
if any path ever forgets, the pool still gets a clean connection.
|
|
147
|
-
"""
|
|
148
148
|
if self._pool and conn:
|
|
149
149
|
self._drain_transaction(conn)
|
|
150
150
|
self._pool.putconn(conn)
|
|
151
151
|
|
|
152
|
+
# ---------------------------------------------------------------------
|
|
153
|
+
# QUERY TIMING HELPER
|
|
154
|
+
# ---------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
def _timed_execute(
|
|
157
|
+
self,
|
|
158
|
+
cursor: Any,
|
|
159
|
+
sql: str,
|
|
160
|
+
params: Any,
|
|
161
|
+
*,
|
|
162
|
+
operation: str = "execute",
|
|
163
|
+
table: str = "",
|
|
164
|
+
) -> float:
|
|
165
|
+
t0 = time.perf_counter()
|
|
166
|
+
cursor.execute(sql, params)
|
|
167
|
+
duration_ms = (time.perf_counter() - t0) * 1000.0
|
|
168
|
+
|
|
169
|
+
self.logger.debug(
|
|
170
|
+
"SQL executed",
|
|
171
|
+
extra={"operation": operation, "table": table, "duration_ms": round(duration_ms, 3)},
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
if duration_ms > self._slow_query_ms:
|
|
175
|
+
self.logger.warning(
|
|
176
|
+
"Slow query detected: operation=%s table=%s duration_ms=%.1f threshold=%.1f",
|
|
177
|
+
operation, table, duration_ms, self._slow_query_ms,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
return duration_ms
|
|
181
|
+
|
|
152
182
|
# ---------------------------------------------------------------------
|
|
153
183
|
# TRANSACTIONS
|
|
154
184
|
# ---------------------------------------------------------------------
|
|
185
|
+
|
|
155
186
|
def reset_pool(self):
|
|
156
|
-
"""Reset the entire pool (call during startup or after major failures)"""
|
|
157
187
|
with self._lock:
|
|
158
188
|
if self._pool:
|
|
159
189
|
try:
|
|
@@ -164,27 +194,17 @@ class PostgreSQLAdapter:
|
|
|
164
194
|
self._initialize_pool()
|
|
165
195
|
|
|
166
196
|
def begin_transaction(self) -> Any:
|
|
167
|
-
"""Begin a transaction and return the connection handle.
|
|
168
|
-
|
|
169
|
-
Defensively drains any leftover transaction state on the pooled
|
|
170
|
-
connection before toggling autocommit. Without this, a connection
|
|
171
|
-
that's still in TRANSACTION_STATUS_INTRANS (from a previous user
|
|
172
|
-
or from the pool's connection check) causes psycopg2 to raise
|
|
173
|
-
``set_session cannot be used inside a transaction``.
|
|
174
|
-
"""
|
|
175
197
|
conn = self._get_connection()
|
|
176
198
|
self._drain_transaction(conn)
|
|
177
199
|
conn.autocommit = False
|
|
178
200
|
return conn
|
|
179
201
|
|
|
180
202
|
def commit(self, tx: Any):
|
|
181
|
-
"""Commit the transaction using the provided connection."""
|
|
182
203
|
if tx:
|
|
183
204
|
tx.commit()
|
|
184
205
|
self._return_connection(tx)
|
|
185
206
|
|
|
186
207
|
def rollback(self, tx: Any):
|
|
187
|
-
"""Rollback the transaction using the provided connection."""
|
|
188
208
|
if tx:
|
|
189
209
|
tx.rollback()
|
|
190
210
|
self._return_connection(tx)
|
|
@@ -192,12 +212,8 @@ class PostgreSQLAdapter:
|
|
|
192
212
|
# ---------------------------------------------------------------------
|
|
193
213
|
# JSON HELPERS
|
|
194
214
|
# ---------------------------------------------------------------------
|
|
215
|
+
|
|
195
216
|
def _json_safe(self, obj: Any):
|
|
196
|
-
"""
|
|
197
|
-
Ensure JSON serialization never fails. Used only for Json() wrapping.
|
|
198
|
-
Recurses into dicts, lists, AND tuples so nested datetime/Decimal
|
|
199
|
-
values are made safe at any depth.
|
|
200
|
-
"""
|
|
201
217
|
if isinstance(obj, datetime):
|
|
202
218
|
return obj.isoformat()
|
|
203
219
|
if isinstance(obj, Decimal):
|
|
@@ -211,21 +227,6 @@ class PostgreSQLAdapter:
|
|
|
211
227
|
return obj
|
|
212
228
|
|
|
213
229
|
def _serialize_value(self, v: Any) -> Any:
|
|
214
|
-
"""
|
|
215
|
-
Serialize a value being WRITTEN to a column (insert / update SET /
|
|
216
|
-
upsert data values).
|
|
217
|
-
|
|
218
|
-
This platform provisions every ARRAY and OBJECT field as a JSONB
|
|
219
|
-
column (SchemaProvisioner._FIELD_TO_SQL_TYPE never emits a native
|
|
220
|
-
text[]/int[] column), so dicts AND lists/tuples are ALWAYS
|
|
221
|
-
JSON-encoded via psycopg2's Json adapter.
|
|
222
|
-
|
|
223
|
-
Json() handles empty {} and [] correctly ('{}'::jsonb / '[]'::jsonb),
|
|
224
|
-
handles nesting, and never produces a quoted string — which is what
|
|
225
|
-
broke the earlier native-list, json.dumps, and NULL-ify-empties
|
|
226
|
-
attempts. Empties are NOT turned into NULL: an empty JSONB array/
|
|
227
|
-
object is a valid, meaningful value.
|
|
228
|
-
"""
|
|
229
230
|
if v is None:
|
|
230
231
|
return None
|
|
231
232
|
if isinstance(v, (dict, list, tuple)):
|
|
@@ -237,15 +238,6 @@ class PostgreSQLAdapter:
|
|
|
237
238
|
return v
|
|
238
239
|
|
|
239
240
|
def _serialize_param(self, v: Any) -> Any:
|
|
240
|
-
"""
|
|
241
|
-
Serialize a value used as a QUERY PARAMETER (WHERE values, IN / ANY
|
|
242
|
-
lists, LIMIT/OFFSET, raw execute() params).
|
|
243
|
-
|
|
244
|
-
Unlike _serialize_value, primitive lists/tuples are kept NATIVE so
|
|
245
|
-
that ``IN %s`` / ``= ANY(%s)`` parameters expand correctly. A dict
|
|
246
|
-
(or a list that contains dicts) is JSON-encoded so it can be compared
|
|
247
|
-
against a JSONB column.
|
|
248
|
-
"""
|
|
249
241
|
if v is None:
|
|
250
242
|
return None
|
|
251
243
|
if isinstance(v, dict):
|
|
@@ -254,7 +246,7 @@ class PostgreSQLAdapter:
|
|
|
254
246
|
seq = list(v)
|
|
255
247
|
if any(isinstance(x, dict) for x in seq):
|
|
256
248
|
return Json(self._json_safe(seq))
|
|
257
|
-
return seq
|
|
249
|
+
return seq
|
|
258
250
|
if isinstance(v, (datetime, date)):
|
|
259
251
|
return v
|
|
260
252
|
if isinstance(v, Decimal):
|
|
@@ -265,14 +257,9 @@ class PostgreSQLAdapter:
|
|
|
265
257
|
return [self._serialize_param(p) for p in params]
|
|
266
258
|
|
|
267
259
|
def _deserialize_row(self, row: JsonDict) -> JsonDict:
|
|
268
|
-
"""
|
|
269
|
-
Postgres JSON/JSONB often comes back as dict/list already depending on
|
|
270
|
-
driver config. If it comes as a string, try json.loads safely.
|
|
271
|
-
"""
|
|
272
260
|
for k, v in list(row.items()):
|
|
273
261
|
if isinstance(v, str):
|
|
274
262
|
s = v.strip()
|
|
275
|
-
# quick cheap check to avoid parsing normal strings
|
|
276
263
|
if (s.startswith("{") and s.endswith("}")) or (
|
|
277
264
|
s.startswith("[") and s.endswith("]")
|
|
278
265
|
):
|
|
@@ -300,24 +287,18 @@ class PostgreSQLAdapter:
|
|
|
300
287
|
|
|
301
288
|
try:
|
|
302
289
|
cursor = conn.cursor()
|
|
303
|
-
|
|
304
290
|
columns = ", ".join(data.keys())
|
|
305
291
|
placeholders = ", ".join(["%s"] * len(data))
|
|
306
292
|
query = f"INSERT INTO {table} ({columns}) VALUES ({placeholders}) RETURNING *"
|
|
307
|
-
|
|
308
293
|
values = [self._serialize_value(v) for v in data.values()]
|
|
309
|
-
|
|
310
|
-
|
|
294
|
+
self._timed_execute(cursor, query, values, operation="insert", table=table)
|
|
311
295
|
result_row = cursor.fetchone()
|
|
312
296
|
columns_list = [desc[0] for desc in cursor.description]
|
|
313
297
|
result = dict(zip(columns_list, result_row))
|
|
314
|
-
|
|
315
298
|
if own_conn:
|
|
316
299
|
conn.commit()
|
|
317
|
-
|
|
318
300
|
cursor.close()
|
|
319
301
|
return self._deserialize_row(result)
|
|
320
|
-
|
|
321
302
|
except Exception as e:
|
|
322
303
|
if own_conn:
|
|
323
304
|
conn.rollback()
|
|
@@ -348,7 +329,6 @@ class PostgreSQLAdapter:
|
|
|
348
329
|
|
|
349
330
|
try:
|
|
350
331
|
cursor = conn.cursor()
|
|
351
|
-
|
|
352
332
|
sql = f"SELECT * FROM {table}"
|
|
353
333
|
params: List[Any] = []
|
|
354
334
|
|
|
@@ -356,8 +336,6 @@ class PostgreSQLAdapter:
|
|
|
356
336
|
where_parts: List[str] = []
|
|
357
337
|
for k, v in query.items():
|
|
358
338
|
validate_column_name(k)
|
|
359
|
-
|
|
360
|
-
# IMPORTANT: None must be "IS NULL" not "= %s"
|
|
361
339
|
if v is None:
|
|
362
340
|
where_parts.append(f"{k} IS NULL")
|
|
363
341
|
elif isinstance(v, (list, tuple)):
|
|
@@ -367,7 +345,6 @@ class PostgreSQLAdapter:
|
|
|
367
345
|
else:
|
|
368
346
|
where_parts.append(f"{k} = %s")
|
|
369
347
|
params.append(v)
|
|
370
|
-
|
|
371
348
|
if where_parts:
|
|
372
349
|
sql += " WHERE " + " AND ".join(where_parts)
|
|
373
350
|
|
|
@@ -378,14 +355,14 @@ class PostgreSQLAdapter:
|
|
|
378
355
|
sql += " OFFSET %s"
|
|
379
356
|
params.append(offset)
|
|
380
357
|
|
|
381
|
-
|
|
358
|
+
self._timed_execute(
|
|
359
|
+
cursor, sql, self._serialize_params(params), operation="select", table=table
|
|
360
|
+
)
|
|
382
361
|
columns = [desc[0] for desc in cursor.description]
|
|
383
362
|
results = [self._deserialize_row(dict(zip(columns, row))) for row in cursor.fetchall()]
|
|
384
363
|
cursor.close()
|
|
385
|
-
|
|
386
364
|
if own_conn:
|
|
387
365
|
conn.commit()
|
|
388
|
-
|
|
389
366
|
return results
|
|
390
367
|
except Exception as e:
|
|
391
368
|
if own_conn:
|
|
@@ -413,11 +390,9 @@ class PostgreSQLAdapter:
|
|
|
413
390
|
) -> Tuple[List[JsonDict], Optional[str]]:
|
|
414
391
|
offset = int(continuation_token) if continuation_token else 0
|
|
415
392
|
results = self.select(table, query, limit=page_size + 1, offset=offset, tx=tx)
|
|
416
|
-
|
|
417
393
|
has_more = len(results) > page_size
|
|
418
394
|
if has_more:
|
|
419
395
|
results = results[:page_size]
|
|
420
|
-
|
|
421
396
|
next_token = str(offset + page_size) if has_more else None
|
|
422
397
|
return results, next_token
|
|
423
398
|
|
|
@@ -445,9 +420,7 @@ class PostgreSQLAdapter:
|
|
|
445
420
|
|
|
446
421
|
try:
|
|
447
422
|
cursor = conn.cursor()
|
|
448
|
-
|
|
449
423
|
set_clause = ", ".join([f"{k} = %s" for k in data.keys()])
|
|
450
|
-
# SET values are written into columns -> write serializer (JSONB).
|
|
451
424
|
params: List[Any] = [self._serialize_value(v) for v in data.values()]
|
|
452
425
|
|
|
453
426
|
if isinstance(entity_id, dict):
|
|
@@ -458,7 +431,6 @@ class PostgreSQLAdapter:
|
|
|
458
431
|
where_parts.append(f"{k} IS NULL")
|
|
459
432
|
else:
|
|
460
433
|
where_parts.append(f"{k} = %s")
|
|
461
|
-
# WHERE values are query params -> param serializer.
|
|
462
434
|
params.append(self._serialize_param(v))
|
|
463
435
|
where_clause = " AND ".join(where_parts)
|
|
464
436
|
else:
|
|
@@ -466,18 +438,14 @@ class PostgreSQLAdapter:
|
|
|
466
438
|
params.append(entity_id)
|
|
467
439
|
|
|
468
440
|
query = f"UPDATE {table} SET {set_clause} WHERE {where_clause} RETURNING *"
|
|
469
|
-
|
|
470
|
-
|
|
441
|
+
self._timed_execute(cursor, query, params, operation="update", table=table)
|
|
471
442
|
result_row = cursor.fetchone()
|
|
472
443
|
if not result_row:
|
|
473
444
|
raise DatabaseError("No rows updated")
|
|
474
|
-
|
|
475
445
|
columns = [desc[0] for desc in cursor.description]
|
|
476
446
|
result = dict(zip(columns, result_row))
|
|
477
|
-
|
|
478
447
|
if own_conn:
|
|
479
448
|
conn.commit()
|
|
480
|
-
|
|
481
449
|
cursor.close()
|
|
482
450
|
return self._deserialize_row(result)
|
|
483
451
|
except Exception as e:
|
|
@@ -506,10 +474,8 @@ class PostgreSQLAdapter:
|
|
|
506
474
|
|
|
507
475
|
try:
|
|
508
476
|
cursor = conn.cursor()
|
|
509
|
-
|
|
510
477
|
columns = ", ".join(data.keys())
|
|
511
478
|
placeholders = ", ".join(["%s"] * len(data))
|
|
512
|
-
|
|
513
479
|
conflict_columns = ["id"] if "id" in data else list(data.keys())[:1]
|
|
514
480
|
update_fields = [k for k in data.keys() if k not in conflict_columns]
|
|
515
481
|
|
|
@@ -526,14 +492,10 @@ class PostgreSQLAdapter:
|
|
|
526
492
|
{on_conflict}
|
|
527
493
|
RETURNING *
|
|
528
494
|
"""
|
|
529
|
-
|
|
530
495
|
values = [self._serialize_value(v) for v in data.values()]
|
|
531
|
-
|
|
532
|
-
|
|
496
|
+
self._timed_execute(cursor, query, values, operation="upsert", table=table)
|
|
533
497
|
result_row = cursor.fetchone()
|
|
534
498
|
if not result_row:
|
|
535
|
-
# DO NOTHING case: fetch existing row (best effort)
|
|
536
|
-
# If conflict is on id, we can read it back.
|
|
537
499
|
if "id" in conflict_columns and "id" in data:
|
|
538
500
|
cursor.execute(f"SELECT * FROM {table} WHERE id = %s", [data["id"]])
|
|
539
501
|
result_row = cursor.fetchone()
|
|
@@ -544,10 +506,8 @@ class PostgreSQLAdapter:
|
|
|
544
506
|
|
|
545
507
|
columns_list = [desc[0] for desc in cursor.description]
|
|
546
508
|
result = dict(zip(columns_list, result_row))
|
|
547
|
-
|
|
548
509
|
if own_conn:
|
|
549
510
|
conn.commit()
|
|
550
|
-
|
|
551
511
|
cursor.close()
|
|
552
512
|
return self._deserialize_row(result)
|
|
553
513
|
except Exception as e:
|
|
@@ -575,7 +535,6 @@ class PostgreSQLAdapter:
|
|
|
575
535
|
|
|
576
536
|
try:
|
|
577
537
|
cursor = conn.cursor()
|
|
578
|
-
|
|
579
538
|
params: List[Any] = []
|
|
580
539
|
if isinstance(entity_id, dict):
|
|
581
540
|
where_parts: List[str] = []
|
|
@@ -585,7 +544,6 @@ class PostgreSQLAdapter:
|
|
|
585
544
|
where_parts.append(f"{k} IS NULL")
|
|
586
545
|
else:
|
|
587
546
|
where_parts.append(f"{k} = %s")
|
|
588
|
-
# WHERE values are query params -> param serializer.
|
|
589
547
|
params.append(self._serialize_param(v))
|
|
590
548
|
where_clause = " AND ".join(where_parts)
|
|
591
549
|
else:
|
|
@@ -593,18 +551,14 @@ class PostgreSQLAdapter:
|
|
|
593
551
|
params.append(entity_id)
|
|
594
552
|
|
|
595
553
|
query = f"DELETE FROM {table} WHERE {where_clause} RETURNING *"
|
|
596
|
-
|
|
554
|
+
self._timed_execute(cursor, query, params, operation="delete", table=table)
|
|
597
555
|
result_row = cursor.fetchone()
|
|
598
|
-
|
|
599
556
|
if not result_row:
|
|
600
557
|
raise DatabaseError("No rows deleted")
|
|
601
|
-
|
|
602
558
|
columns = [desc[0] for desc in cursor.description]
|
|
603
559
|
result = dict(zip(columns, result_row))
|
|
604
|
-
|
|
605
560
|
if own_conn:
|
|
606
561
|
conn.commit()
|
|
607
|
-
|
|
608
562
|
cursor.close()
|
|
609
563
|
return self._deserialize_row(result)
|
|
610
564
|
except Exception as e:
|
|
@@ -623,9 +577,7 @@ class PostgreSQLAdapter:
|
|
|
623
577
|
def query_linq(
|
|
624
578
|
self, table: str, builder: QueryBuilder, tx: Optional[Any] = None
|
|
625
579
|
) -> Union[List[JsonDict], int]:
|
|
626
|
-
|
|
627
580
|
table = validate_table_name(table)
|
|
628
|
-
|
|
629
581
|
conn = tx
|
|
630
582
|
own_conn = False
|
|
631
583
|
if not conn:
|
|
@@ -634,117 +586,67 @@ class PostgreSQLAdapter:
|
|
|
634
586
|
|
|
635
587
|
try:
|
|
636
588
|
cursor = conn.cursor()
|
|
637
|
-
|
|
638
|
-
# ------------------------------------------------
|
|
639
|
-
# SELECT clause
|
|
640
|
-
# ------------------------------------------------
|
|
641
|
-
|
|
642
589
|
if builder.count_only:
|
|
643
590
|
sql = f"SELECT COUNT(*) FROM {table}"
|
|
644
|
-
|
|
645
591
|
elif builder.selected_fields:
|
|
646
592
|
for f in builder.selected_fields:
|
|
647
593
|
validate_column_name(f)
|
|
648
|
-
|
|
649
594
|
fields = ", ".join(builder.selected_fields)
|
|
650
|
-
|
|
651
595
|
if builder.distinct_flag:
|
|
652
596
|
sql = f"SELECT DISTINCT {fields} FROM {table}"
|
|
653
597
|
else:
|
|
654
598
|
sql = f"SELECT {fields} FROM {table}"
|
|
655
|
-
|
|
656
599
|
else:
|
|
657
600
|
sql = f"SELECT * FROM {table}"
|
|
658
601
|
|
|
659
602
|
params: List[Any] = []
|
|
660
|
-
|
|
661
|
-
# ------------------------------------------------
|
|
662
|
-
# WHERE
|
|
663
|
-
# ------------------------------------------------
|
|
664
|
-
|
|
665
603
|
where_clause, where_params = builder.to_sql_where()
|
|
666
|
-
|
|
667
604
|
if where_clause:
|
|
668
605
|
sql += f" WHERE {where_clause}"
|
|
669
606
|
params.extend(where_params)
|
|
670
607
|
|
|
671
|
-
# ------------------------------------------------
|
|
672
|
-
# GROUP BY
|
|
673
|
-
# ------------------------------------------------
|
|
674
|
-
|
|
675
608
|
if builder.group_by_fields:
|
|
676
|
-
|
|
677
609
|
for f in builder.group_by_fields:
|
|
678
610
|
validate_column_name(f)
|
|
679
|
-
|
|
680
611
|
sql += f" GROUP BY {', '.join(builder.group_by_fields)}"
|
|
681
612
|
|
|
682
|
-
# ------------------------------------------------
|
|
683
|
-
# ORDER BY
|
|
684
|
-
# ------------------------------------------------
|
|
685
|
-
|
|
686
613
|
if builder.order_by_fields:
|
|
687
|
-
|
|
688
614
|
order_parts = []
|
|
689
|
-
|
|
690
615
|
for field, desc in builder.order_by_fields:
|
|
691
|
-
|
|
692
616
|
validate_column_name(field)
|
|
693
|
-
|
|
694
617
|
direction = "DESC" if desc else "ASC"
|
|
695
|
-
|
|
696
618
|
order_parts.append(f"{field} {direction}")
|
|
697
|
-
|
|
698
619
|
sql += f" ORDER BY {', '.join(order_parts)}"
|
|
699
620
|
|
|
700
|
-
# ------------------------------------------------
|
|
701
|
-
# LIMIT
|
|
702
|
-
# ------------------------------------------------
|
|
703
|
-
|
|
704
621
|
if builder.take_count is not None:
|
|
705
622
|
sql += " LIMIT %s"
|
|
706
623
|
params.append(builder.take_count)
|
|
707
624
|
|
|
708
|
-
# ------------------------------------------------
|
|
709
|
-
# OFFSET
|
|
710
|
-
# ------------------------------------------------
|
|
711
|
-
|
|
712
625
|
if builder.skip_count:
|
|
713
626
|
sql += " OFFSET %s"
|
|
714
627
|
params.append(builder.skip_count)
|
|
715
628
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
cursor.execute(sql, self._serialize_params(params))
|
|
629
|
+
self._timed_execute(
|
|
630
|
+
cursor, sql, self._serialize_params(params), operation="query_linq", table=table
|
|
631
|
+
)
|
|
721
632
|
|
|
722
633
|
if builder.count_only:
|
|
723
634
|
result = cursor.fetchone()[0]
|
|
724
|
-
|
|
725
635
|
else:
|
|
726
636
|
columns = [desc[0] for desc in cursor.description]
|
|
727
|
-
|
|
728
637
|
result = [
|
|
729
638
|
self._deserialize_row(dict(zip(columns, row))) for row in cursor.fetchall()
|
|
730
639
|
]
|
|
731
640
|
|
|
732
641
|
cursor.close()
|
|
733
|
-
|
|
734
642
|
if own_conn:
|
|
735
643
|
conn.commit()
|
|
736
|
-
|
|
737
644
|
return result
|
|
738
|
-
|
|
739
645
|
except Exception as e:
|
|
740
|
-
|
|
741
646
|
if own_conn:
|
|
742
647
|
conn.rollback()
|
|
743
|
-
|
|
744
648
|
raise DatabaseError(f"LINQ query failed: {str(e)}")
|
|
745
|
-
|
|
746
649
|
finally:
|
|
747
|
-
|
|
748
650
|
if own_conn and conn:
|
|
749
651
|
self._return_connection(conn)
|
|
750
652
|
|
|
@@ -771,10 +673,10 @@ class PostgreSQLAdapter:
|
|
|
771
673
|
cursor = None
|
|
772
674
|
try:
|
|
773
675
|
cursor = conn.cursor()
|
|
774
|
-
|
|
775
|
-
|
|
676
|
+
# Log operation shape only, never parameter values
|
|
677
|
+
self.logger.debug("Executing raw SQL (%d params)", len(params or []))
|
|
776
678
|
exec_params = self._serialize_params(params or [])
|
|
777
|
-
|
|
679
|
+
self._timed_execute(cursor, sql, exec_params, operation="execute", table="")
|
|
778
680
|
|
|
779
681
|
if fetch_one:
|
|
780
682
|
row = cursor.fetchone()
|
|
@@ -797,7 +699,6 @@ class PostgreSQLAdapter:
|
|
|
797
699
|
if own_conn:
|
|
798
700
|
conn.commit()
|
|
799
701
|
return None
|
|
800
|
-
|
|
801
702
|
except Exception as e:
|
|
802
703
|
if own_conn:
|
|
803
704
|
try:
|
|
@@ -805,7 +706,6 @@ class PostgreSQLAdapter:
|
|
|
805
706
|
except Exception:
|
|
806
707
|
pass
|
|
807
708
|
raise DatabaseError(f"Execute failed: {str(e)}")
|
|
808
|
-
|
|
809
709
|
finally:
|
|
810
710
|
if cursor:
|
|
811
711
|
try:
|
|
@@ -815,17 +715,234 @@ class PostgreSQLAdapter:
|
|
|
815
715
|
if own_conn and conn:
|
|
816
716
|
self._return_connection(conn)
|
|
817
717
|
|
|
718
|
+
# ---------------------------------------------------------------------
|
|
719
|
+
# ATOMIC OPERATIONS
|
|
720
|
+
# ---------------------------------------------------------------------
|
|
721
|
+
|
|
722
|
+
def atomic_decrement_if_sufficient(
|
|
723
|
+
self,
|
|
724
|
+
table: str,
|
|
725
|
+
balance_field: str,
|
|
726
|
+
amount: Any,
|
|
727
|
+
where_field: str,
|
|
728
|
+
where_value: Any,
|
|
729
|
+
*,
|
|
730
|
+
tx: Optional[Any] = None,
|
|
731
|
+
) -> JsonDict:
|
|
732
|
+
table = validate_table_name(table)
|
|
733
|
+
validate_column_name(balance_field)
|
|
734
|
+
validate_column_name(where_field)
|
|
735
|
+
|
|
736
|
+
conn = tx
|
|
737
|
+
own_conn = False
|
|
738
|
+
if not conn:
|
|
739
|
+
conn = self._get_connection()
|
|
740
|
+
own_conn = True
|
|
741
|
+
|
|
742
|
+
try:
|
|
743
|
+
cursor = conn.cursor()
|
|
744
|
+
sql = (
|
|
745
|
+
f"UPDATE {table} "
|
|
746
|
+
f"SET {balance_field} = {balance_field} - %s "
|
|
747
|
+
f"WHERE {where_field} = %s AND {balance_field} >= %s "
|
|
748
|
+
f"RETURNING *"
|
|
749
|
+
)
|
|
750
|
+
cursor.execute(sql, [amount, where_value, amount])
|
|
751
|
+
result_row = cursor.fetchone()
|
|
752
|
+
|
|
753
|
+
if not result_row:
|
|
754
|
+
if own_conn:
|
|
755
|
+
conn.rollback()
|
|
756
|
+
raise InsufficientBalanceError(
|
|
757
|
+
f"Insufficient balance in {table}.{balance_field} for {where_field}={where_value!r}"
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
columns = [desc[0] for desc in cursor.description]
|
|
761
|
+
result = dict(zip(columns, result_row))
|
|
762
|
+
if own_conn:
|
|
763
|
+
conn.commit()
|
|
764
|
+
cursor.close()
|
|
765
|
+
return self._deserialize_row(result)
|
|
766
|
+
except InsufficientBalanceError:
|
|
767
|
+
raise
|
|
768
|
+
except Exception as e:
|
|
769
|
+
if own_conn:
|
|
770
|
+
try:
|
|
771
|
+
conn.rollback()
|
|
772
|
+
except Exception:
|
|
773
|
+
pass
|
|
774
|
+
raise DatabaseError(f"atomic_decrement_if_sufficient failed: {str(e)}")
|
|
775
|
+
finally:
|
|
776
|
+
if own_conn and conn:
|
|
777
|
+
self._return_connection(conn)
|
|
778
|
+
|
|
779
|
+
def atomic_set_if(
|
|
780
|
+
self,
|
|
781
|
+
table: str,
|
|
782
|
+
id_field: str,
|
|
783
|
+
id_value: Any,
|
|
784
|
+
field: str,
|
|
785
|
+
value: Any,
|
|
786
|
+
expected: Any,
|
|
787
|
+
*,
|
|
788
|
+
tx: Optional[Any] = None,
|
|
789
|
+
) -> Optional[JsonDict]:
|
|
790
|
+
table = validate_table_name(table)
|
|
791
|
+
validate_column_name(field)
|
|
792
|
+
validate_column_name(id_field)
|
|
793
|
+
|
|
794
|
+
conn = tx
|
|
795
|
+
own_conn = not conn
|
|
796
|
+
if own_conn:
|
|
797
|
+
conn = self._get_connection()
|
|
798
|
+
try:
|
|
799
|
+
cursor = conn.cursor()
|
|
800
|
+
sql = (
|
|
801
|
+
f"UPDATE {table} SET {field} = %s "
|
|
802
|
+
f"WHERE {id_field} = %s AND {field} = %s RETURNING *"
|
|
803
|
+
)
|
|
804
|
+
cursor.execute(sql, [value, id_value, expected])
|
|
805
|
+
row = cursor.fetchone()
|
|
806
|
+
if own_conn:
|
|
807
|
+
conn.commit()
|
|
808
|
+
cursor.close()
|
|
809
|
+
if row is None:
|
|
810
|
+
return None
|
|
811
|
+
cols = [d[0] for d in cursor.description]
|
|
812
|
+
return self._deserialize_row(dict(zip(cols, row)))
|
|
813
|
+
except Exception as e:
|
|
814
|
+
if own_conn:
|
|
815
|
+
try:
|
|
816
|
+
conn.rollback()
|
|
817
|
+
except Exception:
|
|
818
|
+
pass
|
|
819
|
+
raise DatabaseError(f"atomic_set_if failed: {e}") from e
|
|
820
|
+
finally:
|
|
821
|
+
if own_conn and conn:
|
|
822
|
+
self._return_connection(conn)
|
|
823
|
+
|
|
824
|
+
def atomic_max(
|
|
825
|
+
self,
|
|
826
|
+
table: str,
|
|
827
|
+
id_field: str,
|
|
828
|
+
id_value: Any,
|
|
829
|
+
field: str,
|
|
830
|
+
value: Any,
|
|
831
|
+
*,
|
|
832
|
+
tx: Optional[Any] = None,
|
|
833
|
+
) -> JsonDict:
|
|
834
|
+
table = validate_table_name(table)
|
|
835
|
+
validate_column_name(field)
|
|
836
|
+
validate_column_name(id_field)
|
|
837
|
+
|
|
838
|
+
conn = tx
|
|
839
|
+
own_conn = not conn
|
|
840
|
+
if own_conn:
|
|
841
|
+
conn = self._get_connection()
|
|
842
|
+
try:
|
|
843
|
+
cursor = conn.cursor()
|
|
844
|
+
cursor.execute(
|
|
845
|
+
f"UPDATE {table} SET {field} = GREATEST({field}, %s) WHERE {id_field} = %s RETURNING *",
|
|
846
|
+
[value, id_value],
|
|
847
|
+
)
|
|
848
|
+
row = cursor.fetchone()
|
|
849
|
+
if own_conn:
|
|
850
|
+
conn.commit()
|
|
851
|
+
cursor.close()
|
|
852
|
+
if row is None:
|
|
853
|
+
raise DatabaseError(f"atomic_max: row not found {id_field}={id_value!r}")
|
|
854
|
+
cols = [d[0] for d in cursor.description]
|
|
855
|
+
return self._deserialize_row(dict(zip(cols, row)))
|
|
856
|
+
except DatabaseError:
|
|
857
|
+
raise
|
|
858
|
+
except Exception as e:
|
|
859
|
+
if own_conn:
|
|
860
|
+
try:
|
|
861
|
+
conn.rollback()
|
|
862
|
+
except Exception:
|
|
863
|
+
pass
|
|
864
|
+
raise DatabaseError(f"atomic_max failed: {e}") from e
|
|
865
|
+
finally:
|
|
866
|
+
if own_conn and conn:
|
|
867
|
+
self._return_connection(conn)
|
|
868
|
+
|
|
869
|
+
def atomic_min(
|
|
870
|
+
self,
|
|
871
|
+
table: str,
|
|
872
|
+
id_field: str,
|
|
873
|
+
id_value: Any,
|
|
874
|
+
field: str,
|
|
875
|
+
value: Any,
|
|
876
|
+
*,
|
|
877
|
+
tx: Optional[Any] = None,
|
|
878
|
+
) -> JsonDict:
|
|
879
|
+
table = validate_table_name(table)
|
|
880
|
+
validate_column_name(field)
|
|
881
|
+
validate_column_name(id_field)
|
|
882
|
+
|
|
883
|
+
conn = tx
|
|
884
|
+
own_conn = not conn
|
|
885
|
+
if own_conn:
|
|
886
|
+
conn = self._get_connection()
|
|
887
|
+
try:
|
|
888
|
+
cursor = conn.cursor()
|
|
889
|
+
cursor.execute(
|
|
890
|
+
f"UPDATE {table} SET {field} = LEAST({field}, %s) WHERE {id_field} = %s RETURNING *",
|
|
891
|
+
[value, id_value],
|
|
892
|
+
)
|
|
893
|
+
row = cursor.fetchone()
|
|
894
|
+
if own_conn:
|
|
895
|
+
conn.commit()
|
|
896
|
+
cursor.close()
|
|
897
|
+
if row is None:
|
|
898
|
+
raise DatabaseError(f"atomic_min: row not found {id_field}={id_value!r}")
|
|
899
|
+
cols = [d[0] for d in cursor.description]
|
|
900
|
+
return self._deserialize_row(dict(zip(cols, row)))
|
|
901
|
+
except DatabaseError:
|
|
902
|
+
raise
|
|
903
|
+
except Exception as e:
|
|
904
|
+
if own_conn:
|
|
905
|
+
try:
|
|
906
|
+
conn.rollback()
|
|
907
|
+
except Exception:
|
|
908
|
+
pass
|
|
909
|
+
raise DatabaseError(f"atomic_min failed: {e}") from e
|
|
910
|
+
finally:
|
|
911
|
+
if own_conn and conn:
|
|
912
|
+
self._return_connection(conn)
|
|
913
|
+
|
|
914
|
+
# ---------------------------------------------------------------------
|
|
915
|
+
# SAVEPOINTS
|
|
916
|
+
# Security: use pg_sql.Identifier to prevent SQL injection via savepoint names.
|
|
917
|
+
# ---------------------------------------------------------------------
|
|
918
|
+
|
|
919
|
+
def begin_savepoint(self, name: str, tx: Any) -> None:
|
|
920
|
+
try:
|
|
921
|
+
with tx.cursor() as cur:
|
|
922
|
+
cur.execute(pg_sql.SQL("SAVEPOINT {}").format(pg_sql.Identifier(name)))
|
|
923
|
+
except Exception as e:
|
|
924
|
+
raise DatabaseError(f"begin_savepoint({name!r}) failed: {str(e)}")
|
|
925
|
+
|
|
926
|
+
def rollback_to_savepoint(self, name: str, tx: Any) -> None:
|
|
927
|
+
try:
|
|
928
|
+
with tx.cursor() as cur:
|
|
929
|
+
cur.execute(pg_sql.SQL("ROLLBACK TO SAVEPOINT {}").format(pg_sql.Identifier(name)))
|
|
930
|
+
except Exception as e:
|
|
931
|
+
raise DatabaseError(f"rollback_to_savepoint({name!r}) failed: {str(e)}")
|
|
932
|
+
|
|
933
|
+
def release_savepoint(self, name: str, tx: Any) -> None:
|
|
934
|
+
try:
|
|
935
|
+
with tx.cursor() as cur:
|
|
936
|
+
cur.execute(pg_sql.SQL("RELEASE SAVEPOINT {}").format(pg_sql.Identifier(name)))
|
|
937
|
+
except Exception as e:
|
|
938
|
+
raise DatabaseError(f"release_savepoint({name!r}) failed: {str(e)}")
|
|
939
|
+
|
|
818
940
|
# ---------------------------------------------------------------------
|
|
819
941
|
# DISTRIBUTED LOCK
|
|
820
942
|
# ---------------------------------------------------------------------
|
|
821
943
|
|
|
822
944
|
@contextmanager
|
|
823
945
|
def distributed_lock(self, lock_name: str) -> Iterator[None]:
|
|
824
|
-
"""
|
|
825
|
-
PostgreSQL advisory lock (session scoped).
|
|
826
|
-
- Always unlock before returning the pooled connection.
|
|
827
|
-
- Never wrap exceptions raised by the user block.
|
|
828
|
-
"""
|
|
829
946
|
conn = None
|
|
830
947
|
cursor = None
|
|
831
948
|
lock_id = int(hashlib.sha256(lock_name.encode()).hexdigest(), 16) % (2**63)
|
|
@@ -833,8 +950,6 @@ class PostgreSQLAdapter:
|
|
|
833
950
|
try:
|
|
834
951
|
conn = self._get_connection()
|
|
835
952
|
cursor = conn.cursor()
|
|
836
|
-
|
|
837
|
-
# Acquire lock (DB op) — if this fails, it's a DatabaseError
|
|
838
953
|
try:
|
|
839
954
|
cursor.execute("SELECT pg_advisory_lock(%s);", (lock_id,))
|
|
840
955
|
self.logger.debug("Acquired distributed lock: %s", lock_name)
|
|
@@ -842,21 +957,15 @@ class PostgreSQLAdapter:
|
|
|
842
957
|
raise DatabaseError(f"Distributed lock acquire failed: {e}") from e
|
|
843
958
|
|
|
844
959
|
try:
|
|
845
|
-
# User code runs here. If it raises, it MUST propagate unchanged.
|
|
846
960
|
yield
|
|
847
961
|
finally:
|
|
848
|
-
# Release lock (DB op). Always attempted.
|
|
849
962
|
try:
|
|
850
963
|
cursor.execute("SELECT pg_advisory_unlock(%s);", (lock_id,))
|
|
851
964
|
self.logger.debug("Released distributed lock: %s", lock_name)
|
|
852
965
|
except Exception as e:
|
|
853
|
-
# IMPORTANT: don't mask user exceptions.
|
|
854
|
-
# If unlock fails, raise DatabaseError only if user block didn't already fail.
|
|
855
|
-
# Easiest safe behavior: just log and continue.
|
|
856
966
|
self.logger.exception(
|
|
857
967
|
"Distributed lock release failed for %s: %s", lock_name, e
|
|
858
968
|
)
|
|
859
|
-
|
|
860
969
|
finally:
|
|
861
970
|
if cursor:
|
|
862
971
|
try:
|