jaaql-middleware-python 5.3.20__py3-none-any.whl → 5.3.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jaaql/constants.py +4 -1
- jaaql/db/db_interface.py +61 -35
- jaaql/db/db_pg_interface.py +183 -120
- jaaql/db/db_utils.py +10 -1
- jaaql/db/db_utils_no_circ.py +4 -1
- jaaql/exceptions/http_status_exception.py +5 -4
- jaaql/exceptions/jaaql_interpretable_handled_errors.py +37 -1
- jaaql/interpreter/interpret_jaaql.py +72 -92
- jaaql/mvc/handmade_queries.py +14 -0
- jaaql/mvc/model.py +38 -12
- jaaql/scripts/03.install_super_user.handwritten.sql +7 -0
- jaaql/test/test_call_proc.py +214 -0
- jaaql/test/test_connection_lifecycle.py +675 -0
- jaaql/test/test_execute.py +61 -0
- jaaql/test/test_requested_database.py +130 -0
- {jaaql_middleware_python-5.3.20.dist-info → jaaql_middleware_python-5.3.22.dist-info}/METADATA +1 -1
- {jaaql_middleware_python-5.3.20.dist-info → jaaql_middleware_python-5.3.22.dist-info}/RECORD +20 -16
- {jaaql_middleware_python-5.3.20.dist-info → jaaql_middleware_python-5.3.22.dist-info}/WHEEL +1 -1
- {jaaql_middleware_python-5.3.20.dist-info → jaaql_middleware_python-5.3.22.dist-info}/licenses/LICENSE.txt +0 -0
- {jaaql_middleware_python-5.3.20.dist-info → jaaql_middleware_python-5.3.22.dist-info}/top_level.txt +0 -0
jaaql/constants.py
CHANGED
|
@@ -63,6 +63,9 @@ CRON_dayOfWeek = "dayOfWeek"
|
|
|
63
63
|
|
|
64
64
|
REGEX__dmbs_object_name = r'^[0-9a-zA-Z_]{1,63}$'
|
|
65
65
|
REGEX__dmbs_procedure_name = r'^[0-9a-zA-Z_$-.\+]{1,63}$'
|
|
66
|
+
# An explicit type is spliced into the SQL after '::', so it must be a type name as FIESTA writes one and nothing more (use
|
|
67
|
+
# fullmatch): a realm name, or a Postgres datatype with an optional modifier, such as integer, numeric(4,2) or character varying(64)
|
|
68
|
+
REGEX__dmbs_type_name = r'(?:[A-Za-z_][A-Za-z0-9_]{0,62}|character varying)(?:\(\d{1,8}(?:,\d{1,8})?\))?'
|
|
66
69
|
|
|
67
70
|
# The optional request key KEY__timelines ({"<timeline>": "<ISO-8601 moment>" | null}) sets, for the request's transaction
|
|
68
71
|
# only, the moment each timeline is viewed at. A moment becomes the transaction-local setting GUC__timeline_prefix + name,
|
|
@@ -236,5 +239,5 @@ ROLE__dba = "dba"
|
|
|
236
239
|
|
|
237
240
|
PROTOCOL__postgres = "postgresql://"
|
|
238
241
|
|
|
239
|
-
VERSION = "5.3.
|
|
242
|
+
VERSION = "5.3.22"
|
|
240
243
|
|
jaaql/db/db_interface.py
CHANGED
|
@@ -68,9 +68,19 @@ class DBInterface(ABC):
|
|
|
68
68
|
def get_conn(self):
|
|
69
69
|
pass
|
|
70
70
|
|
|
71
|
-
def
|
|
72
|
-
# Overridden by the concrete interface. True when
|
|
73
|
-
#
|
|
71
|
+
def is_connection_closed(self, conn) -> bool:
|
|
72
|
+
# Overridden by the concrete interface. True when the connection/backend went away, taking any
|
|
73
|
+
# uncommitted transaction with it (so that transaction persisted nothing)
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
def statement_may_end_transaction(self, query: str) -> bool:
|
|
77
|
+
# Overridden by the concrete interface. True when running the text may end the transaction it runs
|
|
78
|
+
# in (a COMMIT of the request's own), so what ran before it may persist whatever happens after
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
def has_ended_transaction(self, conn) -> bool:
|
|
82
|
+
# Overridden by the concrete interface. True when a transactional connection is out of its
|
|
83
|
+
# transaction after a statement succeeded: that statement ended it
|
|
74
84
|
return False
|
|
75
85
|
|
|
76
86
|
def log_warning(self, exc):
|
|
@@ -81,19 +91,25 @@ class DBInterface(ABC):
|
|
|
81
91
|
if self.logging:
|
|
82
92
|
logging.warning(exc, exc_info=False)
|
|
83
93
|
|
|
84
|
-
def __attempt_commit_rollback(self, conn, err):
|
|
94
|
+
def __attempt_commit_rollback(self, conn, err, commit: bool = True):
|
|
85
95
|
# Returns the commit failure rather than swallowing it. A failed commit means the transaction
|
|
86
96
|
# did NOT persist, so the caller must not report success. Silently swallowing it turned lost
|
|
87
97
|
# writes into a misleading downstream error - e.g. create_account's CREATE ROLE lost on a
|
|
88
98
|
# connection killed by a \wipe dbms reboot, then "GRANT registered TO <role>" failing with
|
|
89
99
|
# "role does not exist" on the very next statement.
|
|
100
|
+
# commit False (a read_only request): nothing it did may persist, so it is rolled back like a
|
|
101
|
+
# failed request, never left open for whoever handles the connection next to commit
|
|
90
102
|
try:
|
|
91
|
-
if err is None:
|
|
103
|
+
if err is None and commit:
|
|
104
|
+
if self.is_connection_closed(conn):
|
|
105
|
+
# The COMMIT cannot reach the server, so nothing persisted: a caller that owns the
|
|
106
|
+
# request may re-run it on a fresh connection
|
|
107
|
+
return ConnectionLostError("the connection was closed before COMMIT")
|
|
92
108
|
self.commit(conn)
|
|
93
109
|
else:
|
|
94
110
|
self.rollback(conn)
|
|
95
111
|
except Exception as ex:
|
|
96
|
-
if err is None:
|
|
112
|
+
if err is None and commit:
|
|
97
113
|
self.log_warning(ex) # commit failed - the transaction did not persist
|
|
98
114
|
return ex
|
|
99
115
|
else:
|
|
@@ -113,43 +129,53 @@ class DBInterface(ABC):
|
|
|
113
129
|
self.log_warning(ex) # Serious error, connection failure to db or similar
|
|
114
130
|
raise ex
|
|
115
131
|
|
|
116
|
-
def
|
|
117
|
-
# A failed commit persisted nothing; surface it (instead of the
|
|
118
|
-
# caller cannot treat lost writes as success and hit a misleading error
|
|
119
|
-
#
|
|
120
|
-
# re-run on a fresh connection; any other commit failure is a genuine error.
|
|
121
|
-
if
|
|
122
|
-
|
|
123
|
-
|
|
132
|
+
def __commit_failure(self, conn, commit_err, commit_error_set):
|
|
133
|
+
# A failed commit persisted nothing (or, lost in flight, possibly did); surface it (instead of the
|
|
134
|
+
# old silent swallow) so the caller cannot treat lost writes as success and hit a misleading error
|
|
135
|
+
# further on. A COMMIT that never reached the server is the retriable marker, so a self-contained
|
|
136
|
+
# caller can re-run on a fresh connection; any other commit failure is a genuine error.
|
|
137
|
+
if isinstance(commit_err, ConnectionLostError):
|
|
138
|
+
return commit_err
|
|
139
|
+
try:
|
|
140
|
+
translated = self.translate_commit_error(conn, commit_err, commit_error_set)
|
|
141
|
+
except Exception as ex:
|
|
142
|
+
# Translating reads what the database raised. Failing to must not cost the answer, nor (put_conn_handle_error) the
|
|
143
|
+
# connection, so the failure is answered untranslated
|
|
144
|
+
self.log_warning(ex)
|
|
145
|
+
translated = None
|
|
124
146
|
if translated is not None:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def translate_commit_error(self, commit_err):
|
|
130
|
-
# A deferred constraint
|
|
131
|
-
#
|
|
132
|
-
#
|
|
147
|
+
return translated
|
|
148
|
+
return HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
|
|
149
|
+
HTTPStatus.INTERNAL_SERVER_ERROR)
|
|
150
|
+
|
|
151
|
+
def translate_commit_error(self, conn, commit_err, error_set: str = None):
|
|
152
|
+
# A deferred constraint raises at COMMIT rather than during its statement. A subclass returns the
|
|
153
|
+
# error the statement path would have handed back for it, attributed to error_set (the request's
|
|
154
|
+
# query set, when it has only one), so the caller sees the same response either way; None keeps the 500
|
|
133
155
|
return None
|
|
134
156
|
|
|
135
|
-
def handle_error(self, conn, err, echo=ECHO__none):
|
|
157
|
+
def handle_error(self, conn, err, echo=ECHO__none, commit_error_set: str = None):
|
|
136
158
|
commit_err = self.__attempt_commit_rollback(conn, err)
|
|
137
159
|
if err is None and commit_err is not None:
|
|
138
|
-
self.
|
|
160
|
+
raise self.__commit_failure(conn, commit_err, commit_error_set)
|
|
139
161
|
self.__err_to_exception(err, echo)
|
|
140
162
|
|
|
141
|
-
def put_conn_handle_error(self, conn, err, echo=ECHO__none,
|
|
142
|
-
|
|
143
|
-
if not skip_rollback_commit:
|
|
144
|
-
commit_err = self.__attempt_commit_rollback(conn, err)
|
|
145
|
-
|
|
163
|
+
def put_conn_handle_error(self, conn, err, echo=ECHO__none, skip_commit: bool = False, commit_error_set: str = None):
|
|
164
|
+
commit_failure = None
|
|
146
165
|
try:
|
|
147
|
-
self.
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
166
|
+
commit_err = self.__attempt_commit_rollback(conn, err, commit=not skip_commit)
|
|
167
|
+
# Decided before the connection goes back: from then on the putback thread is working on it
|
|
168
|
+
if err is None and commit_err is not None:
|
|
169
|
+
commit_failure = self.__commit_failure(conn, commit_err, commit_error_set)
|
|
170
|
+
finally:
|
|
171
|
+
# Whatever ending the transaction or deciding the answer raised, the connection goes back to the pool
|
|
172
|
+
try:
|
|
173
|
+
self.put_conn(conn)
|
|
174
|
+
except Exception as ex:
|
|
175
|
+
self.log_warning(ex)
|
|
176
|
+
|
|
177
|
+
if commit_failure is not None:
|
|
178
|
+
raise commit_failure
|
|
153
179
|
|
|
154
180
|
self.__err_to_exception(err, echo)
|
|
155
181
|
|
jaaql/db/db_pg_interface.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import re
|
|
1
2
|
import uuid
|
|
2
3
|
|
|
3
4
|
import psycopg
|
|
4
5
|
from psycopg import OperationalError
|
|
6
|
+
from psycopg.conninfo import make_conninfo
|
|
7
|
+
from psycopg.pq import TransactionStatus
|
|
5
8
|
from psycopg_pool import ConnectionPool, PoolClosed
|
|
6
9
|
import queue
|
|
7
10
|
from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
|
|
@@ -12,10 +15,12 @@ from jaaql.constants import ERR__invalid_token
|
|
|
12
15
|
from jaaql.db.db_interface import DBInterface, ECHO__none, CHAR__newline
|
|
13
16
|
from jaaql.exceptions.http_status_exception import *
|
|
14
17
|
from jaaql.exceptions.custom_http_status import CustomHTTPStatus
|
|
15
|
-
from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized,
|
|
18
|
+
from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, DatabaseOperationalError, handled_error_from_database_error, \
|
|
19
|
+
database_error_descriptor
|
|
16
20
|
from jaaql.constants import KEY__database
|
|
17
21
|
|
|
18
22
|
ERR__connect_db = "Could not create connection to database!"
|
|
23
|
+
ERR__commit_outcome_unknown = "The connection was lost during COMMIT, so whether the transaction was persisted is unknown: "
|
|
19
24
|
|
|
20
25
|
PGCONN__min_conns = 5
|
|
21
26
|
PGCONN__max_conns = 10
|
|
@@ -111,6 +116,48 @@ def _statement_is_preparable(query: str) -> bool:
|
|
|
111
116
|
return ";" not in query.rstrip().rstrip(";")
|
|
112
117
|
|
|
113
118
|
|
|
119
|
+
REGEX__word = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*")
|
|
120
|
+
STATEMENTS__ending_transaction = {"COMMIT", "END", "ROLLBACK", "ABORT"}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _leading_word(query: str, i: int = 0) -> (str, int):
|
|
124
|
+
# The first word at or after i, upper-cased, and where it ends. Whitespace, -- comments and /* */ comments, which nest, are
|
|
125
|
+
# skipped as Postgres skips them
|
|
126
|
+
n = len(query)
|
|
127
|
+
while i < n:
|
|
128
|
+
if query[i].isspace():
|
|
129
|
+
i += 1
|
|
130
|
+
elif query.startswith("--", i):
|
|
131
|
+
while i < n and query[i] not in "\r\n":
|
|
132
|
+
i += 1
|
|
133
|
+
elif query.startswith("/*", i):
|
|
134
|
+
depth = 0
|
|
135
|
+
while i < n:
|
|
136
|
+
if query.startswith("/*", i):
|
|
137
|
+
depth += 1
|
|
138
|
+
i += 2
|
|
139
|
+
elif query.startswith("*/", i):
|
|
140
|
+
depth -= 1
|
|
141
|
+
i += 2
|
|
142
|
+
if depth == 0:
|
|
143
|
+
break
|
|
144
|
+
else:
|
|
145
|
+
i += 1
|
|
146
|
+
else:
|
|
147
|
+
break
|
|
148
|
+
match = REGEX__word.match(query, i)
|
|
149
|
+
return (match.group(0).upper(), match.end()) if match is not None else ("", i)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _statement_may_end_transaction(query: str) -> bool:
|
|
153
|
+
# True when running the text may end the transaction it runs in, persisting what ran before it: several statements, any of
|
|
154
|
+
# which may (a ';' in a literal counts too, erring towards True), or one COMMIT, END, ROLLBACK, ABORT or PREPARE TRANSACTION
|
|
155
|
+
if not _statement_is_preparable(query):
|
|
156
|
+
return True
|
|
157
|
+
word, end = _leading_word(query)
|
|
158
|
+
return word in STATEMENTS__ending_transaction or (word == "PREPARE" and _leading_word(query, end)[0] == "TRANSACTION")
|
|
159
|
+
|
|
160
|
+
|
|
114
161
|
def _escape_unescaped_percent(query: str) -> str:
|
|
115
162
|
# psycopg3 scans every '%' in the SQL when parameters are supplied and
|
|
116
163
|
# rejects anything that isn't a placeholder or '%%'. JAAQL uses named
|
|
@@ -182,28 +229,35 @@ class DBPGInterface(DBInterface):
|
|
|
182
229
|
conn.jaaql_take_pending_auth()
|
|
183
230
|
do_reset = False
|
|
184
231
|
if do_reset:
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
232
|
+
try:
|
|
233
|
+
if conn.info.transaction_status != TransactionStatus.IDLE:
|
|
234
|
+
# Whatever the request left open is not the putback's to persist (a read_only request, a caller that
|
|
235
|
+
# returned a connection mid-transaction): the commit below is for the reset statements alone
|
|
236
|
+
conn.rollback()
|
|
237
|
+
with conn.cursor() as cursor:
|
|
238
|
+
cursor.execute("RESET ROLE;")
|
|
239
|
+
if hasattr(conn, "jaaql_reset_key"):
|
|
190
240
|
cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
|
|
191
|
-
except:
|
|
192
|
-
did_close = True
|
|
193
|
-
conn.close()
|
|
194
|
-
if not did_close:
|
|
195
241
|
cursor.execute("RESET ALL;")
|
|
196
|
-
|
|
242
|
+
conn.commit()
|
|
243
|
+
except Exception:
|
|
244
|
+
# A connection that could not be reset must never serve another request, and must not be kept from the pool
|
|
245
|
+
# either: closed, putconn discards it and the pool opens a replacement
|
|
246
|
+
conn.close()
|
|
197
247
|
DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
|
|
198
248
|
|
|
199
249
|
@staticmethod
|
|
200
250
|
def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
|
|
201
251
|
while True:
|
|
252
|
+
conn, do_reset = the_queue.get()
|
|
202
253
|
try:
|
|
203
|
-
conn, do_reset = the_queue.get()
|
|
204
254
|
DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
|
|
205
255
|
except Exception:
|
|
206
|
-
|
|
256
|
+
# The pool has likely been wiped or replaced. Close the connection so it cannot linger outside any pool
|
|
257
|
+
try:
|
|
258
|
+
conn.close()
|
|
259
|
+
except Exception:
|
|
260
|
+
pass
|
|
207
261
|
|
|
208
262
|
def __init__(self, config, host: str, port: int, db_name: str, username: str, role: str = None, password: str = None, sub_role: str = None,
|
|
209
263
|
session_settings: dict = None):
|
|
@@ -233,15 +287,17 @@ class DBPGInterface(DBInterface):
|
|
|
233
287
|
|
|
234
288
|
if password is not None:
|
|
235
289
|
try:
|
|
236
|
-
|
|
290
|
+
conn_params = {"user": username, "password": password, "dbname": db_name}
|
|
237
291
|
# Important we don't list the host as this will force a unix socket
|
|
238
292
|
if host is not None and host not in ['localhost', '127.0.0.1']:
|
|
239
|
-
|
|
293
|
+
conn_params["host"] = host
|
|
240
294
|
|
|
241
295
|
if str(port) != "5432":
|
|
242
|
-
|
|
296
|
+
conn_params["port"] = str(port)
|
|
243
297
|
|
|
244
|
-
|
|
298
|
+
# make_conninfo quotes every value, so no value can add a setting of its own (a database named
|
|
299
|
+
# "x host=elsewhere" would otherwise send the superuser credentials to that host)
|
|
300
|
+
conn_str = make_conninfo(CONN_STR__keepalives.strip(), **conn_params)
|
|
245
301
|
|
|
246
302
|
self.conn_str = conn_str
|
|
247
303
|
|
|
@@ -280,9 +336,9 @@ class DBPGInterface(DBInterface):
|
|
|
280
336
|
if self.role is not None or self.sub_role is not None or self.session_settings:
|
|
281
337
|
# Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
|
|
282
338
|
# JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
|
|
283
|
-
# raise are translated by _translate_session_auth_error at execution time; dead pool
|
|
284
|
-
#
|
|
285
|
-
#
|
|
339
|
+
# raise are translated by _translate_session_auth_error at execution time; a dead pool
|
|
340
|
+
# connection, previously detected here, surfaces at that first query, and the request
|
|
341
|
+
# is re-run on a fresh connection (InterpretJAAQL.transform, ConnectionLostError)
|
|
286
342
|
pending = []
|
|
287
343
|
if self.role is not None:
|
|
288
344
|
pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
|
|
@@ -355,107 +411,102 @@ class DBPGInterface(DBInterface):
|
|
|
355
411
|
|
|
356
412
|
def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False,
|
|
357
413
|
capture_provenance: list = None):
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
try:
|
|
363
|
-
cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
|
|
364
|
-
with cursor_factory() as cursor:
|
|
365
|
-
do_prepare = prepare and _statement_is_preparable(query)
|
|
414
|
+
try:
|
|
415
|
+
cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
|
|
416
|
+
with cursor_factory() as cursor:
|
|
417
|
+
do_prepare = prepare and _statement_is_preparable(query)
|
|
366
418
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
if
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
419
|
+
if wait_hook:
|
|
420
|
+
try:
|
|
421
|
+
verdict = wait_hook.get(timeout=WAIT_HOOK__timeout)
|
|
422
|
+
except queue.Empty:
|
|
423
|
+
# The parallel verifier never delivered a verdict (its single serial thread
|
|
424
|
+
# wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
|
|
425
|
+
# the worker unwind - without this bound the worker blocks forever holding
|
|
426
|
+
# an open transaction, and enough of them starve the pool into nginx 504s.
|
|
427
|
+
raise Exception(ERR__verification_timed_out)
|
|
428
|
+
# The verdict belongs to the request, not to this attempt at it: left on the hook, a
|
|
429
|
+
# request re-run on a fresh connection after losing its first one reads it again
|
|
430
|
+
# instead of waiting out WAIT_HOOK__timeout for a second verdict that never comes
|
|
431
|
+
wait_hook.put(verdict)
|
|
432
|
+
res, err, code = verdict
|
|
433
|
+
if not res:
|
|
434
|
+
if code == 500:
|
|
435
|
+
raise Exception(err)
|
|
436
|
+
raise UserUnauthorized()
|
|
437
|
+
|
|
438
|
+
def execute_main_query():
|
|
439
|
+
if parameters is None or len(parameters.keys()) == 0:
|
|
440
|
+
cursor.execute(query, prepare=do_prepare)
|
|
441
|
+
else:
|
|
442
|
+
cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
|
|
386
443
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
444
|
+
# Claimed only now, after the wait_hook: if verification rejects the request,
|
|
445
|
+
# the statements remain pending and the putback thread discards them unsent
|
|
446
|
+
pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
|
|
390
447
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
for statement in pending_auth:
|
|
408
|
-
_execute_pending_statement(auth_cursor, statement)
|
|
409
|
-
execute_main_query()
|
|
410
|
-
else:
|
|
448
|
+
if pending_auth:
|
|
449
|
+
auth_cursor = conn.jaaql_raw_cursor()
|
|
450
|
+
try:
|
|
451
|
+
# Pipeline mode is unsafe for two kinds of statement, so restrict it to
|
|
452
|
+
# single-command queries on transactional (non-autocommit) connections;
|
|
453
|
+
# everything else falls back to sequential execution:
|
|
454
|
+
# - it forces the extended protocol on every execute, which rejects a
|
|
455
|
+
# multi-command string ("cannot insert multiple commands into a
|
|
456
|
+
# prepared statement"); the sequential fallback runs the main query
|
|
457
|
+
# under the simple protocol that permits several commands.
|
|
458
|
+
# - it wraps statements in an implicit transaction, which statements
|
|
459
|
+
# that cannot run in a transaction block (CREATE DATABASE, VACUUM,
|
|
460
|
+
# CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
|
|
461
|
+
# autocommit=True, so skipping the pipeline there keeps them standalone.
|
|
462
|
+
if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
|
|
463
|
+
with conn.pipeline():
|
|
411
464
|
for statement in pending_auth:
|
|
412
465
|
_execute_pending_statement(auth_cursor, statement)
|
|
413
466
|
execute_main_query()
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
return [], [], []
|
|
431
|
-
else:
|
|
432
|
-
if capture_provenance is not None:
|
|
433
|
-
capture_provenance.clear()
|
|
434
|
-
pgresult = cursor.pgresult
|
|
435
|
-
capture_provenance.extend(
|
|
436
|
-
(pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
|
|
437
|
-
for idx in range(pgresult.nfields))
|
|
438
|
-
return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
|
|
439
|
-
except OperationalError as ex:
|
|
440
|
-
if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
|
|
441
|
-
if isinstance(conn, JaaqlPGConnection):
|
|
442
|
-
# This putconn bypasses the reset queue, so make sure no unsent
|
|
443
|
-
# authorization statements ride back into the pool where check() or a
|
|
444
|
-
# later checkout could flush them as the wrong user
|
|
445
|
-
conn.jaaql_take_pending_auth()
|
|
446
|
-
DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
|
|
447
|
-
DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
|
|
448
|
-
conn = self.get_conn()
|
|
449
|
-
err = ex
|
|
467
|
+
else:
|
|
468
|
+
for statement in pending_auth:
|
|
469
|
+
_execute_pending_statement(auth_cursor, statement)
|
|
470
|
+
execute_main_query()
|
|
471
|
+
except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
|
|
472
|
+
translated = self._translate_session_auth_error(ex)
|
|
473
|
+
if translated is not None:
|
|
474
|
+
raise translated
|
|
475
|
+
if isinstance(ex, InternalError):
|
|
476
|
+
traceback.print_exc()
|
|
477
|
+
raise ex
|
|
478
|
+
finally:
|
|
479
|
+
try:
|
|
480
|
+
auth_cursor.close()
|
|
481
|
+
except Exception:
|
|
482
|
+
pass
|
|
450
483
|
else:
|
|
451
|
-
|
|
452
|
-
except Exception as ex:
|
|
453
|
-
if self.output_query_exceptions:
|
|
454
|
-
traceback.print_exc()
|
|
455
|
-
raise ex
|
|
484
|
+
execute_main_query()
|
|
456
485
|
|
|
457
|
-
|
|
458
|
-
|
|
486
|
+
if cursor.description is None:
|
|
487
|
+
return [], [], []
|
|
488
|
+
else:
|
|
489
|
+
if capture_provenance is not None:
|
|
490
|
+
capture_provenance.clear()
|
|
491
|
+
pgresult = cursor.pgresult
|
|
492
|
+
capture_provenance.extend(
|
|
493
|
+
(pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
|
|
494
|
+
for idx in range(pgresult.nfields))
|
|
495
|
+
return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
|
|
496
|
+
except Exception as ex:
|
|
497
|
+
if isinstance(ex, OperationalError) and conn.closed:
|
|
498
|
+
# The connection is gone and the transaction on it with it. The query is not retried here on
|
|
499
|
+
# another connection: the caller holds this one and would go on to commit and return it, so
|
|
500
|
+
# the request is re-run as a whole by the owner of its connection (InterpretJAAQL.transform,
|
|
501
|
+
# ConnectionLostError). A database restart leaves every idle pooled connection dead, so have
|
|
502
|
+
# the pool replace those before that re-run checks one out
|
|
503
|
+
try:
|
|
504
|
+
DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
|
|
505
|
+
except Exception:
|
|
506
|
+
pass
|
|
507
|
+
if self.output_query_exceptions:
|
|
508
|
+
traceback.print_exc()
|
|
509
|
+
raise ex
|
|
459
510
|
|
|
460
511
|
def commit(self, conn):
|
|
461
512
|
conn.commit()
|
|
@@ -463,16 +514,28 @@ class DBPGInterface(DBInterface):
|
|
|
463
514
|
def rollback(self, conn):
|
|
464
515
|
conn.rollback()
|
|
465
516
|
|
|
466
|
-
def
|
|
467
|
-
# psycopg
|
|
468
|
-
#
|
|
469
|
-
|
|
470
|
-
|
|
517
|
+
def is_connection_closed(self, conn) -> bool:
|
|
518
|
+
# True once psycopg has seen the connection fail (backend terminated by \wipe dbms or a restart, network
|
|
519
|
+
# loss) or it was closed. Whatever transaction was open on it can no longer commit
|
|
520
|
+
return conn.closed
|
|
521
|
+
|
|
522
|
+
def statement_may_end_transaction(self, query: str) -> bool:
|
|
523
|
+
return _statement_may_end_transaction(query)
|
|
524
|
+
|
|
525
|
+
def has_ended_transaction(self, conn) -> bool:
|
|
526
|
+
return not conn.autocommit and conn.info.transaction_status == TransactionStatus.IDLE
|
|
471
527
|
|
|
472
|
-
def translate_commit_error(self, commit_err):
|
|
528
|
+
def translate_commit_error(self, conn, commit_err, error_set: str = None):
|
|
473
529
|
if not isinstance(commit_err, psycopg.Error):
|
|
474
530
|
return None
|
|
475
|
-
|
|
531
|
+
if conn.closed:
|
|
532
|
+
# Lost while the COMMIT was in flight: the server may have committed before the connection went (a crash
|
|
533
|
+
# after the commit record was flushed, a network loss before the reply). The outcome is unknown, so this
|
|
534
|
+
# is reported as it stands and never re-run, which could apply the request twice
|
|
535
|
+
return DatabaseOperationalError(message=ERR__commit_outcome_unknown + str(commit_err), descriptor=database_error_descriptor(commit_err))
|
|
536
|
+
# The server refused the COMMIT (a deferred constraint, a serialization failure, ...), so the transaction
|
|
537
|
+
# was rolled back: the error maps exactly as it does when a statement raises it
|
|
538
|
+
return handled_error_from_database_error(commit_err, error_set)
|
|
476
539
|
|
|
477
540
|
def handle_db_error(self, err, echo):
|
|
478
541
|
if isinstance(err, ProgrammingError) and hasattr(err, 'pgresult'):
|
jaaql/db/db_utils.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from jaaql.exceptions.http_status_exception import HttpStatusException, HttpSingletonStatusException, ConnectionLostError
|
|
2
2
|
from jaaql.interpreter.interpret_jaaql import InterpretJAAQL
|
|
3
|
-
from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials
|
|
3
|
+
from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials, REGEX__dmbs_object_name
|
|
4
4
|
from typing import Union
|
|
5
|
+
import re
|
|
5
6
|
import jaaql.utilities.crypt_utils as crypt_utils
|
|
6
7
|
from jaaql.db.db_pg_interface import DBPGInterface
|
|
7
8
|
from jaaql.db.db_interface import DBInterface
|
|
@@ -15,6 +16,7 @@ ERR__duplicated_encryption_salt = "Duplicated value in encryption_salts list"
|
|
|
15
16
|
ERR__expected_single_row = "Expected single row response but received '%d' rows"
|
|
16
17
|
ERR__unsupported_interface = "Unsupported interface '%s'. We only support %s"
|
|
17
18
|
ERR__schema_invalid = "Schema invalid!"
|
|
19
|
+
ERR__database_name_unsafe = "Unsafe database name"
|
|
18
20
|
|
|
19
21
|
# A lost connection persisted nothing, so a self-contained statement is re-run on a fresh connection.
|
|
20
22
|
# Bounded so a genuinely-down database still surfaces an error rather than looping.
|
|
@@ -194,6 +196,13 @@ def execute_supplied_statements(db_interface, queries: Union[str, list],
|
|
|
194
196
|
return data
|
|
195
197
|
|
|
196
198
|
|
|
199
|
+
def requested_database(database) -> str:
|
|
200
|
+
# A database named by the request itself: a plain name, as every database JAAQL uses has one
|
|
201
|
+
if not isinstance(database, str) or re.fullmatch(REGEX__dmbs_object_name, database) is None:
|
|
202
|
+
raise HttpStatusException(ERR__database_name_unsafe)
|
|
203
|
+
return database
|
|
204
|
+
|
|
205
|
+
|
|
197
206
|
def create_interface_for_db(vault, config, user_id: str, database: str, sub_role: str = None, session_settings: dict = None):
|
|
198
207
|
jaaql_uri = vault.get_obj(VAULT_KEY__super_db_credentials)
|
|
199
208
|
address, port, _, username, password = DBInterface.fracture_uri(jaaql_uri)
|
jaaql/db/db_utils_no_circ.py
CHANGED
|
@@ -4,7 +4,8 @@ from http import HTTPStatus
|
|
|
4
4
|
from jaaql.mvc.exception_queries import QUERY__fetch_application_schemas, KG__application_schema__application, KG__application__is_live, \
|
|
5
5
|
KG__application_schema__name, KEY__is_default
|
|
6
6
|
from queue import Queue
|
|
7
|
-
from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts
|
|
7
|
+
from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts, \
|
|
8
|
+
requested_database
|
|
8
9
|
from jaaql.exceptions.http_status_exception import HttpStatusException, ConnectionLostError
|
|
9
10
|
from jaaql.interpreter.interpret_jaaql import InterpretJAAQL, KEY_autocommit
|
|
10
11
|
from jaaql.constants import KEY__application, KEY__database, KEY__schema, KEY__role, DB__jaaql, \
|
|
@@ -122,6 +123,8 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
|
|
|
122
123
|
raise HttpStatusException(ERR__schema_invalid)
|
|
123
124
|
|
|
124
125
|
inputs[KEY__database] = found_db
|
|
126
|
+
elif KEY__database in inputs:
|
|
127
|
+
requested_database(inputs[KEY__database])
|
|
125
128
|
|
|
126
129
|
if KEY__database not in inputs:
|
|
127
130
|
inputs[KEY__database] = DB__jaaql
|
|
@@ -36,11 +36,12 @@ class HttpSingletonStatusException(HttpStatusException):
|
|
|
36
36
|
|
|
37
37
|
class ConnectionLostError(HttpStatusException):
|
|
38
38
|
# Raised when a database operation failed because its connection was lost (e.g. the backend was
|
|
39
|
-
# terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed
|
|
40
|
-
#
|
|
41
|
-
#
|
|
39
|
+
# terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed: during a statement of
|
|
40
|
+
# a transaction, or before its COMMIT was sent. Nothing persisted, so a self-contained operation may
|
|
41
|
+
# safely be retried on a fresh connection. Subclasses HttpStatusException so that if retries are
|
|
42
|
+
# exhausted it still surfaces as a clean 500.
|
|
42
43
|
def __init__(self, message: str):
|
|
43
|
-
super().__init__("
|
|
44
|
+
super().__init__("Connection lost, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
class JaaqlInterpretableHandledError(Exception):
|