jaaql-middleware-python 5.3.20__tar.gz → 5.3.21__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/PKG-INFO +1 -1
  2. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/constants.py +1 -1
  3. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/db/db_interface.py +61 -35
  4. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/db/db_pg_interface.py +176 -116
  5. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/exceptions/http_status_exception.py +5 -4
  6. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/exceptions/jaaql_interpretable_handled_errors.py +37 -1
  7. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/interpreter/interpret_jaaql.py +72 -92
  8. jaaql_middleware_python-5.3.21/jaaql/test/test_connection_lifecycle.py +675 -0
  9. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql_middleware_python.egg-info/PKG-INFO +1 -1
  10. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql_middleware_python.egg-info/SOURCES.txt +1 -0
  11. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/LICENSE.txt +0 -0
  12. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/README.md +0 -0
  13. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/__init__.py +0 -0
  14. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/config/__init__.py +0 -0
  15. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/config/config-docker.ini +0 -0
  16. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/config/config-test.ini +0 -0
  17. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/config/config.ini +0 -0
  18. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/config_constants.py +0 -0
  19. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/db/__init__.py +0 -0
  20. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/db/db_utils.py +0 -0
  21. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/db/db_utils_no_circ.py +0 -0
  22. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/documentation/__init__.py +0 -0
  23. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/documentation/documentation_internal.py +0 -0
  24. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/documentation/documentation_public.py +0 -0
  25. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/documentation/documentation_shared.py +0 -0
  26. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/email/__init__.py +0 -0
  27. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/email/email_manager.py +0 -0
  28. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/email/email_manager_service.py +0 -0
  29. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/email/patch_ems.py +0 -0
  30. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/exceptions/__init__.py +0 -0
  31. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/exceptions/custom_http_status.py +0 -0
  32. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/exceptions/not_yet_implement_exception.py +0 -0
  33. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/generated_constants.py +0 -0
  34. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/interpreter/__init__.py +0 -0
  35. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/jaaql.py +0 -0
  36. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/migrations/__init__.py +0 -0
  37. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/migrations/migrations.py +0 -0
  38. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/__init__.py +0 -0
  39. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/base_controller.py +0 -0
  40. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/base_model.py +0 -0
  41. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/controller.py +0 -0
  42. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/controller_interface.py +0 -0
  43. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/exception_queries.py +0 -0
  44. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/generated_queries.py +0 -0
  45. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/handmade_queries.py +0 -0
  46. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/model.py +0 -0
  47. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/model_interface.py +0 -0
  48. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/mvc/response.py +0 -0
  49. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/openapi/__init__.py +0 -0
  50. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/openapi/swagger_documentation.py +0 -0
  51. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/patch.py +0 -0
  52. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/01.install_domains.generated.sql +0 -0
  53. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/02.install_super_user.exceptions.sql +0 -0
  54. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/03.install_super_user.handwritten.sql +0 -0
  55. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/04.install_jaaql_data_structures.generated.sql +0 -0
  56. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/05.install_static_data.generated.sql +0 -0
  57. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/06.install_jaaql.exceptions.sql +0 -0
  58. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/ZZZZ.generated_functions_views_and_permissions.sql +0 -0
  59. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/ZZZZ.reset_references.sql +0 -0
  60. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/scripts/swagger_template.html +0 -0
  61. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/__init__.py +0 -0
  62. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/cached_canned_query_service.py +0 -0
  63. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/migrations_manager_service.py +0 -0
  64. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/patch_mms.py +0 -0
  65. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/patch_shared_var_service.py +0 -0
  66. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/services/shared_var_service.py +0 -0
  67. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/test/__init__.py +0 -0
  68. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/test/test_timelines.py +0 -0
  69. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/__init__.py +0 -0
  70. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/bootstrap_secrets.py +0 -0
  71. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/cron.py +0 -0
  72. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/crypt_utils.py +0 -0
  73. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/options.py +0 -0
  74. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/utils.py +0 -0
  75. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/utils_no_project_imports.py +0 -0
  76. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql/utilities/vault.py +0 -0
  77. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql_middleware_python.egg-info/dependency_links.txt +0 -0
  78. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql_middleware_python.egg-info/requires.txt +0 -0
  79. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/jaaql_middleware_python.egg-info/top_level.txt +0 -0
  80. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/setup.cfg +0 -0
  81. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.21}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jaaql-middleware-python
3
- Version: 5.3.20
3
+ Version: 5.3.21
4
4
  Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
5
5
  Home-page: https://github.com/JAAQL/JAAQL-middleware-python
6
6
  Author: Software Quality Measurement and Improvement bv
@@ -236,5 +236,5 @@ ROLE__dba = "dba"
236
236
 
237
237
  PROTOCOL__postgres = "postgresql://"
238
238
 
239
- VERSION = "5.3.20"
239
+ VERSION = "5.3.21"
240
240
 
@@ -68,9 +68,19 @@ class DBInterface(ABC):
68
68
  def get_conn(self):
69
69
  pass
70
70
 
71
- def is_connection_error(self, ex) -> bool:
72
- # Overridden by the concrete interface. True when a failure means the connection/backend went
73
- # away (so nothing persisted and the operation is safe to retry on a fresh connection).
71
+ def is_connection_closed(self, conn) -> bool:
72
+ # Overridden by the concrete interface. True when the connection/backend went away, taking any
73
+ # uncommitted transaction with it (so that transaction persisted nothing)
74
+ return False
75
+
76
+ def statement_may_end_transaction(self, query: str) -> bool:
77
+ # Overridden by the concrete interface. True when running the text may end the transaction it runs
78
+ # in (a COMMIT of the request's own), so what ran before it may persist whatever happens after
79
+ return True
80
+
81
+ def has_ended_transaction(self, conn) -> bool:
82
+ # Overridden by the concrete interface. True when a transactional connection is out of its
83
+ # transaction after a statement succeeded: that statement ended it
74
84
  return False
75
85
 
76
86
  def log_warning(self, exc):
@@ -81,19 +91,25 @@ class DBInterface(ABC):
81
91
  if self.logging:
82
92
  logging.warning(exc, exc_info=False)
83
93
 
84
- def __attempt_commit_rollback(self, conn, err):
94
+ def __attempt_commit_rollback(self, conn, err, commit: bool = True):
85
95
  # Returns the commit failure rather than swallowing it. A failed commit means the transaction
86
96
  # did NOT persist, so the caller must not report success. Silently swallowing it turned lost
87
97
  # writes into a misleading downstream error - e.g. create_account's CREATE ROLE lost on a
88
98
  # connection killed by a \wipe dbms reboot, then "GRANT registered TO <role>" failing with
89
99
  # "role does not exist" on the very next statement.
100
+ # commit False (a read_only request): nothing it did may persist, so it is rolled back like a
101
+ # failed request, never left open for whoever handles the connection next to commit
90
102
  try:
91
- if err is None:
103
+ if err is None and commit:
104
+ if self.is_connection_closed(conn):
105
+ # The COMMIT cannot reach the server, so nothing persisted: a caller that owns the
106
+ # request may re-run it on a fresh connection
107
+ return ConnectionLostError("the connection was closed before COMMIT")
92
108
  self.commit(conn)
93
109
  else:
94
110
  self.rollback(conn)
95
111
  except Exception as ex:
96
- if err is None:
112
+ if err is None and commit:
97
113
  self.log_warning(ex) # commit failed - the transaction did not persist
98
114
  return ex
99
115
  else:
@@ -113,43 +129,53 @@ class DBInterface(ABC):
113
129
  self.log_warning(ex) # Serious error, connection failure to db or similar
114
130
  raise ex
115
131
 
116
- def __raise_commit_failure(self, commit_err):
117
- # A failed commit persisted nothing; surface it (instead of the old silent swallow) so the
118
- # caller cannot treat lost writes as success and hit a misleading error further on. A
119
- # connection-loss failure is raised as a retriable marker so a self-contained caller can
120
- # re-run on a fresh connection; any other commit failure is a genuine error.
121
- if self.is_connection_error(commit_err):
122
- raise ConnectionLostError(str(commit_err))
123
- translated = self.translate_commit_error(commit_err)
132
+ def __commit_failure(self, conn, commit_err, commit_error_set):
133
+ # A failed commit persisted nothing (or, lost in flight, possibly did); surface it (instead of the
134
+ # old silent swallow) so the caller cannot treat lost writes as success and hit a misleading error
135
+ # further on. A COMMIT that never reached the server is the retriable marker, so a self-contained
136
+ # caller can re-run on a fresh connection; any other commit failure is a genuine error.
137
+ if isinstance(commit_err, ConnectionLostError):
138
+ return commit_err
139
+ try:
140
+ translated = self.translate_commit_error(conn, commit_err, commit_error_set)
141
+ except Exception as ex:
142
+ # Translating reads what the database raised. Failing to must not cost the answer, nor (put_conn_handle_error) the
143
+ # connection, so the failure is answered untranslated
144
+ self.log_warning(ex)
145
+ translated = None
124
146
  if translated is not None:
125
- raise translated
126
- raise HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
127
- HTTPStatus.INTERNAL_SERVER_ERROR)
128
-
129
- def translate_commit_error(self, commit_err):
130
- # A deferred constraint trigger raises at COMMIT rather than during its statement. When what it
131
- # raises is an error the statement path would have handed back as a handled error, a subclass
132
- # returns that error here so the caller sees the same response either way; None keeps the 500
147
+ return translated
148
+ return HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
149
+ HTTPStatus.INTERNAL_SERVER_ERROR)
150
+
151
+ def translate_commit_error(self, conn, commit_err, error_set: str = None):
152
+ # A deferred constraint raises at COMMIT rather than during its statement. A subclass returns the
153
+ # error the statement path would have handed back for it, attributed to error_set (the request's
154
+ # query set, when it has only one), so the caller sees the same response either way; None keeps the 500
133
155
  return None
134
156
 
135
- def handle_error(self, conn, err, echo=ECHO__none):
157
+ def handle_error(self, conn, err, echo=ECHO__none, commit_error_set: str = None):
136
158
  commit_err = self.__attempt_commit_rollback(conn, err)
137
159
  if err is None and commit_err is not None:
138
- self.__raise_commit_failure(commit_err)
160
+ raise self.__commit_failure(conn, commit_err, commit_error_set)
139
161
  self.__err_to_exception(err, echo)
140
162
 
141
- def put_conn_handle_error(self, conn, err, echo=ECHO__none, skip_rollback_commit: bool = False):
142
- commit_err = None
143
- if not skip_rollback_commit:
144
- commit_err = self.__attempt_commit_rollback(conn, err)
145
-
163
+ def put_conn_handle_error(self, conn, err, echo=ECHO__none, skip_commit: bool = False, commit_error_set: str = None):
164
+ commit_failure = None
146
165
  try:
147
- self.put_conn(conn)
148
- except Exception as ex:
149
- self.log_warning(ex)
150
-
151
- if err is None and commit_err is not None:
152
- self.__raise_commit_failure(commit_err)
166
+ commit_err = self.__attempt_commit_rollback(conn, err, commit=not skip_commit)
167
+ # Decided before the connection goes back: from then on the putback thread is working on it
168
+ if err is None and commit_err is not None:
169
+ commit_failure = self.__commit_failure(conn, commit_err, commit_error_set)
170
+ finally:
171
+ # Whatever ending the transaction or deciding the answer raised, the connection goes back to the pool
172
+ try:
173
+ self.put_conn(conn)
174
+ except Exception as ex:
175
+ self.log_warning(ex)
176
+
177
+ if commit_failure is not None:
178
+ raise commit_failure
153
179
 
154
180
  self.__err_to_exception(err, echo)
155
181
 
@@ -1,7 +1,9 @@
1
+ import re
1
2
  import uuid
2
3
 
3
4
  import psycopg
4
5
  from psycopg import OperationalError
6
+ from psycopg.pq import TransactionStatus
5
7
  from psycopg_pool import ConnectionPool, PoolClosed
6
8
  import queue
7
9
  from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
@@ -12,10 +14,12 @@ from jaaql.constants import ERR__invalid_token
12
14
  from jaaql.db.db_interface import DBInterface, ECHO__none, CHAR__newline
13
15
  from jaaql.exceptions.http_status_exception import *
14
16
  from jaaql.exceptions.custom_http_status import CustomHTTPStatus
15
- from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, handled_procedure_error_from_raise
17
+ from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, DatabaseOperationalError, handled_error_from_database_error, \
18
+ database_error_descriptor
16
19
  from jaaql.constants import KEY__database
17
20
 
18
21
  ERR__connect_db = "Could not create connection to database!"
22
+ ERR__commit_outcome_unknown = "The connection was lost during COMMIT, so whether the transaction was persisted is unknown: "
19
23
 
20
24
  PGCONN__min_conns = 5
21
25
  PGCONN__max_conns = 10
@@ -111,6 +115,48 @@ def _statement_is_preparable(query: str) -> bool:
111
115
  return ";" not in query.rstrip().rstrip(";")
112
116
 
113
117
 
118
+ REGEX__word = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*")
119
+ STATEMENTS__ending_transaction = {"COMMIT", "END", "ROLLBACK", "ABORT"}
120
+
121
+
122
+ def _leading_word(query: str, i: int = 0) -> (str, int):
123
+ # The first word at or after i, upper-cased, and where it ends. Whitespace, -- comments and /* */ comments, which nest, are
124
+ # skipped as Postgres skips them
125
+ n = len(query)
126
+ while i < n:
127
+ if query[i].isspace():
128
+ i += 1
129
+ elif query.startswith("--", i):
130
+ while i < n and query[i] not in "\r\n":
131
+ i += 1
132
+ elif query.startswith("/*", i):
133
+ depth = 0
134
+ while i < n:
135
+ if query.startswith("/*", i):
136
+ depth += 1
137
+ i += 2
138
+ elif query.startswith("*/", i):
139
+ depth -= 1
140
+ i += 2
141
+ if depth == 0:
142
+ break
143
+ else:
144
+ i += 1
145
+ else:
146
+ break
147
+ match = REGEX__word.match(query, i)
148
+ return (match.group(0).upper(), match.end()) if match is not None else ("", i)
149
+
150
+
151
+ def _statement_may_end_transaction(query: str) -> bool:
152
+ # True when running the text may end the transaction it runs in, persisting what ran before it: several statements, any of
153
+ # which may (a ';' in a literal counts too, erring towards True), or one COMMIT, END, ROLLBACK, ABORT or PREPARE TRANSACTION
154
+ if not _statement_is_preparable(query):
155
+ return True
156
+ word, end = _leading_word(query)
157
+ return word in STATEMENTS__ending_transaction or (word == "PREPARE" and _leading_word(query, end)[0] == "TRANSACTION")
158
+
159
+
114
160
  def _escape_unescaped_percent(query: str) -> str:
115
161
  # psycopg3 scans every '%' in the SQL when parameters are supplied and
116
162
  # rejects anything that isn't a placeholder or '%%'. JAAQL uses named
@@ -182,28 +228,35 @@ class DBPGInterface(DBInterface):
182
228
  conn.jaaql_take_pending_auth()
183
229
  do_reset = False
184
230
  if do_reset:
185
- with conn.cursor() as cursor:
186
- cursor.execute("RESET ROLE;")
187
- did_close = False
188
- if hasattr(conn, "jaaql_reset_key"):
189
- try:
231
+ try:
232
+ if conn.info.transaction_status != TransactionStatus.IDLE:
233
+ # Whatever the request left open is not the putback's to persist (a read_only request, a caller that
234
+ # returned a connection mid-transaction): the commit below is for the reset statements alone
235
+ conn.rollback()
236
+ with conn.cursor() as cursor:
237
+ cursor.execute("RESET ROLE;")
238
+ if hasattr(conn, "jaaql_reset_key"):
190
239
  cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
191
- except:
192
- did_close = True
193
- conn.close()
194
- if not did_close:
195
240
  cursor.execute("RESET ALL;")
196
- conn.commit()
241
+ conn.commit()
242
+ except Exception:
243
+ # A connection that could not be reset must never serve another request, and must not be kept from the pool
244
+ # either: closed, putconn discards it and the pool opens a replacement
245
+ conn.close()
197
246
  DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
198
247
 
199
248
  @staticmethod
200
249
  def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
201
250
  while True:
251
+ conn, do_reset = the_queue.get()
202
252
  try:
203
- conn, do_reset = the_queue.get()
204
253
  DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
205
254
  except Exception:
206
- pass # Ignore, pool has likely been wiped
255
+ # The pool has likely been wiped or replaced. Close the connection so it cannot linger outside any pool
256
+ try:
257
+ conn.close()
258
+ except Exception:
259
+ pass
207
260
 
208
261
  def __init__(self, config, host: str, port: int, db_name: str, username: str, role: str = None, password: str = None, sub_role: str = None,
209
262
  session_settings: dict = None):
@@ -280,9 +333,9 @@ class DBPGInterface(DBInterface):
280
333
  if self.role is not None or self.sub_role is not None or self.session_settings:
281
334
  # Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
282
335
  # JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
283
- # raise are translated by _translate_session_auth_error at execution time; dead pool
284
- # connections, previously detected here, are recycled by execute_query's
285
- # OperationalError retry loop
336
+ # raise are translated by _translate_session_auth_error at execution time; a dead pool
337
+ # connection, previously detected here, surfaces at that first query, and the request
338
+ # is re-run on a fresh connection (InterpretJAAQL.transform, ConnectionLostError)
286
339
  pending = []
287
340
  if self.role is not None:
288
341
  pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
@@ -355,107 +408,102 @@ class DBPGInterface(DBInterface):
355
408
 
356
409
  def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False,
357
410
  capture_provenance: list = None):
358
- x = 0
359
- err = None
360
- while x < PGCONN__max_conns:
361
- x += 1
362
- try:
363
- cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
364
- with cursor_factory() as cursor:
365
- do_prepare = prepare and _statement_is_preparable(query)
411
+ try:
412
+ cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
413
+ with cursor_factory() as cursor:
414
+ do_prepare = prepare and _statement_is_preparable(query)
366
415
 
367
- if wait_hook:
368
- try:
369
- res, err, code = wait_hook.get(timeout=WAIT_HOOK__timeout)
370
- except queue.Empty:
371
- # The parallel verifier never delivered a verdict (its single serial thread
372
- # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
373
- # the worker unwind - without this bound the worker blocks forever holding
374
- # an open transaction, and enough of them starve the pool into nginx 504s.
375
- raise Exception(ERR__verification_timed_out)
376
- if not res:
377
- if code == 500:
378
- raise Exception(err)
379
- raise UserUnauthorized()
380
-
381
- def execute_main_query():
382
- if parameters is None or len(parameters.keys()) == 0:
383
- cursor.execute(query, prepare=do_prepare)
384
- else:
385
- cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
416
+ if wait_hook:
417
+ try:
418
+ verdict = wait_hook.get(timeout=WAIT_HOOK__timeout)
419
+ except queue.Empty:
420
+ # The parallel verifier never delivered a verdict (its single serial thread
421
+ # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
422
+ # the worker unwind - without this bound the worker blocks forever holding
423
+ # an open transaction, and enough of them starve the pool into nginx 504s.
424
+ raise Exception(ERR__verification_timed_out)
425
+ # The verdict belongs to the request, not to this attempt at it: left on the hook, a
426
+ # request re-run on a fresh connection after losing its first one reads it again
427
+ # instead of waiting out WAIT_HOOK__timeout for a second verdict that never comes
428
+ wait_hook.put(verdict)
429
+ res, err, code = verdict
430
+ if not res:
431
+ if code == 500:
432
+ raise Exception(err)
433
+ raise UserUnauthorized()
434
+
435
+ def execute_main_query():
436
+ if parameters is None or len(parameters.keys()) == 0:
437
+ cursor.execute(query, prepare=do_prepare)
438
+ else:
439
+ cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
386
440
 
387
- # Claimed only now, after the wait_hook: if verification rejects the request,
388
- # the statements remain pending and the putback thread discards them unsent
389
- pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
441
+ # Claimed only now, after the wait_hook: if verification rejects the request,
442
+ # the statements remain pending and the putback thread discards them unsent
443
+ pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
390
444
 
391
- if pending_auth:
392
- auth_cursor = conn.jaaql_raw_cursor()
393
- try:
394
- # Pipeline mode is unsafe for two kinds of statement, so restrict it to
395
- # single-command queries on transactional (non-autocommit) connections;
396
- # everything else falls back to sequential execution:
397
- # - it forces the extended protocol on every execute, which rejects a
398
- # multi-command string ("cannot insert multiple commands into a
399
- # prepared statement"); the sequential fallback runs the main query
400
- # under the simple protocol that permits several commands.
401
- # - it wraps statements in an implicit transaction, which statements
402
- # that cannot run in a transaction block (CREATE DATABASE, VACUUM,
403
- # CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
404
- # autocommit=True, so skipping the pipeline there keeps them standalone.
405
- if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
406
- with conn.pipeline():
407
- for statement in pending_auth:
408
- _execute_pending_statement(auth_cursor, statement)
409
- execute_main_query()
410
- else:
445
+ if pending_auth:
446
+ auth_cursor = conn.jaaql_raw_cursor()
447
+ try:
448
+ # Pipeline mode is unsafe for two kinds of statement, so restrict it to
449
+ # single-command queries on transactional (non-autocommit) connections;
450
+ # everything else falls back to sequential execution:
451
+ # - it forces the extended protocol on every execute, which rejects a
452
+ # multi-command string ("cannot insert multiple commands into a
453
+ # prepared statement"); the sequential fallback runs the main query
454
+ # under the simple protocol that permits several commands.
455
+ # - it wraps statements in an implicit transaction, which statements
456
+ # that cannot run in a transaction block (CREATE DATABASE, VACUUM,
457
+ # CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
458
+ # autocommit=True, so skipping the pipeline there keeps them standalone.
459
+ if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
460
+ with conn.pipeline():
411
461
  for statement in pending_auth:
412
462
  _execute_pending_statement(auth_cursor, statement)
413
463
  execute_main_query()
414
- except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
415
- translated = self._translate_session_auth_error(ex)
416
- if translated is not None:
417
- raise translated
418
- if isinstance(ex, InternalError):
419
- traceback.print_exc()
420
- raise ex
421
- finally:
422
- try:
423
- auth_cursor.close()
424
- except Exception:
425
- pass
426
- else:
427
- execute_main_query()
428
-
429
- if cursor.description is None:
430
- return [], [], []
431
- else:
432
- if capture_provenance is not None:
433
- capture_provenance.clear()
434
- pgresult = cursor.pgresult
435
- capture_provenance.extend(
436
- (pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
437
- for idx in range(pgresult.nfields))
438
- return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
439
- except OperationalError as ex:
440
- if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
441
- if isinstance(conn, JaaqlPGConnection):
442
- # This putconn bypasses the reset queue, so make sure no unsent
443
- # authorization statements ride back into the pool where check() or a
444
- # later checkout could flush them as the wrong user
445
- conn.jaaql_take_pending_auth()
446
- DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
447
- DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
448
- conn = self.get_conn()
449
- err = ex
464
+ else:
465
+ for statement in pending_auth:
466
+ _execute_pending_statement(auth_cursor, statement)
467
+ execute_main_query()
468
+ except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
469
+ translated = self._translate_session_auth_error(ex)
470
+ if translated is not None:
471
+ raise translated
472
+ if isinstance(ex, InternalError):
473
+ traceback.print_exc()
474
+ raise ex
475
+ finally:
476
+ try:
477
+ auth_cursor.close()
478
+ except Exception:
479
+ pass
450
480
  else:
451
- raise ex
452
- except Exception as ex:
453
- if self.output_query_exceptions:
454
- traceback.print_exc()
455
- raise ex
481
+ execute_main_query()
456
482
 
457
- if err:
458
- raise err
483
+ if cursor.description is None:
484
+ return [], [], []
485
+ else:
486
+ if capture_provenance is not None:
487
+ capture_provenance.clear()
488
+ pgresult = cursor.pgresult
489
+ capture_provenance.extend(
490
+ (pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
491
+ for idx in range(pgresult.nfields))
492
+ return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
493
+ except Exception as ex:
494
+ if isinstance(ex, OperationalError) and conn.closed:
495
+ # The connection is gone and the transaction on it with it. The query is not retried here on
496
+ # another connection: the caller holds this one and would go on to commit and return it, so
497
+ # the request is re-run as a whole by the owner of its connection (InterpretJAAQL.transform,
498
+ # ConnectionLostError). A database restart leaves every idle pooled connection dead, so have
499
+ # the pool replace those before that re-run checks one out
500
+ try:
501
+ DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
502
+ except Exception:
503
+ pass
504
+ if self.output_query_exceptions:
505
+ traceback.print_exc()
506
+ raise ex
459
507
 
460
508
  def commit(self, conn):
461
509
  conn.commit()
@@ -463,16 +511,28 @@ class DBPGInterface(DBInterface):
463
511
  def rollback(self, conn):
464
512
  conn.rollback()
465
513
 
466
- def is_connection_error(self, ex) -> bool:
467
- # psycopg raises OperationalError when the backend/connection went away (e.g. terminated by
468
- # \wipe dbms). The same class the execute retry loop keys off; a commit that fails this way
469
- # persisted nothing, so the operation is safe to retry on a fresh connection.
470
- return isinstance(ex, OperationalError)
514
+ def is_connection_closed(self, conn) -> bool:
515
+ # True once psycopg has seen the connection fail (backend terminated by \wipe dbms or a restart, network
516
+ # loss) or it was closed. Whatever transaction was open on it can no longer commit
517
+ return conn.closed
518
+
519
+ def statement_may_end_transaction(self, query: str) -> bool:
520
+ return _statement_may_end_transaction(query)
521
+
522
+ def has_ended_transaction(self, conn) -> bool:
523
+ return not conn.autocommit and conn.info.transaction_status == TransactionStatus.IDLE
471
524
 
472
- def translate_commit_error(self, commit_err):
525
+ def translate_commit_error(self, conn, commit_err, error_set: str = None):
473
526
  if not isinstance(commit_err, psycopg.Error):
474
527
  return None
475
- return handled_procedure_error_from_raise(commit_err)
528
+ if conn.closed:
529
+ # Lost while the COMMIT was in flight: the server may have committed before the connection went (a crash
530
+ # after the commit record was flushed, a network loss before the reply). The outcome is unknown, so this
531
+ # is reported as it stands and never re-run, which could apply the request twice
532
+ return DatabaseOperationalError(message=ERR__commit_outcome_unknown + str(commit_err), descriptor=database_error_descriptor(commit_err))
533
+ # The server refused the COMMIT (a deferred constraint, a serialization failure, ...), so the transaction
534
+ # was rolled back: the error maps exactly as it does when a statement raises it
535
+ return handled_error_from_database_error(commit_err, error_set)
476
536
 
477
537
  def handle_db_error(self, err, echo):
478
538
  if isinstance(err, ProgrammingError) and hasattr(err, 'pgresult'):
@@ -36,11 +36,12 @@ class HttpSingletonStatusException(HttpStatusException):
36
36
 
37
37
  class ConnectionLostError(HttpStatusException):
38
38
  # Raised when a database operation failed because its connection was lost (e.g. the backend was
39
- # terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed. Nothing persisted,
40
- # so a self-contained operation may safely be retried on a fresh connection. Subclasses
41
- # HttpStatusException so that if retries are exhausted it still surfaces as a clean 500.
39
+ # terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed: during a statement of
40
+ # a transaction, or before its COMMIT was sent. Nothing persisted, so a self-contained operation may
41
+ # safely be retried on a fresh connection. Subclasses HttpStatusException so that if retries are
42
+ # exhausted it still surfaces as a clean 500.
42
43
  def __init__(self, message: str):
43
- super().__init__("Commit failed, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
44
+ super().__init__("Connection lost, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
44
45
 
45
46
 
46
47
  class JaaqlInterpretableHandledError(Exception):
@@ -1,5 +1,7 @@
1
1
  import json
2
2
 
3
+ from psycopg import OperationalError
4
+
3
5
  from jaaql.constants import SQLStateJaaql
4
6
  from jaaql.exceptions.http_status_exception import JaaqlInterpretableHandledError
5
7
 
@@ -54,7 +56,9 @@ def handled_procedure_error_from_raise(ex):
54
56
  return None
55
57
  try:
56
58
  descriptor = json.loads(diag.message_primary)
57
- except (TypeError, ValueError):
59
+ except Exception:
60
+ # Not JSON, or JSON the parser cannot take (nested past its recursion limit, a RecursionError): whoever raised it, the
61
+ # answer is the unhandled error, never an exception out of the code answering it
58
62
  return None
59
63
  return HandledProcedureError(message=None, index=None, table_name=None, descriptor=descriptor)
60
64
 
@@ -87,6 +91,38 @@ class UnhandledProcedureError(JaaqlInterpretableHandledError):
87
91
  )
88
92
 
89
93
 
94
+ def database_error_descriptor(ex):
95
+ # The diagnostics of a psycopg error. One raised client side (a lost connection, a NUL byte in a parameter) has no SQLSTATE
96
+ sqlstate = ex.diag.sqlstate
97
+ return {
98
+ "class": None if sqlstate is None else sqlstate[0:2],
99
+ "constraint_name": ex.diag.constraint_name,
100
+ "context": ex.diag.context,
101
+ "datatype_name": ex.diag.datatype_name,
102
+ "message_detail": ex.diag.message_detail,
103
+ "message_primary": ex.diag.message_primary,
104
+ "message_hint": ex.diag.message_hint,
105
+ "schema_name": ex.diag.schema_name,
106
+ "severity": ex.diag.severity,
107
+ "sqlstate": sqlstate
108
+ }
109
+
110
+
111
+ def handled_error_from_database_error(ex, query_set):
112
+ # The error a request answers with for a psycopg error, whether its statement raised it or its COMMIT did (a deferred
113
+ # constraint). query_set is the key of the request's query set the error is attributed to; "_jaaql_procedure" is /call-proc's
114
+ if isinstance(ex, OperationalError):
115
+ return DatabaseOperationalError(message=str(ex), descriptor=database_error_descriptor(ex))
116
+ handled = handled_procedure_error_from_raise(ex)
117
+ if handled is not None:
118
+ return handled
119
+ if query_set == "_jaaql_procedure":
120
+ return UnhandledProcedureError(message=str(ex), table_name=ex.diag.table_name, column_name=ex.diag.column_name,
121
+ descriptor=database_error_descriptor(ex))
122
+ return UnhandledQueryError(message=str(ex), _set=query_set, table_name=ex.diag.table_name, column_name=ex.diag.column_name,
123
+ descriptor=database_error_descriptor(ex))
124
+
125
+
90
126
  class AccountAlreadyConfirmed(JaaqlInterpretableHandledError):
91
127
  def __init__(self, descriptor=None):
92
128
  super().__init__(