jaaql-middleware-python 5.3.20__tar.gz → 5.3.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/PKG-INFO +1 -1
  2. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/constants.py +4 -1
  3. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/db/db_interface.py +61 -35
  4. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/db/db_pg_interface.py +183 -120
  5. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/db/db_utils.py +10 -1
  6. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/db/db_utils_no_circ.py +4 -1
  7. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/exceptions/http_status_exception.py +5 -4
  8. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/exceptions/jaaql_interpretable_handled_errors.py +37 -1
  9. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/interpreter/interpret_jaaql.py +72 -92
  10. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/handmade_queries.py +14 -0
  11. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/model.py +38 -12
  12. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/03.install_super_user.handwritten.sql +7 -0
  13. jaaql_middleware_python-5.3.22/jaaql/test/test_call_proc.py +214 -0
  14. jaaql_middleware_python-5.3.22/jaaql/test/test_connection_lifecycle.py +675 -0
  15. jaaql_middleware_python-5.3.22/jaaql/test/test_execute.py +61 -0
  16. jaaql_middleware_python-5.3.22/jaaql/test/test_requested_database.py +130 -0
  17. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql_middleware_python.egg-info/PKG-INFO +1 -1
  18. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql_middleware_python.egg-info/SOURCES.txt +4 -0
  19. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/LICENSE.txt +0 -0
  20. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/README.md +0 -0
  21. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/__init__.py +0 -0
  22. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/config/__init__.py +0 -0
  23. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/config/config-docker.ini +0 -0
  24. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/config/config-test.ini +0 -0
  25. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/config/config.ini +0 -0
  26. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/config_constants.py +0 -0
  27. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/db/__init__.py +0 -0
  28. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/documentation/__init__.py +0 -0
  29. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/documentation/documentation_internal.py +0 -0
  30. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/documentation/documentation_public.py +0 -0
  31. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/documentation/documentation_shared.py +0 -0
  32. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/email/__init__.py +0 -0
  33. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/email/email_manager.py +0 -0
  34. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/email/email_manager_service.py +0 -0
  35. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/email/patch_ems.py +0 -0
  36. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/exceptions/__init__.py +0 -0
  37. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/exceptions/custom_http_status.py +0 -0
  38. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/exceptions/not_yet_implement_exception.py +0 -0
  39. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/generated_constants.py +0 -0
  40. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/interpreter/__init__.py +0 -0
  41. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/jaaql.py +0 -0
  42. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/migrations/__init__.py +0 -0
  43. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/migrations/migrations.py +0 -0
  44. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/__init__.py +0 -0
  45. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/base_controller.py +0 -0
  46. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/base_model.py +0 -0
  47. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/controller.py +0 -0
  48. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/controller_interface.py +0 -0
  49. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/exception_queries.py +0 -0
  50. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/generated_queries.py +0 -0
  51. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/model_interface.py +0 -0
  52. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/mvc/response.py +0 -0
  53. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/openapi/__init__.py +0 -0
  54. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/openapi/swagger_documentation.py +0 -0
  55. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/patch.py +0 -0
  56. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/01.install_domains.generated.sql +0 -0
  57. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/02.install_super_user.exceptions.sql +0 -0
  58. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/04.install_jaaql_data_structures.generated.sql +0 -0
  59. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/05.install_static_data.generated.sql +0 -0
  60. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/06.install_jaaql.exceptions.sql +0 -0
  61. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/ZZZZ.generated_functions_views_and_permissions.sql +0 -0
  62. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/ZZZZ.reset_references.sql +0 -0
  63. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/scripts/swagger_template.html +0 -0
  64. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/__init__.py +0 -0
  65. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/cached_canned_query_service.py +0 -0
  66. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/migrations_manager_service.py +0 -0
  67. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/patch_mms.py +0 -0
  68. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/patch_shared_var_service.py +0 -0
  69. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/services/shared_var_service.py +0 -0
  70. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/test/__init__.py +0 -0
  71. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/test/test_timelines.py +0 -0
  72. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/__init__.py +0 -0
  73. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/bootstrap_secrets.py +0 -0
  74. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/cron.py +0 -0
  75. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/crypt_utils.py +0 -0
  76. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/options.py +0 -0
  77. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/utils.py +0 -0
  78. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/utils_no_project_imports.py +0 -0
  79. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql/utilities/vault.py +0 -0
  80. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql_middleware_python.egg-info/dependency_links.txt +0 -0
  81. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql_middleware_python.egg-info/requires.txt +0 -0
  82. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/jaaql_middleware_python.egg-info/top_level.txt +0 -0
  83. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/setup.cfg +0 -0
  84. {jaaql_middleware_python-5.3.20 → jaaql_middleware_python-5.3.22}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jaaql-middleware-python
3
- Version: 5.3.20
3
+ Version: 5.3.22
4
4
  Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
5
5
  Home-page: https://github.com/JAAQL/JAAQL-middleware-python
6
6
  Author: Software Quality Measurement and Improvement bv
@@ -63,6 +63,9 @@ CRON_dayOfWeek = "dayOfWeek"
63
63
 
64
64
  REGEX__dmbs_object_name = r'^[0-9a-zA-Z_]{1,63}$'
65
65
  REGEX__dmbs_procedure_name = r'^[0-9a-zA-Z_$-.\+]{1,63}$'
66
+ # An explicit type is spliced into the SQL after '::', so it must be a type name as FIESTA writes one and nothing more (use
67
+ # fullmatch): a realm name, or a Postgres datatype with an optional modifier, such as integer, numeric(4,2) or character varying(64)
68
+ REGEX__dmbs_type_name = r'(?:[A-Za-z_][A-Za-z0-9_]{0,62}|character varying)(?:\(\d{1,8}(?:,\d{1,8})?\))?'
66
69
 
67
70
  # The optional request key KEY__timelines ({"<timeline>": "<ISO-8601 moment>" | null}) sets, for the request's transaction
68
71
  # only, the moment each timeline is viewed at. A moment becomes the transaction-local setting GUC__timeline_prefix + name,
@@ -236,5 +239,5 @@ ROLE__dba = "dba"
236
239
 
237
240
  PROTOCOL__postgres = "postgresql://"
238
241
 
239
- VERSION = "5.3.20"
242
+ VERSION = "5.3.22"
240
243
 
@@ -68,9 +68,19 @@ class DBInterface(ABC):
68
68
  def get_conn(self):
69
69
  pass
70
70
 
71
- def is_connection_error(self, ex) -> bool:
72
- # Overridden by the concrete interface. True when a failure means the connection/backend went
73
- # away (so nothing persisted and the operation is safe to retry on a fresh connection).
71
+ def is_connection_closed(self, conn) -> bool:
72
+ # Overridden by the concrete interface. True when the connection/backend went away, taking any
73
+ # uncommitted transaction with it (so that transaction persisted nothing)
74
+ return False
75
+
76
+ def statement_may_end_transaction(self, query: str) -> bool:
77
+ # Overridden by the concrete interface. True when running the text may end the transaction it runs
78
+ # in (a COMMIT of the request's own), so what ran before it may persist whatever happens after
79
+ return True
80
+
81
+ def has_ended_transaction(self, conn) -> bool:
82
+ # Overridden by the concrete interface. True when a transactional connection is out of its
83
+ # transaction after a statement succeeded: that statement ended it
74
84
  return False
75
85
 
76
86
  def log_warning(self, exc):
@@ -81,19 +91,25 @@ class DBInterface(ABC):
81
91
  if self.logging:
82
92
  logging.warning(exc, exc_info=False)
83
93
 
84
- def __attempt_commit_rollback(self, conn, err):
94
+ def __attempt_commit_rollback(self, conn, err, commit: bool = True):
85
95
  # Returns the commit failure rather than swallowing it. A failed commit means the transaction
86
96
  # did NOT persist, so the caller must not report success. Silently swallowing it turned lost
87
97
  # writes into a misleading downstream error - e.g. create_account's CREATE ROLE lost on a
88
98
  # connection killed by a \wipe dbms reboot, then "GRANT registered TO <role>" failing with
89
99
  # "role does not exist" on the very next statement.
100
+ # commit False (a read_only request): nothing it did may persist, so it is rolled back like a
101
+ # failed request, never left open for whoever handles the connection next to commit
90
102
  try:
91
- if err is None:
103
+ if err is None and commit:
104
+ if self.is_connection_closed(conn):
105
+ # The COMMIT cannot reach the server, so nothing persisted: a caller that owns the
106
+ # request may re-run it on a fresh connection
107
+ return ConnectionLostError("the connection was closed before COMMIT")
92
108
  self.commit(conn)
93
109
  else:
94
110
  self.rollback(conn)
95
111
  except Exception as ex:
96
- if err is None:
112
+ if err is None and commit:
97
113
  self.log_warning(ex) # commit failed - the transaction did not persist
98
114
  return ex
99
115
  else:
@@ -113,43 +129,53 @@ class DBInterface(ABC):
113
129
  self.log_warning(ex) # Serious error, connection failure to db or similar
114
130
  raise ex
115
131
 
116
- def __raise_commit_failure(self, commit_err):
117
- # A failed commit persisted nothing; surface it (instead of the old silent swallow) so the
118
- # caller cannot treat lost writes as success and hit a misleading error further on. A
119
- # connection-loss failure is raised as a retriable marker so a self-contained caller can
120
- # re-run on a fresh connection; any other commit failure is a genuine error.
121
- if self.is_connection_error(commit_err):
122
- raise ConnectionLostError(str(commit_err))
123
- translated = self.translate_commit_error(commit_err)
132
+ def __commit_failure(self, conn, commit_err, commit_error_set):
133
+ # A failed commit persisted nothing (or, lost in flight, possibly did); surface it (instead of the
134
+ # old silent swallow) so the caller cannot treat lost writes as success and hit a misleading error
135
+ # further on. A COMMIT that never reached the server is the retriable marker, so a self-contained
136
+ # caller can re-run on a fresh connection; any other commit failure is a genuine error.
137
+ if isinstance(commit_err, ConnectionLostError):
138
+ return commit_err
139
+ try:
140
+ translated = self.translate_commit_error(conn, commit_err, commit_error_set)
141
+ except Exception as ex:
142
+ # Translating reads what the database raised. Failing to must not cost the answer, nor (put_conn_handle_error) the
143
+ # connection, so the failure is answered untranslated
144
+ self.log_warning(ex)
145
+ translated = None
124
146
  if translated is not None:
125
- raise translated
126
- raise HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
127
- HTTPStatus.INTERNAL_SERVER_ERROR)
128
-
129
- def translate_commit_error(self, commit_err):
130
- # A deferred constraint trigger raises at COMMIT rather than during its statement. When what it
131
- # raises is an error the statement path would have handed back as a handled error, a subclass
132
- # returns that error here so the caller sees the same response either way; None keeps the 500
147
+ return translated
148
+ return HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
149
+ HTTPStatus.INTERNAL_SERVER_ERROR)
150
+
151
+ def translate_commit_error(self, conn, commit_err, error_set: str = None):
152
+ # A deferred constraint raises at COMMIT rather than during its statement. A subclass returns the
153
+ # error the statement path would have handed back for it, attributed to error_set (the request's
154
+ # query set, when it has only one), so the caller sees the same response either way; None keeps the 500
133
155
  return None
134
156
 
135
- def handle_error(self, conn, err, echo=ECHO__none):
157
+ def handle_error(self, conn, err, echo=ECHO__none, commit_error_set: str = None):
136
158
  commit_err = self.__attempt_commit_rollback(conn, err)
137
159
  if err is None and commit_err is not None:
138
- self.__raise_commit_failure(commit_err)
160
+ raise self.__commit_failure(conn, commit_err, commit_error_set)
139
161
  self.__err_to_exception(err, echo)
140
162
 
141
- def put_conn_handle_error(self, conn, err, echo=ECHO__none, skip_rollback_commit: bool = False):
142
- commit_err = None
143
- if not skip_rollback_commit:
144
- commit_err = self.__attempt_commit_rollback(conn, err)
145
-
163
+ def put_conn_handle_error(self, conn, err, echo=ECHO__none, skip_commit: bool = False, commit_error_set: str = None):
164
+ commit_failure = None
146
165
  try:
147
- self.put_conn(conn)
148
- except Exception as ex:
149
- self.log_warning(ex)
150
-
151
- if err is None and commit_err is not None:
152
- self.__raise_commit_failure(commit_err)
166
+ commit_err = self.__attempt_commit_rollback(conn, err, commit=not skip_commit)
167
+ # Decided before the connection goes back: from then on the putback thread is working on it
168
+ if err is None and commit_err is not None:
169
+ commit_failure = self.__commit_failure(conn, commit_err, commit_error_set)
170
+ finally:
171
+ # Whatever ending the transaction or deciding the answer raised, the connection goes back to the pool
172
+ try:
173
+ self.put_conn(conn)
174
+ except Exception as ex:
175
+ self.log_warning(ex)
176
+
177
+ if commit_failure is not None:
178
+ raise commit_failure
153
179
 
154
180
  self.__err_to_exception(err, echo)
155
181
 
@@ -1,7 +1,10 @@
1
+ import re
1
2
  import uuid
2
3
 
3
4
  import psycopg
4
5
  from psycopg import OperationalError
6
+ from psycopg.conninfo import make_conninfo
7
+ from psycopg.pq import TransactionStatus
5
8
  from psycopg_pool import ConnectionPool, PoolClosed
6
9
  import queue
7
10
  from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
@@ -12,10 +15,12 @@ from jaaql.constants import ERR__invalid_token
12
15
  from jaaql.db.db_interface import DBInterface, ECHO__none, CHAR__newline
13
16
  from jaaql.exceptions.http_status_exception import *
14
17
  from jaaql.exceptions.custom_http_status import CustomHTTPStatus
15
- from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, handled_procedure_error_from_raise
18
+ from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, DatabaseOperationalError, handled_error_from_database_error, \
19
+ database_error_descriptor
16
20
  from jaaql.constants import KEY__database
17
21
 
18
22
  ERR__connect_db = "Could not create connection to database!"
23
+ ERR__commit_outcome_unknown = "The connection was lost during COMMIT, so whether the transaction was persisted is unknown: "
19
24
 
20
25
  PGCONN__min_conns = 5
21
26
  PGCONN__max_conns = 10
@@ -111,6 +116,48 @@ def _statement_is_preparable(query: str) -> bool:
111
116
  return ";" not in query.rstrip().rstrip(";")
112
117
 
113
118
 
119
+ REGEX__word = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*")
120
+ STATEMENTS__ending_transaction = {"COMMIT", "END", "ROLLBACK", "ABORT"}
121
+
122
+
123
+ def _leading_word(query: str, i: int = 0) -> (str, int):
124
+ # The first word at or after i, upper-cased, and where it ends. Whitespace, -- comments and /* */ comments, which nest, are
125
+ # skipped as Postgres skips them
126
+ n = len(query)
127
+ while i < n:
128
+ if query[i].isspace():
129
+ i += 1
130
+ elif query.startswith("--", i):
131
+ while i < n and query[i] not in "\r\n":
132
+ i += 1
133
+ elif query.startswith("/*", i):
134
+ depth = 0
135
+ while i < n:
136
+ if query.startswith("/*", i):
137
+ depth += 1
138
+ i += 2
139
+ elif query.startswith("*/", i):
140
+ depth -= 1
141
+ i += 2
142
+ if depth == 0:
143
+ break
144
+ else:
145
+ i += 1
146
+ else:
147
+ break
148
+ match = REGEX__word.match(query, i)
149
+ return (match.group(0).upper(), match.end()) if match is not None else ("", i)
150
+
151
+
152
+ def _statement_may_end_transaction(query: str) -> bool:
153
+ # True when running the text may end the transaction it runs in, persisting what ran before it: several statements, any of
154
+ # which may (a ';' in a literal counts too, erring towards True), or one COMMIT, END, ROLLBACK, ABORT or PREPARE TRANSACTION
155
+ if not _statement_is_preparable(query):
156
+ return True
157
+ word, end = _leading_word(query)
158
+ return word in STATEMENTS__ending_transaction or (word == "PREPARE" and _leading_word(query, end)[0] == "TRANSACTION")
159
+
160
+
114
161
  def _escape_unescaped_percent(query: str) -> str:
115
162
  # psycopg3 scans every '%' in the SQL when parameters are supplied and
116
163
  # rejects anything that isn't a placeholder or '%%'. JAAQL uses named
@@ -182,28 +229,35 @@ class DBPGInterface(DBInterface):
182
229
  conn.jaaql_take_pending_auth()
183
230
  do_reset = False
184
231
  if do_reset:
185
- with conn.cursor() as cursor:
186
- cursor.execute("RESET ROLE;")
187
- did_close = False
188
- if hasattr(conn, "jaaql_reset_key"):
189
- try:
232
+ try:
233
+ if conn.info.transaction_status != TransactionStatus.IDLE:
234
+ # Whatever the request left open is not the putback's to persist (a read_only request, a caller that
235
+ # returned a connection mid-transaction): the commit below is for the reset statements alone
236
+ conn.rollback()
237
+ with conn.cursor() as cursor:
238
+ cursor.execute("RESET ROLE;")
239
+ if hasattr(conn, "jaaql_reset_key"):
190
240
  cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
191
- except:
192
- did_close = True
193
- conn.close()
194
- if not did_close:
195
241
  cursor.execute("RESET ALL;")
196
- conn.commit()
242
+ conn.commit()
243
+ except Exception:
244
+ # A connection that could not be reset must never serve another request, and must not be kept from the pool
245
+ # either: closed, putconn discards it and the pool opens a replacement
246
+ conn.close()
197
247
  DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
198
248
 
199
249
  @staticmethod
200
250
  def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
201
251
  while True:
252
+ conn, do_reset = the_queue.get()
202
253
  try:
203
- conn, do_reset = the_queue.get()
204
254
  DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
205
255
  except Exception:
206
- pass # Ignore, pool has likely been wiped
256
+ # The pool has likely been wiped or replaced. Close the connection so it cannot linger outside any pool
257
+ try:
258
+ conn.close()
259
+ except Exception:
260
+ pass
207
261
 
208
262
  def __init__(self, config, host: str, port: int, db_name: str, username: str, role: str = None, password: str = None, sub_role: str = None,
209
263
  session_settings: dict = None):
@@ -233,15 +287,17 @@ class DBPGInterface(DBInterface):
233
287
 
234
288
  if password is not None:
235
289
  try:
236
- conn_str = "user=" + username + " password=" + password + " dbname=" + db_name
290
+ conn_params = {"user": username, "password": password, "dbname": db_name}
237
291
  # Important we don't list the host as this will force a unix socket
238
292
  if host is not None and host not in ['localhost', '127.0.0.1']:
239
- conn_str += " host=" + host
293
+ conn_params["host"] = host
240
294
 
241
295
  if str(port) != "5432":
242
- conn_str += " port=" + str(port)
296
+ conn_params["port"] = str(port)
243
297
 
244
- conn_str += CONN_STR__keepalives
298
+ # make_conninfo quotes every value, so no value can add a setting of its own (a database named
299
+ # "x host=elsewhere" would otherwise send the superuser credentials to that host)
300
+ conn_str = make_conninfo(CONN_STR__keepalives.strip(), **conn_params)
245
301
 
246
302
  self.conn_str = conn_str
247
303
 
@@ -280,9 +336,9 @@ class DBPGInterface(DBInterface):
280
336
  if self.role is not None or self.sub_role is not None or self.session_settings:
281
337
  # Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
282
338
  # JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
283
- # raise are translated by _translate_session_auth_error at execution time; dead pool
284
- # connections, previously detected here, are recycled by execute_query's
285
- # OperationalError retry loop
339
+ # raise are translated by _translate_session_auth_error at execution time; a dead pool
340
+ # connection, previously detected here, surfaces at that first query, and the request
341
+ # is re-run on a fresh connection (InterpretJAAQL.transform, ConnectionLostError)
286
342
  pending = []
287
343
  if self.role is not None:
288
344
  pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
@@ -355,107 +411,102 @@ class DBPGInterface(DBInterface):
355
411
 
356
412
  def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False,
357
413
  capture_provenance: list = None):
358
- x = 0
359
- err = None
360
- while x < PGCONN__max_conns:
361
- x += 1
362
- try:
363
- cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
364
- with cursor_factory() as cursor:
365
- do_prepare = prepare and _statement_is_preparable(query)
414
+ try:
415
+ cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
416
+ with cursor_factory() as cursor:
417
+ do_prepare = prepare and _statement_is_preparable(query)
366
418
 
367
- if wait_hook:
368
- try:
369
- res, err, code = wait_hook.get(timeout=WAIT_HOOK__timeout)
370
- except queue.Empty:
371
- # The parallel verifier never delivered a verdict (its single serial thread
372
- # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
373
- # the worker unwind - without this bound the worker blocks forever holding
374
- # an open transaction, and enough of them starve the pool into nginx 504s.
375
- raise Exception(ERR__verification_timed_out)
376
- if not res:
377
- if code == 500:
378
- raise Exception(err)
379
- raise UserUnauthorized()
380
-
381
- def execute_main_query():
382
- if parameters is None or len(parameters.keys()) == 0:
383
- cursor.execute(query, prepare=do_prepare)
384
- else:
385
- cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
419
+ if wait_hook:
420
+ try:
421
+ verdict = wait_hook.get(timeout=WAIT_HOOK__timeout)
422
+ except queue.Empty:
423
+ # The parallel verifier never delivered a verdict (its single serial thread
424
+ # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
425
+ # the worker unwind - without this bound the worker blocks forever holding
426
+ # an open transaction, and enough of them starve the pool into nginx 504s.
427
+ raise Exception(ERR__verification_timed_out)
428
+ # The verdict belongs to the request, not to this attempt at it: left on the hook, a
429
+ # request re-run on a fresh connection after losing its first one reads it again
430
+ # instead of waiting out WAIT_HOOK__timeout for a second verdict that never comes
431
+ wait_hook.put(verdict)
432
+ res, err, code = verdict
433
+ if not res:
434
+ if code == 500:
435
+ raise Exception(err)
436
+ raise UserUnauthorized()
437
+
438
+ def execute_main_query():
439
+ if parameters is None or len(parameters.keys()) == 0:
440
+ cursor.execute(query, prepare=do_prepare)
441
+ else:
442
+ cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
386
443
 
387
- # Claimed only now, after the wait_hook: if verification rejects the request,
388
- # the statements remain pending and the putback thread discards them unsent
389
- pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
444
+ # Claimed only now, after the wait_hook: if verification rejects the request,
445
+ # the statements remain pending and the putback thread discards them unsent
446
+ pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
390
447
 
391
- if pending_auth:
392
- auth_cursor = conn.jaaql_raw_cursor()
393
- try:
394
- # Pipeline mode is unsafe for two kinds of statement, so restrict it to
395
- # single-command queries on transactional (non-autocommit) connections;
396
- # everything else falls back to sequential execution:
397
- # - it forces the extended protocol on every execute, which rejects a
398
- # multi-command string ("cannot insert multiple commands into a
399
- # prepared statement"); the sequential fallback runs the main query
400
- # under the simple protocol that permits several commands.
401
- # - it wraps statements in an implicit transaction, which statements
402
- # that cannot run in a transaction block (CREATE DATABASE, VACUUM,
403
- # CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
404
- # autocommit=True, so skipping the pipeline there keeps them standalone.
405
- if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
406
- with conn.pipeline():
407
- for statement in pending_auth:
408
- _execute_pending_statement(auth_cursor, statement)
409
- execute_main_query()
410
- else:
448
+ if pending_auth:
449
+ auth_cursor = conn.jaaql_raw_cursor()
450
+ try:
451
+ # Pipeline mode is unsafe for two kinds of statement, so restrict it to
452
+ # single-command queries on transactional (non-autocommit) connections;
453
+ # everything else falls back to sequential execution:
454
+ # - it forces the extended protocol on every execute, which rejects a
455
+ # multi-command string ("cannot insert multiple commands into a
456
+ # prepared statement"); the sequential fallback runs the main query
457
+ # under the simple protocol that permits several commands.
458
+ # - it wraps statements in an implicit transaction, which statements
459
+ # that cannot run in a transaction block (CREATE DATABASE, VACUUM,
460
+ # CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
461
+ # autocommit=True, so skipping the pipeline there keeps them standalone.
462
+ if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
463
+ with conn.pipeline():
411
464
  for statement in pending_auth:
412
465
  _execute_pending_statement(auth_cursor, statement)
413
466
  execute_main_query()
414
- except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
415
- translated = self._translate_session_auth_error(ex)
416
- if translated is not None:
417
- raise translated
418
- if isinstance(ex, InternalError):
419
- traceback.print_exc()
420
- raise ex
421
- finally:
422
- try:
423
- auth_cursor.close()
424
- except Exception:
425
- pass
426
- else:
427
- execute_main_query()
428
-
429
- if cursor.description is None:
430
- return [], [], []
431
- else:
432
- if capture_provenance is not None:
433
- capture_provenance.clear()
434
- pgresult = cursor.pgresult
435
- capture_provenance.extend(
436
- (pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
437
- for idx in range(pgresult.nfields))
438
- return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
439
- except OperationalError as ex:
440
- if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
441
- if isinstance(conn, JaaqlPGConnection):
442
- # This putconn bypasses the reset queue, so make sure no unsent
443
- # authorization statements ride back into the pool where check() or a
444
- # later checkout could flush them as the wrong user
445
- conn.jaaql_take_pending_auth()
446
- DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
447
- DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
448
- conn = self.get_conn()
449
- err = ex
467
+ else:
468
+ for statement in pending_auth:
469
+ _execute_pending_statement(auth_cursor, statement)
470
+ execute_main_query()
471
+ except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
472
+ translated = self._translate_session_auth_error(ex)
473
+ if translated is not None:
474
+ raise translated
475
+ if isinstance(ex, InternalError):
476
+ traceback.print_exc()
477
+ raise ex
478
+ finally:
479
+ try:
480
+ auth_cursor.close()
481
+ except Exception:
482
+ pass
450
483
  else:
451
- raise ex
452
- except Exception as ex:
453
- if self.output_query_exceptions:
454
- traceback.print_exc()
455
- raise ex
484
+ execute_main_query()
456
485
 
457
- if err:
458
- raise err
486
+ if cursor.description is None:
487
+ return [], [], []
488
+ else:
489
+ if capture_provenance is not None:
490
+ capture_provenance.clear()
491
+ pgresult = cursor.pgresult
492
+ capture_provenance.extend(
493
+ (pgresult.ftype(idx), pgresult.ftable(idx), pgresult.ftablecol(idx))
494
+ for idx in range(pgresult.nfields))
495
+ return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
496
+ except Exception as ex:
497
+ if isinstance(ex, OperationalError) and conn.closed:
498
+ # The connection is gone and the transaction on it with it. The query is not retried here on
499
+ # another connection: the caller holds this one and would go on to commit and return it, so
500
+ # the request is re-run as a whole by the owner of its connection (InterpretJAAQL.transform,
501
+ # ConnectionLostError). A database restart leaves every idle pooled connection dead, so have
502
+ # the pool replace those before that re-run checks one out
503
+ try:
504
+ DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
505
+ except Exception:
506
+ pass
507
+ if self.output_query_exceptions:
508
+ traceback.print_exc()
509
+ raise ex
459
510
 
460
511
  def commit(self, conn):
461
512
  conn.commit()
@@ -463,16 +514,28 @@ class DBPGInterface(DBInterface):
463
514
  def rollback(self, conn):
464
515
  conn.rollback()
465
516
 
466
- def is_connection_error(self, ex) -> bool:
467
- # psycopg raises OperationalError when the backend/connection went away (e.g. terminated by
468
- # \wipe dbms). The same class the execute retry loop keys off; a commit that fails this way
469
- # persisted nothing, so the operation is safe to retry on a fresh connection.
470
- return isinstance(ex, OperationalError)
517
+ def is_connection_closed(self, conn) -> bool:
518
+ # True once psycopg has seen the connection fail (backend terminated by \wipe dbms or a restart, network
519
+ # loss) or it was closed. Whatever transaction was open on it can no longer commit
520
+ return conn.closed
521
+
522
+ def statement_may_end_transaction(self, query: str) -> bool:
523
+ return _statement_may_end_transaction(query)
524
+
525
+ def has_ended_transaction(self, conn) -> bool:
526
+ return not conn.autocommit and conn.info.transaction_status == TransactionStatus.IDLE
471
527
 
472
- def translate_commit_error(self, commit_err):
528
+ def translate_commit_error(self, conn, commit_err, error_set: str = None):
473
529
  if not isinstance(commit_err, psycopg.Error):
474
530
  return None
475
- return handled_procedure_error_from_raise(commit_err)
531
+ if conn.closed:
532
+ # Lost while the COMMIT was in flight: the server may have committed before the connection went (a crash
533
+ # after the commit record was flushed, a network loss before the reply). The outcome is unknown, so this
534
+ # is reported as it stands and never re-run, which could apply the request twice
535
+ return DatabaseOperationalError(message=ERR__commit_outcome_unknown + str(commit_err), descriptor=database_error_descriptor(commit_err))
536
+ # The server refused the COMMIT (a deferred constraint, a serialization failure, ...), so the transaction
537
+ # was rolled back: the error maps exactly as it does when a statement raises it
538
+ return handled_error_from_database_error(commit_err, error_set)
476
539
 
477
540
  def handle_db_error(self, err, echo):
478
541
  if isinstance(err, ProgrammingError) and hasattr(err, 'pgresult'):
@@ -1,7 +1,8 @@
1
1
  from jaaql.exceptions.http_status_exception import HttpStatusException, HttpSingletonStatusException, ConnectionLostError
2
2
  from jaaql.interpreter.interpret_jaaql import InterpretJAAQL
3
- from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials
3
+ from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials, REGEX__dmbs_object_name
4
4
  from typing import Union
5
+ import re
5
6
  import jaaql.utilities.crypt_utils as crypt_utils
6
7
  from jaaql.db.db_pg_interface import DBPGInterface
7
8
  from jaaql.db.db_interface import DBInterface
@@ -15,6 +16,7 @@ ERR__duplicated_encryption_salt = "Duplicated value in encryption_salts list"
15
16
  ERR__expected_single_row = "Expected single row response but received '%d' rows"
16
17
  ERR__unsupported_interface = "Unsupported interface '%s'. We only support %s"
17
18
  ERR__schema_invalid = "Schema invalid!"
19
+ ERR__database_name_unsafe = "Unsafe database name"
18
20
 
19
21
  # A lost connection persisted nothing, so a self-contained statement is re-run on a fresh connection.
20
22
  # Bounded so a genuinely-down database still surfaces an error rather than looping.
@@ -194,6 +196,13 @@ def execute_supplied_statements(db_interface, queries: Union[str, list],
194
196
  return data
195
197
 
196
198
 
199
+ def requested_database(database) -> str:
200
+ # A database named by the request itself: a plain name, as every database JAAQL uses has one
201
+ if not isinstance(database, str) or re.fullmatch(REGEX__dmbs_object_name, database) is None:
202
+ raise HttpStatusException(ERR__database_name_unsafe)
203
+ return database
204
+
205
+
197
206
  def create_interface_for_db(vault, config, user_id: str, database: str, sub_role: str = None, session_settings: dict = None):
198
207
  jaaql_uri = vault.get_obj(VAULT_KEY__super_db_credentials)
199
208
  address, port, _, username, password = DBInterface.fracture_uri(jaaql_uri)
@@ -4,7 +4,8 @@ from http import HTTPStatus
4
4
  from jaaql.mvc.exception_queries import QUERY__fetch_application_schemas, KG__application_schema__application, KG__application__is_live, \
5
5
  KG__application_schema__name, KEY__is_default
6
6
  from queue import Queue
7
- from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts
7
+ from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts, \
8
+ requested_database
8
9
  from jaaql.exceptions.http_status_exception import HttpStatusException, ConnectionLostError
9
10
  from jaaql.interpreter.interpret_jaaql import InterpretJAAQL, KEY_autocommit
10
11
  from jaaql.constants import KEY__application, KEY__database, KEY__schema, KEY__role, DB__jaaql, \
@@ -122,6 +123,8 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
122
123
  raise HttpStatusException(ERR__schema_invalid)
123
124
 
124
125
  inputs[KEY__database] = found_db
126
+ elif KEY__database in inputs:
127
+ requested_database(inputs[KEY__database])
125
128
 
126
129
  if KEY__database not in inputs:
127
130
  inputs[KEY__database] = DB__jaaql
@@ -36,11 +36,12 @@ class HttpSingletonStatusException(HttpStatusException):
36
36
 
37
37
  class ConnectionLostError(HttpStatusException):
38
38
  # Raised when a database operation failed because its connection was lost (e.g. the backend was
39
- # terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed. Nothing persisted,
40
- # so a self-contained operation may safely be retried on a fresh connection. Subclasses
41
- # HttpStatusException so that if retries are exhausted it still surfaces as a clean 500.
39
+ # terminated by \wipe dbms's pg_terminate_backend) BEFORE anything committed: during a statement of
40
+ # a transaction, or before its COMMIT was sent. Nothing persisted, so a self-contained operation may
41
+ # safely be retried on a fresh connection. Subclasses HttpStatusException so that if retries are
42
+ # exhausted it still surfaces as a clean 500.
42
43
  def __init__(self, message: str):
43
- super().__init__("Commit failed, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
44
+ super().__init__("Connection lost, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
44
45
 
45
46
 
46
47
  class JaaqlInterpretableHandledError(Exception):