jaaql-middleware-python 5.2.7__tar.gz → 5.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {jaaql_middleware_python-5.2.7/jaaql_middleware_python.egg-info → jaaql_middleware_python-5.3.2}/PKG-INFO +6 -5
  2. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/constants.py +8 -1
  3. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/db/db_interface.py +3 -3
  4. jaaql_middleware_python-5.3.2/jaaql/db/db_pg_interface.py +404 -0
  5. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/db/db_utils_no_circ.py +2 -2
  6. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/documentation/documentation_internal.py +34 -23
  7. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/interpreter/interpret_jaaql.py +65 -20
  8. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/jaaql.py +13 -0
  9. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/base_controller.py +49 -19
  10. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/base_model.py +38 -12
  11. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/controller.py +10 -4
  12. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/model.py +62 -12
  13. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2/jaaql_middleware_python.egg-info}/PKG-INFO +6 -5
  14. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql_middleware_python.egg-info/requires.txt +5 -4
  15. jaaql_middleware_python-5.2.7/jaaql/db/db_pg_interface.py +0 -281
  16. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/LICENSE.txt +0 -0
  17. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/README.md +0 -0
  18. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/__init__.py +0 -0
  19. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/config/__init__.py +0 -0
  20. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/config/config-docker.ini +0 -0
  21. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/config/config-test.ini +0 -0
  22. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/config/config.ini +0 -0
  23. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/config_constants.py +0 -0
  24. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/db/__init__.py +0 -0
  25. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/db/db_utils.py +0 -0
  26. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/documentation/__init__.py +0 -0
  27. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/documentation/documentation_public.py +0 -0
  28. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/documentation/documentation_shared.py +0 -0
  29. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/email/__init__.py +0 -0
  30. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/email/email_manager.py +0 -0
  31. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/email/email_manager_service.py +0 -0
  32. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/email/patch_ems.py +0 -0
  33. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/exceptions/__init__.py +0 -0
  34. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/exceptions/custom_http_status.py +0 -0
  35. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/exceptions/http_status_exception.py +0 -0
  36. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/exceptions/jaaql_interpretable_handled_errors.py +0 -0
  37. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/exceptions/not_yet_implement_exception.py +0 -0
  38. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/generated_constants.py +0 -0
  39. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/interpreter/__init__.py +0 -0
  40. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/migrations/__init__.py +0 -0
  41. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/migrations/migrations.py +0 -0
  42. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/__init__.py +0 -0
  43. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/controller_interface.py +0 -0
  44. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/exception_queries.py +0 -0
  45. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/generated_queries.py +0 -0
  46. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/handmade_queries.py +0 -0
  47. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/model_interface.py +0 -0
  48. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/mvc/response.py +0 -0
  49. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/openapi/__init__.py +0 -0
  50. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/openapi/swagger_documentation.py +0 -0
  51. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/patch.py +0 -0
  52. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/01.install_domains.generated.sql +0 -0
  53. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/02.install_super_user.exceptions.sql +0 -0
  54. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/03.install_super_user.handwritten.sql +0 -0
  55. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/04.install_jaaql_data_structures.generated.sql +0 -0
  56. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/05.install_static_data.generated.sql +0 -0
  57. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/06.install_jaaql.exceptions.sql +0 -0
  58. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/ZZZZ.generated_functions_views_and_permissions.sql +0 -0
  59. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/ZZZZ.reset_references.sql +0 -0
  60. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/scripts/swagger_template.html +0 -0
  61. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/__init__.py +0 -0
  62. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/cached_canned_query_service.py +0 -0
  63. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/migrations_manager_service.py +0 -0
  64. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/patch_mms.py +0 -0
  65. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/patch_shared_var_service.py +0 -0
  66. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/services/shared_var_service.py +0 -0
  67. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/__init__.py +0 -0
  68. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/bootstrap_secrets.py +0 -0
  69. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/cron.py +0 -0
  70. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/crypt_utils.py +0 -0
  71. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/options.py +0 -0
  72. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/utils.py +0 -0
  73. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/utils_no_project_imports.py +0 -0
  74. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql/utilities/vault.py +0 -0
  75. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql_middleware_python.egg-info/SOURCES.txt +0 -0
  76. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql_middleware_python.egg-info/dependency_links.txt +0 -0
  77. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/jaaql_middleware_python.egg-info/top_level.txt +0 -0
  78. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/setup.cfg +0 -0
  79. {jaaql_middleware_python-5.2.7 → jaaql_middleware_python-5.3.2}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jaaql-middleware-python
3
- Version: 5.2.7
3
+ Version: 5.3.2
4
4
  Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
5
5
  Home-page: https://github.com/JAAQL/JAAQL-middleware-python
6
6
  Author: Software Quality Measurement and Improvement bv
@@ -10,19 +10,20 @@ Description-Content-Type: text/markdown
10
10
  License-File: LICENSE.txt
11
11
  Requires-Dist: jaaql-monitor~=1.6.3
12
12
  Requires-Dist: psycopg[binary]~=3.2.13
13
- Requires-Dist: Pillow~=11.2.1
13
+ Requires-Dist: Pillow~=11.3.0
14
14
  Requires-Dist: cryptography~=44.0.2
15
15
  Requires-Dist: flask~=3.1.0
16
- Requires-Dist: coverage~=7.4.3
16
+ Requires-Dist: orjson~=3.11.9
17
+ Requires-Dist: coverage~=7.15.2
17
18
  Requires-Dist: psycopg-pool~=3.2.1
18
19
  Requires-Dist: pyjwt~=2.10.1
19
- Requires-Dist: pyyaml~=6.0.1
20
+ Requires-Dist: pyyaml~=6.0.3
20
21
  Requires-Dist: werkzeug~=3.1.3
21
22
  Requires-Dist: argon2-cffi~=23.1.0
22
23
  Requires-Dist: pyotp~=2.9.0
23
24
  Requires-Dist: qrcode~=7.4.2
24
25
  Requires-Dist: gunicorn~=23.0.0
25
- Requires-Dist: gevent~=24.11.1
26
+ Requires-Dist: gevent~=25.9.1
26
27
  Requires-Dist: requests~=2.32.3
27
28
  Requires-Dist: pyzbar~=0.1.9
28
29
  Requires-Dist: parsys-requests-unixsocket~=0.3.2
@@ -187,6 +187,13 @@ ENDPOINT__internal_applications = "/internal/applications"
187
187
  ENDPOINT__internal_templates = "/internal/emails/templates"
188
188
  ENDPOINT__internal_accounts = "/internal/emails/accounts"
189
189
  ENDPOINT__is_alive = "/internal/is-alive"
190
+ ENDPOINT__deep_health = "/internal/deep-health"
191
+
192
+ # Synthetic query the BATON microcompiler injects into every queries.json so
193
+ # deep_health() can resolve a query through the cache. Must match the microcompiler
194
+ # constant HEALTH_QUERY_CACHE_KEY (index is always 0).
195
+ QUERY_CACHE_KEY__health = "__health__"
196
+ QUERY_CACHE_REF__health = QUERY_CACHE_KEY__health + ":0"
190
197
  ENDPOINT__report_sentinel_error = "/sentinel/reporting/error"
191
198
  ENDPOINT__install = "/internal/install"
192
199
  ENDPOINT__set_shared_var = "/set-shared-var"
@@ -211,5 +218,5 @@ ROLE__dba = "dba"
211
218
 
212
219
  PROTOCOL__postgres = "postgresql://"
213
220
 
214
- VERSION = "5.2.7"
221
+ VERSION = "5.3.2"
215
222
 
@@ -125,7 +125,7 @@ class DBInterface(ABC):
125
125
  pass
126
126
 
127
127
  def execute_query_fetching_results(self, conn, query, parameters=None, echo=ECHO__none, as_objects=False, wait_hook: queue.Queue = None,
128
- requires_dba_check: bool = False):
128
+ requires_dba_check: bool = False, prepare: bool = False):
129
129
  if echo not in ECHO__allowed:
130
130
  allowed_echoes = ", ".join([str(allowed_echo) for allowed_echo in ECHO__allowed])
131
131
  raise HttpStatusException(ERR__unknown_echo % (str(echo), allowed_echoes), HTTPStatus.BAD_REQUEST)
@@ -149,7 +149,7 @@ class DBInterface(ABC):
149
149
  if wait_hook:
150
150
  wait_hook = None
151
151
 
152
- columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook)
152
+ columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook, prepare=prepare)
153
153
 
154
154
  ret = {
155
155
  RET__columns: columns,
@@ -225,7 +225,7 @@ class DBInterface(ABC):
225
225
  pass
226
226
 
227
227
  @abstractmethod
228
- def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None):
228
+ def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None, prepare: bool = False):
229
229
  pass
230
230
 
231
231
  @abstractmethod
@@ -0,0 +1,404 @@
1
+ import uuid
2
+
3
+ import psycopg
4
+ from psycopg import OperationalError
5
+ from psycopg_pool import ConnectionPool, PoolClosed
6
+ import queue
7
+ from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
8
+ import threading
9
+ import traceback
10
+ from jaaql.constants import ERR__invalid_token
11
+
12
+ from jaaql.db.db_interface import DBInterface, ECHO__none, CHAR__newline
13
+ from jaaql.exceptions.http_status_exception import *
14
+ from jaaql.exceptions.custom_http_status import CustomHTTPStatus
15
+ from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized
16
+ from jaaql.constants import KEY__database
17
+
18
+ ERR__connect_db = "Could not create connection to database!"
19
+
20
+ PGCONN__min_conns = 5
21
+ PGCONN__max_conns = 10
22
+
23
+ TIMEOUT = 2.5
24
+
25
+ ERR__invalid_role = "Role not allowed, invalid format!"
26
+ ERR__must_use_canned_query = "Must use canned query as you are not an admin!"
27
+
28
+ QUERY__dba_query = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = %(database)s;"
29
+ QUERY__dba_query_external = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = current_database();"
30
+
31
+ try:
32
+ # Pipeline mode needs libpq >= 14. create_app refuses to boot when this is False so a server
33
+ # can never silently degrade; the sequential fallback in execute_query exists only for
34
+ # non-server tooling that imports this module directly
35
+ PIPELINE_SUPPORTED = psycopg.Pipeline.is_supported()
36
+ except Exception:
37
+ PIPELINE_SUPPORTED = False
38
+
39
+
40
+ class JaaqlPGConnection(psycopg.Connection):
41
+ """
42
+ Connection whose per-checkout session-authorization statements (jaaql__set_session_authorization
43
+ / SET ROLE) are deferred at checkout and sent pipelined with the first real query, saving a
44
+ network round trip per checkout.
45
+
46
+ Safety property: authorization must never be skippable. Any cursor obtained through the normal
47
+ cursor() call while statements are still pending executes them eagerly first, so raw connection
48
+ usage cannot run as the pool user. execute_query claims the statements via
49
+ jaaql_take_pending_auth and is then responsible for sending them before (or batched with) the
50
+ query it executes.
51
+ """
52
+
53
+ _jaaql_pending_auth = None
54
+ _jaaql_flushing = False
55
+
56
+ def jaaql_set_pending_auth(self, statements: list):
57
+ self._jaaql_pending_auth = statements
58
+
59
+ def jaaql_has_pending_auth(self) -> bool:
60
+ return self._jaaql_pending_auth is not None
61
+
62
+ def jaaql_take_pending_auth(self):
63
+ pending, self._jaaql_pending_auth = self._jaaql_pending_auth, None
64
+ return pending
65
+
66
+ def jaaql_raw_cursor(self, *args, **kwargs):
67
+ # Bypasses the pending-authorization flush; the caller has taken responsibility for
68
+ # executing the pending statements itself
69
+ return super().cursor(*args, **kwargs)
70
+
71
+ def cursor(self, *args, **kwargs):
72
+ if self._jaaql_pending_auth is not None and not self._jaaql_flushing:
73
+ self._jaaql_flushing = True
74
+ try:
75
+ pending = self.jaaql_take_pending_auth()
76
+ with super().cursor() as flush_cursor:
77
+ for statement in pending:
78
+ flush_cursor.execute(statement)
79
+ finally:
80
+ self._jaaql_flushing = False
81
+ return super().cursor(*args, **kwargs)
82
+
83
+
84
+ def _statement_is_preparable(query: str) -> bool:
85
+ # Server-side PREPARE accepts a single statement only. A ';' anywhere except trailing means the
86
+ # text may hold multiple statements (a ';' inside a string literal merely skips the
87
+ # optimisation, which is the safe direction)
88
+ return ";" not in query.rstrip().rstrip(";")
89
+
90
+
91
+ def _escape_unescaped_percent(query: str) -> str:
92
+ # psycopg3 scans every '%' in the SQL when parameters are supplied and
93
+ # rejects anything that isn't a placeholder or '%%'. JAAQL uses named
94
+ # parameters only (:parameter -> %(name)s), so positional %s/%b/%t must
95
+ # never survive: any '%s' in author SQL is a literal inside a string
96
+ # (LIKE '%saas%', LIKE '%twynstra%', etc.), not a placeholder. Escape
97
+ # everything except '%%' and '%(name)X'.
98
+ out = []
99
+ i = 0
100
+ n = len(query)
101
+ while i < n:
102
+ if query[i] != '%':
103
+ out.append(query[i])
104
+ i += 1
105
+ continue
106
+
107
+ nxt = query[i + 1] if i + 1 < n else ''
108
+ if nxt == '%':
109
+ out.append('%%')
110
+ i += 2
111
+ elif nxt == '(':
112
+ close = query.find(')', i + 2)
113
+ if close == -1 or close + 1 >= n:
114
+ out.append('%%')
115
+ i += 1
116
+ else:
117
+ out.append(query[i:close + 2])
118
+ i = close + 2
119
+ else:
120
+ out.append('%%')
121
+ i += 1
122
+ return ''.join(out)
123
+
124
+
125
+ class DBPGInterface(DBInterface):
126
+
127
+ HOST_POOLS = {}
128
+ HOST_POOLS_QUEUES = {}
129
+
130
+ @staticmethod
131
+ def close_all_pools():
132
+ for _, user_pool_dict in DBPGInterface.HOST_POOLS.items():
133
+ for _, pool in user_pool_dict.items():
134
+ pool.close()
135
+ DBPGInterface.HOST_POOLS = {}
136
+ DBPGInterface.HOST_POOLS_QUEUES = {}
137
+
138
+ @staticmethod
139
+ def _process_returned_conn(username: str, db_name: str, conn, do_reset: bool):
140
+ if isinstance(conn, JaaqlPGConnection) and conn.jaaql_has_pending_auth():
141
+ # The deferred authorization statements were never sent, so the session still runs as
142
+ # the pool user and there is nothing to reset. Clearing them here also stops the
143
+ # cursor() below from flushing them
144
+ conn.jaaql_take_pending_auth()
145
+ do_reset = False
146
+ if do_reset:
147
+ with conn.cursor() as cursor:
148
+ cursor.execute("RESET ROLE;")
149
+ did_close = False
150
+ if hasattr(conn, "jaaql_reset_key"):
151
+ try:
152
+ cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
153
+ except:
154
+ did_close = True
155
+ conn.close()
156
+ if not did_close:
157
+ cursor.execute("RESET ALL;")
158
+ conn.commit()
159
+ DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
160
+
161
+ @staticmethod
162
+ def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
163
+ while True:
164
+ try:
165
+ conn, do_reset = the_queue.get()
166
+ DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
167
+ except Exception:
168
+ pass # Ignore, pool has likely been wiped
169
+
170
+ def __init__(self, config, host: str, port: int, db_name: str, username: str, role: str = None, password: str = None, sub_role: str = None):
171
+ super().__init__(config, host, username)
172
+
173
+ self.role = role
174
+ self.sub_role = sub_role
175
+ if sub_role is not None:
176
+ if len([ch for ch in sub_role if not ch.isalnum() and ch not in ['_', '-']]) != 0:
177
+ raise HttpStatusException(ERR__invalid_role, HTTPStatus.UNAUTHORIZED)
178
+
179
+ self.output_query_exceptions = config["DEBUG"]["output_query_exceptions"].lower() == "true"
180
+ self.username = username
181
+ self.db_name = db_name
182
+
183
+ if self.username not in DBPGInterface.HOST_POOLS:
184
+ DBPGInterface.HOST_POOLS[self.username] = {}
185
+ DBPGInterface.HOST_POOLS_QUEUES[self.username] = {}
186
+
187
+ user_pool = DBPGInterface.HOST_POOLS[self.username]
188
+
189
+ self.conn_str = None
190
+ the_pool = None
191
+
192
+ if password is not None:
193
+ try:
194
+ conn_str = "user=" + username + " password=" + password + " dbname=" + db_name
195
+ # Important we don't list the host as this will force a unix socket
196
+ if host is not None and host not in ['localhost', '127.0.0.1']:
197
+ conn_str += " host=" + host
198
+
199
+ if str(port) != "5432":
200
+ conn_str += " port=" + str(port)
201
+
202
+ self.conn_str = conn_str
203
+
204
+ if self.db_name not in user_pool:
205
+ the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30,
206
+ connection_class=JaaqlPGConnection)
207
+ the_pool.getconn(timeout=TIMEOUT)
208
+ user_pool[self.db_name] = the_pool
209
+ the_queue = queue.Queue()
210
+ DBPGInterface.HOST_POOLS_QUEUES[self.username][self.db_name] = the_queue
211
+ threading.Thread(target=DBPGInterface.put_conn_threaded, args=[self.username, self.db_name, the_queue], daemon=True).start()
212
+ except OperationalError as ex:
213
+ if the_pool is not None:
214
+ the_pool.close()
215
+ if "does not exist" in str(ex).split("\"")[-1] or "couldn't get a connection after" in str(ex):
216
+ raise HttpStatusException("Database \"" + self.db_name + "\" does not exist",
217
+ CustomHTTPStatus.DATABASE_NO_EXIST)
218
+ else:
219
+ raise HttpStatusException(str(ex))
220
+
221
+ def _get_conn(self):
222
+ conn = DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
223
+ conn.jaaql_reset_key = str(uuid.uuid4())
224
+ if isinstance(conn, JaaqlPGConnection):
225
+ # Discard any pending authorization left by a previous checkout that was returned
226
+ # without executing (defensive: every return path should already have cleared it).
227
+ # Without this a later checkout could execute a previous user's authorization
228
+ conn.jaaql_take_pending_auth()
229
+ if self.role is not None or self.sub_role is not None:
230
+ # Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
231
+ # JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
232
+ # raise are translated by _translate_session_auth_error at execution time; dead pool
233
+ # connections, previously detected here, are recycled by execute_query's
234
+ # OperationalError retry loop
235
+ pending = []
236
+ if self.role is not None:
237
+ pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
238
+ if self.sub_role is not None:
239
+ pending.append("SET ROLE \"" + self.sub_role + "\"")
240
+ conn.jaaql_set_pending_auth(pending)
241
+ return conn
242
+
243
+ def get_pool(self):
244
+ if self.db_name not in DBPGInterface.HOST_POOLS[self.username]:
245
+ DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
246
+ max_size=PGCONN__max_conns, max_lifetime=60 * 30,
247
+ connection_class=JaaqlPGConnection)
248
+ return DBPGInterface.HOST_POOLS[self.username][self.db_name]
249
+
250
+ def get_conn(self, retry_closed_pool: bool = True):
251
+ try:
252
+ conn = self._get_conn()
253
+ except PoolClosed as ex:
254
+ # Jaaql has likely been reinstalled, this pool has been forcibly closed
255
+ if retry_closed_pool:
256
+ DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
257
+ max_size=PGCONN__max_conns, max_lifetime=60 * 30,
258
+ connection_class=JaaqlPGConnection)
259
+ conn = self.get_conn(retry_closed_pool=False)
260
+ if not conn:
261
+ raise ex
262
+ else:
263
+ return False
264
+ except HttpStatusException as ex:
265
+ raise ex
266
+ except Exception:
267
+ traceback.print_exc()
268
+ raise HttpStatusException(ERR__connect_db, HTTPStatus.INTERNAL_SERVER_ERROR)
269
+
270
+ return conn
271
+
272
+ def put_conn(self, conn):
273
+ DBPGInterface.HOST_POOLS_QUEUES[self.username][self.db_name].put((conn, self.role is not None))
274
+
275
+ def close(self):
276
+ DBPGInterface.HOST_POOLS[self.username].pop(self.db_name).close()
277
+ DBPGInterface.HOST_POOLS_QUEUES[self.username].pop(self.db_name)
278
+
279
+ def check_dba(self, conn, wait_hook: queue.Queue = None):
280
+ columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook, prepare=True)
281
+ if not rows[0]:
282
+ raise HttpStatusException(ERR__must_use_canned_query)
283
+
284
+ def _translate_session_auth_error(self, ex):
285
+ """
286
+ The deferred session-authorization statements execute batched with the first query, so
287
+ their errors surface during execute_query rather than at checkout. This translates them to
288
+ the same exceptions the eager checkout path used to raise. Returns None when the error is
289
+ not recognisably from the authorization statements (the caller re-raises it untouched, so
290
+ a query's own error keeps its normal handling)
291
+ """
292
+ if isinstance(ex, InternalError):
293
+ if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
294
+ return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
295
+ if isinstance(ex, UndefinedFunction) and "jaaql__set_session_authorization" in str(ex):
296
+ return HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
297
+ if isinstance(ex, InvalidParameterValue) and "role" in str(ex).lower():
298
+ return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
299
+ return None
300
+
301
+ def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False):
302
+ x = 0
303
+ err = None
304
+ while x < PGCONN__max_conns:
305
+ x += 1
306
+ try:
307
+ cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
308
+ with cursor_factory() as cursor:
309
+ do_prepare = prepare and _statement_is_preparable(query)
310
+
311
+ if wait_hook:
312
+ res, err, code = wait_hook.get()
313
+ if not res:
314
+ if code == 500:
315
+ raise Exception(err)
316
+ raise UserUnauthorized()
317
+
318
+ def execute_main_query():
319
+ if parameters is None or len(parameters.keys()) == 0:
320
+ cursor.execute(query, prepare=do_prepare)
321
+ else:
322
+ cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
323
+
324
+ # Claimed only now, after the wait_hook: if verification rejects the request,
325
+ # the statements remain pending and the putback thread discards them unsent
326
+ pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
327
+
328
+ if pending_auth:
329
+ auth_cursor = conn.jaaql_raw_cursor()
330
+ try:
331
+ # Pipeline mode is unsafe for two kinds of statement, so restrict it to
332
+ # single-command queries on transactional (non-autocommit) connections;
333
+ # everything else falls back to sequential execution:
334
+ # - it forces the extended protocol on every execute, which rejects a
335
+ # multi-command string ("cannot insert multiple commands into a
336
+ # prepared statement"); the sequential fallback runs the main query
337
+ # under the simple protocol that permits several commands.
338
+ # - it wraps statements in an implicit transaction, which statements
339
+ # that cannot run in a transaction block (CREATE DATABASE, VACUUM,
340
+ # CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
341
+ # autocommit=True, so skipping the pipeline there keeps them standalone.
342
+ if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
343
+ with conn.pipeline():
344
+ for statement in pending_auth:
345
+ auth_cursor.execute(statement)
346
+ execute_main_query()
347
+ else:
348
+ for statement in pending_auth:
349
+ auth_cursor.execute(statement)
350
+ execute_main_query()
351
+ except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
352
+ translated = self._translate_session_auth_error(ex)
353
+ if translated is not None:
354
+ raise translated
355
+ if isinstance(ex, InternalError):
356
+ traceback.print_exc()
357
+ raise ex
358
+ finally:
359
+ try:
360
+ auth_cursor.close()
361
+ except Exception:
362
+ pass
363
+ else:
364
+ execute_main_query()
365
+
366
+ if cursor.description is None:
367
+ return [], [], []
368
+ else:
369
+ return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
370
+ except OperationalError as ex:
371
+ if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
372
+ if isinstance(conn, JaaqlPGConnection):
373
+ # This putconn bypasses the reset queue, so make sure no unsent
374
+ # authorization statements ride back into the pool where check() or a
375
+ # later checkout could flush them as the wrong user
376
+ conn.jaaql_take_pending_auth()
377
+ DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
378
+ DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
379
+ conn = self.get_conn()
380
+ err = ex
381
+ else:
382
+ raise ex
383
+ except Exception as ex:
384
+ if self.output_query_exceptions:
385
+ traceback.print_exc()
386
+ raise ex
387
+
388
+ if err:
389
+ raise err
390
+
391
+ def commit(self, conn):
392
+ conn.commit()
393
+
394
+ def rollback(self, conn):
395
+ conn.rollback()
396
+
397
+ def handle_db_error(self, err, echo):
398
+ if isinstance(err, ProgrammingError) and hasattr(err, 'pgresult'):
399
+ err = err.pgresult.error_message.decode("UTF-8")
400
+
401
+ err = str(err)
402
+ if echo != ECHO__none:
403
+ err += CHAR__newline + echo
404
+ raise HttpStatusException(err, HTTPStatus.UNPROCESSABLE_ENTITY)
@@ -68,7 +68,7 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
68
68
 
69
69
  def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
70
70
  cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
71
- conn=None, interface: DBInterface = None, db_cache=None):
71
+ conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False):
72
72
  if not isinstance(inputs, dict):
73
73
  raise HttpStatusException("Expected object or string input")
74
74
 
@@ -80,7 +80,7 @@ def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: d
80
80
  ).transform(inputs, skip_commit=inputs.get(KEY__read_only), wait_hook=verification_hook,
81
81
  encryption_key=db_crypt_key, conn=conn,
82
82
  canned_query_service=cached_canned_query_service, prevent_unused_parameters=prevent_unused,
83
- and_return_connection_mid_transaction=keep_alive_conn)
83
+ and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements)
84
84
 
85
85
  if as_objects:
86
86
  ret = objectify(ret, singleton=singleton)
@@ -108,6 +108,17 @@ DOCUMENTATION__is_alive = SwaggerDocumentation(
108
108
  )
109
109
  )
110
110
 
111
+ DOCUMENTATION__deep_health = SwaggerDocumentation(
112
+ security=False,
113
+ tags="Deep Health",
114
+ methods=SwaggerMethod(
115
+ name="Deep health check",
116
+ description="Resolves a query through the compiled query cache and runs it, so a stale, partial or "
117
+ "unloaded cache fails the check (unlike is-alive, which only pings the database)",
118
+ method=REST__GET
119
+ )
120
+ )
121
+
111
122
  DOCUMENTATION__clean = SwaggerDocumentation(
112
123
  tags="Installation",
113
124
  methods=SwaggerMethod(
@@ -605,26 +616,26 @@ DOCUMENTATION__security_event = SwaggerDocumentation(
605
616
  ],
606
617
  response=RES__allow_all
607
618
  )
608
- )
609
-
610
- DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
611
- tags="Reporting",
612
- security=False, # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
613
- methods=SwaggerMethod(
614
- name="Report error",
615
- description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
616
- "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
617
- method=REST__POST,
618
- body=[
619
- SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
620
- SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
621
- SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
622
- SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
623
- SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
624
- SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
625
- SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
626
- SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
627
- SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
628
- ]
629
- )
630
- )
619
+ )
620
+
621
+ DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
622
+ tags="Reporting",
623
+ security=False, # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
624
+ methods=SwaggerMethod(
625
+ name="Report error",
626
+ description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
627
+ "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
628
+ method=REST__POST,
629
+ body=[
630
+ SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
631
+ SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
632
+ SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
633
+ SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
634
+ SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
635
+ SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
636
+ SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
637
+ SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
638
+ SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
639
+ ]
640
+ )
641
+ )