jaaql-middleware-python 5.1.0__tar.gz → 5.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jaaql_middleware_python-5.1.0/jaaql_middleware_python.egg-info → jaaql_middleware_python-5.3.1}/PKG-INFO +6 -5
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/constants.py +9 -1
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/db/db_interface.py +3 -3
- jaaql_middleware_python-5.3.1/jaaql/db/db_pg_interface.py +400 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/db/db_utils_no_circ.py +2 -2
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/documentation/documentation_internal.py +34 -1
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/interpreter/interpret_jaaql.py +65 -20
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/jaaql.py +13 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/base_controller.py +49 -19
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/base_model.py +38 -12
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/controller.py +39 -2
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/model.py +240 -62
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/03.install_super_user.handwritten.sql +17 -7
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/migrations_manager_service.py +50 -2
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1/jaaql_middleware_python.egg-info}/PKG-INFO +6 -5
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql_middleware_python.egg-info/requires.txt +5 -4
- jaaql_middleware_python-5.1.0/jaaql/db/db_pg_interface.py +0 -247
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/LICENSE.txt +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/README.md +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/config/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/config/config-docker.ini +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/config/config-test.ini +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/config/config.ini +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/config_constants.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/db/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/db/db_utils.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/documentation/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/documentation/documentation_public.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/documentation/documentation_shared.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/email/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/email/email_manager.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/email/email_manager_service.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/email/patch_ems.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/exceptions/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/exceptions/custom_http_status.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/exceptions/http_status_exception.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/exceptions/jaaql_interpretable_handled_errors.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/exceptions/not_yet_implement_exception.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/generated_constants.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/interpreter/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/migrations/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/migrations/migrations.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/controller_interface.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/exception_queries.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/generated_queries.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/handmade_queries.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/model_interface.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/mvc/response.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/openapi/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/openapi/swagger_documentation.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/patch.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/01.install_domains.generated.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/02.install_super_user.exceptions.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/04.install_jaaql_data_structures.generated.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/05.install_static_data.generated.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/06.install_jaaql.exceptions.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/ZZZZ.generated_functions_views_and_permissions.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/ZZZZ.reset_references.sql +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/scripts/swagger_template.html +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/cached_canned_query_service.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/patch_mms.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/patch_shared_var_service.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/services/shared_var_service.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/__init__.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/bootstrap_secrets.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/cron.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/crypt_utils.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/options.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/utils.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/utils_no_project_imports.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/utilities/vault.py +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql_middleware_python.egg-info/SOURCES.txt +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql_middleware_python.egg-info/dependency_links.txt +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql_middleware_python.egg-info/top_level.txt +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/setup.cfg +0 -0
- {jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: jaaql-middleware-python
|
|
3
|
-
Version: 5.1
|
|
3
|
+
Version: 5.3.1
|
|
4
4
|
Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
|
|
5
5
|
Home-page: https://github.com/JAAQL/JAAQL-middleware-python
|
|
6
6
|
Author: Software Quality Measurement and Improvement bv
|
|
@@ -10,19 +10,20 @@ Description-Content-Type: text/markdown
|
|
|
10
10
|
License-File: LICENSE.txt
|
|
11
11
|
Requires-Dist: jaaql-monitor~=1.6.3
|
|
12
12
|
Requires-Dist: psycopg[binary]~=3.2.13
|
|
13
|
-
Requires-Dist: Pillow~=11.
|
|
13
|
+
Requires-Dist: Pillow~=11.3.0
|
|
14
14
|
Requires-Dist: cryptography~=44.0.2
|
|
15
15
|
Requires-Dist: flask~=3.1.0
|
|
16
|
-
Requires-Dist:
|
|
16
|
+
Requires-Dist: orjson~=3.11.9
|
|
17
|
+
Requires-Dist: coverage~=7.15.2
|
|
17
18
|
Requires-Dist: psycopg-pool~=3.2.1
|
|
18
19
|
Requires-Dist: pyjwt~=2.10.1
|
|
19
|
-
Requires-Dist: pyyaml~=6.0.
|
|
20
|
+
Requires-Dist: pyyaml~=6.0.3
|
|
20
21
|
Requires-Dist: werkzeug~=3.1.3
|
|
21
22
|
Requires-Dist: argon2-cffi~=23.1.0
|
|
22
23
|
Requires-Dist: pyotp~=2.9.0
|
|
23
24
|
Requires-Dist: qrcode~=7.4.2
|
|
24
25
|
Requires-Dist: gunicorn~=23.0.0
|
|
25
|
-
Requires-Dist: gevent~=
|
|
26
|
+
Requires-Dist: gevent~=25.9.1
|
|
26
27
|
Requires-Dist: requests~=2.32.3
|
|
27
28
|
Requires-Dist: pyzbar~=0.1.9
|
|
28
29
|
Requires-Dist: parsys-requests-unixsocket~=0.3.2
|
|
@@ -187,6 +187,13 @@ ENDPOINT__internal_applications = "/internal/applications"
|
|
|
187
187
|
ENDPOINT__internal_templates = "/internal/emails/templates"
|
|
188
188
|
ENDPOINT__internal_accounts = "/internal/emails/accounts"
|
|
189
189
|
ENDPOINT__is_alive = "/internal/is-alive"
|
|
190
|
+
ENDPOINT__deep_health = "/internal/deep-health"
|
|
191
|
+
|
|
192
|
+
# Synthetic query the BATON microcompiler injects into every queries.json so
|
|
193
|
+
# deep_health() can resolve a query through the cache. Must match the microcompiler
|
|
194
|
+
# constant HEALTH_QUERY_CACHE_KEY (index is always 0).
|
|
195
|
+
QUERY_CACHE_KEY__health = "__health__"
|
|
196
|
+
QUERY_CACHE_REF__health = QUERY_CACHE_KEY__health + ":0"
|
|
190
197
|
ENDPOINT__report_sentinel_error = "/sentinel/reporting/error"
|
|
191
198
|
ENDPOINT__install = "/internal/install"
|
|
192
199
|
ENDPOINT__set_shared_var = "/set-shared-var"
|
|
@@ -207,8 +214,9 @@ USERNAME__anonymous = "anonymous"
|
|
|
207
214
|
PASSWORD__anonymous = "jaaql_public_password"
|
|
208
215
|
ROLE__jaaql = "jaaql"
|
|
209
216
|
ROLE__postgres = "postgres"
|
|
217
|
+
ROLE__dba = "dba"
|
|
210
218
|
|
|
211
219
|
PROTOCOL__postgres = "postgresql://"
|
|
212
220
|
|
|
213
|
-
VERSION = "5.1
|
|
221
|
+
VERSION = "5.3.1"
|
|
214
222
|
|
|
@@ -125,7 +125,7 @@ class DBInterface(ABC):
|
|
|
125
125
|
pass
|
|
126
126
|
|
|
127
127
|
def execute_query_fetching_results(self, conn, query, parameters=None, echo=ECHO__none, as_objects=False, wait_hook: queue.Queue = None,
|
|
128
|
-
requires_dba_check: bool = False):
|
|
128
|
+
requires_dba_check: bool = False, prepare: bool = False):
|
|
129
129
|
if echo not in ECHO__allowed:
|
|
130
130
|
allowed_echoes = ", ".join([str(allowed_echo) for allowed_echo in ECHO__allowed])
|
|
131
131
|
raise HttpStatusException(ERR__unknown_echo % (str(echo), allowed_echoes), HTTPStatus.BAD_REQUEST)
|
|
@@ -149,7 +149,7 @@ class DBInterface(ABC):
|
|
|
149
149
|
if wait_hook:
|
|
150
150
|
wait_hook = None
|
|
151
151
|
|
|
152
|
-
columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook)
|
|
152
|
+
columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook, prepare=prepare)
|
|
153
153
|
|
|
154
154
|
ret = {
|
|
155
155
|
RET__columns: columns,
|
|
@@ -225,7 +225,7 @@ class DBInterface(ABC):
|
|
|
225
225
|
pass
|
|
226
226
|
|
|
227
227
|
@abstractmethod
|
|
228
|
-
def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None):
|
|
228
|
+
def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None, prepare: bool = False):
|
|
229
229
|
pass
|
|
230
230
|
|
|
231
231
|
@abstractmethod
|
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
|
|
3
|
+
import psycopg
|
|
4
|
+
from psycopg import OperationalError
|
|
5
|
+
from psycopg_pool import ConnectionPool, PoolClosed
|
|
6
|
+
import queue
|
|
7
|
+
from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
|
|
8
|
+
import threading
|
|
9
|
+
import traceback
|
|
10
|
+
from jaaql.constants import ERR__invalid_token
|
|
11
|
+
|
|
12
|
+
from jaaql.db.db_interface import DBInterface, ECHO__none, CHAR__newline
|
|
13
|
+
from jaaql.exceptions.http_status_exception import *
|
|
14
|
+
from jaaql.exceptions.custom_http_status import CustomHTTPStatus
|
|
15
|
+
from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized
|
|
16
|
+
from jaaql.constants import KEY__database
|
|
17
|
+
|
|
18
|
+
ERR__connect_db = "Could not create connection to database!"
|
|
19
|
+
|
|
20
|
+
PGCONN__min_conns = 5
|
|
21
|
+
PGCONN__max_conns = 10
|
|
22
|
+
|
|
23
|
+
TIMEOUT = 2.5
|
|
24
|
+
|
|
25
|
+
ERR__invalid_role = "Role not allowed, invalid format!"
|
|
26
|
+
ERR__must_use_canned_query = "Must use canned query as you are not an admin!"
|
|
27
|
+
|
|
28
|
+
QUERY__dba_query = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = %(database)s;"
|
|
29
|
+
QUERY__dba_query_external = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = current_database();"
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
# Pipeline mode needs libpq >= 14. create_app refuses to boot when this is False so a server
|
|
33
|
+
# can never silently degrade; the sequential fallback in execute_query exists only for
|
|
34
|
+
# non-server tooling that imports this module directly
|
|
35
|
+
PIPELINE_SUPPORTED = psycopg.Pipeline.is_supported()
|
|
36
|
+
except Exception:
|
|
37
|
+
PIPELINE_SUPPORTED = False
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class JaaqlPGConnection(psycopg.Connection):
|
|
41
|
+
"""
|
|
42
|
+
Connection whose per-checkout session-authorization statements (jaaql__set_session_authorization
|
|
43
|
+
/ SET ROLE) are deferred at checkout and sent pipelined with the first real query, saving a
|
|
44
|
+
network round trip per checkout.
|
|
45
|
+
|
|
46
|
+
Safety property: authorization must never be skippable. Any cursor obtained through the normal
|
|
47
|
+
cursor() call while statements are still pending executes them eagerly first, so raw connection
|
|
48
|
+
usage cannot run as the pool user. execute_query claims the statements via
|
|
49
|
+
jaaql_take_pending_auth and is then responsible for sending them before (or batched with) the
|
|
50
|
+
query it executes.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
_jaaql_pending_auth = None
|
|
54
|
+
_jaaql_flushing = False
|
|
55
|
+
|
|
56
|
+
def jaaql_set_pending_auth(self, statements: list):
|
|
57
|
+
self._jaaql_pending_auth = statements
|
|
58
|
+
|
|
59
|
+
def jaaql_has_pending_auth(self) -> bool:
|
|
60
|
+
return self._jaaql_pending_auth is not None
|
|
61
|
+
|
|
62
|
+
def jaaql_take_pending_auth(self):
|
|
63
|
+
pending, self._jaaql_pending_auth = self._jaaql_pending_auth, None
|
|
64
|
+
return pending
|
|
65
|
+
|
|
66
|
+
def jaaql_raw_cursor(self, *args, **kwargs):
|
|
67
|
+
# Bypasses the pending-authorization flush; the caller has taken responsibility for
|
|
68
|
+
# executing the pending statements itself
|
|
69
|
+
return super().cursor(*args, **kwargs)
|
|
70
|
+
|
|
71
|
+
def cursor(self, *args, **kwargs):
|
|
72
|
+
if self._jaaql_pending_auth is not None and not self._jaaql_flushing:
|
|
73
|
+
self._jaaql_flushing = True
|
|
74
|
+
try:
|
|
75
|
+
pending = self.jaaql_take_pending_auth()
|
|
76
|
+
with super().cursor() as flush_cursor:
|
|
77
|
+
for statement in pending:
|
|
78
|
+
flush_cursor.execute(statement)
|
|
79
|
+
finally:
|
|
80
|
+
self._jaaql_flushing = False
|
|
81
|
+
return super().cursor(*args, **kwargs)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _statement_is_preparable(query: str) -> bool:
|
|
85
|
+
# Server-side PREPARE accepts a single statement only. A ';' anywhere except trailing means the
|
|
86
|
+
# text may hold multiple statements (a ';' inside a string literal merely skips the
|
|
87
|
+
# optimisation, which is the safe direction)
|
|
88
|
+
return ";" not in query.rstrip().rstrip(";")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _escape_unescaped_percent(query: str) -> str:
|
|
92
|
+
# psycopg3 scans every '%' in the SQL when parameters are supplied and
|
|
93
|
+
# rejects anything that isn't a placeholder or '%%'. JAAQL uses named
|
|
94
|
+
# parameters only (:parameter -> %(name)s), so positional %s/%b/%t must
|
|
95
|
+
# never survive: any '%s' in author SQL is a literal inside a string
|
|
96
|
+
# (LIKE '%saas%', LIKE '%twynstra%', etc.), not a placeholder. Escape
|
|
97
|
+
# everything except '%%' and '%(name)X'.
|
|
98
|
+
out = []
|
|
99
|
+
i = 0
|
|
100
|
+
n = len(query)
|
|
101
|
+
while i < n:
|
|
102
|
+
if query[i] != '%':
|
|
103
|
+
out.append(query[i])
|
|
104
|
+
i += 1
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
nxt = query[i + 1] if i + 1 < n else ''
|
|
108
|
+
if nxt == '%':
|
|
109
|
+
out.append('%%')
|
|
110
|
+
i += 2
|
|
111
|
+
elif nxt == '(':
|
|
112
|
+
close = query.find(')', i + 2)
|
|
113
|
+
if close == -1 or close + 1 >= n:
|
|
114
|
+
out.append('%%')
|
|
115
|
+
i += 1
|
|
116
|
+
else:
|
|
117
|
+
out.append(query[i:close + 2])
|
|
118
|
+
i = close + 2
|
|
119
|
+
else:
|
|
120
|
+
out.append('%%')
|
|
121
|
+
i += 1
|
|
122
|
+
return ''.join(out)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class DBPGInterface(DBInterface):
|
|
126
|
+
|
|
127
|
+
HOST_POOLS = {}
|
|
128
|
+
HOST_POOLS_QUEUES = {}
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def close_all_pools():
|
|
132
|
+
for _, user_pool_dict in DBPGInterface.HOST_POOLS.items():
|
|
133
|
+
for _, pool in user_pool_dict.items():
|
|
134
|
+
pool.close()
|
|
135
|
+
DBPGInterface.HOST_POOLS = {}
|
|
136
|
+
DBPGInterface.HOST_POOLS_QUEUES = {}
|
|
137
|
+
|
|
138
|
+
@staticmethod
|
|
139
|
+
def _process_returned_conn(username: str, db_name: str, conn, do_reset: bool):
|
|
140
|
+
if isinstance(conn, JaaqlPGConnection) and conn.jaaql_has_pending_auth():
|
|
141
|
+
# The deferred authorization statements were never sent, so the session still runs as
|
|
142
|
+
# the pool user and there is nothing to reset. Clearing them here also stops the
|
|
143
|
+
# cursor() below from flushing them
|
|
144
|
+
conn.jaaql_take_pending_auth()
|
|
145
|
+
do_reset = False
|
|
146
|
+
if do_reset:
|
|
147
|
+
with conn.cursor() as cursor:
|
|
148
|
+
cursor.execute("RESET ROLE;")
|
|
149
|
+
did_close = False
|
|
150
|
+
if hasattr(conn, "jaaql_reset_key"):
|
|
151
|
+
try:
|
|
152
|
+
cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
|
|
153
|
+
except:
|
|
154
|
+
did_close = True
|
|
155
|
+
conn.close()
|
|
156
|
+
if not did_close:
|
|
157
|
+
cursor.execute("RESET ALL;")
|
|
158
|
+
conn.commit()
|
|
159
|
+
DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
|
|
160
|
+
|
|
161
|
+
@staticmethod
|
|
162
|
+
def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
|
|
163
|
+
while True:
|
|
164
|
+
try:
|
|
165
|
+
conn, do_reset = the_queue.get()
|
|
166
|
+
DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
|
|
167
|
+
except Exception:
|
|
168
|
+
pass # Ignore, pool has likely been wiped
|
|
169
|
+
|
|
170
|
+
def __init__(self, config, host: str, port: int, db_name: str, username: str, role: str = None, password: str = None, sub_role: str = None):
|
|
171
|
+
super().__init__(config, host, username)
|
|
172
|
+
|
|
173
|
+
self.role = role
|
|
174
|
+
self.sub_role = sub_role
|
|
175
|
+
if sub_role is not None:
|
|
176
|
+
if len([ch for ch in sub_role if not ch.isalnum() and ch not in ['_', '-']]) != 0:
|
|
177
|
+
raise HttpStatusException(ERR__invalid_role, HTTPStatus.UNAUTHORIZED)
|
|
178
|
+
|
|
179
|
+
self.output_query_exceptions = config["DEBUG"]["output_query_exceptions"].lower() == "true"
|
|
180
|
+
self.username = username
|
|
181
|
+
self.db_name = db_name
|
|
182
|
+
|
|
183
|
+
if self.username not in DBPGInterface.HOST_POOLS:
|
|
184
|
+
DBPGInterface.HOST_POOLS[self.username] = {}
|
|
185
|
+
DBPGInterface.HOST_POOLS_QUEUES[self.username] = {}
|
|
186
|
+
|
|
187
|
+
user_pool = DBPGInterface.HOST_POOLS[self.username]
|
|
188
|
+
|
|
189
|
+
self.conn_str = None
|
|
190
|
+
the_pool = None
|
|
191
|
+
|
|
192
|
+
if password is not None:
|
|
193
|
+
try:
|
|
194
|
+
conn_str = "user=" + username + " password=" + password + " dbname=" + db_name
|
|
195
|
+
# Important we don't list the host as this will force a unix socket
|
|
196
|
+
if host is not None and host not in ['localhost', '127.0.0.1']:
|
|
197
|
+
conn_str += " host=" + host
|
|
198
|
+
|
|
199
|
+
if str(port) != "5432":
|
|
200
|
+
conn_str += " port=" + str(port)
|
|
201
|
+
|
|
202
|
+
self.conn_str = conn_str
|
|
203
|
+
|
|
204
|
+
if self.db_name not in user_pool:
|
|
205
|
+
the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30,
|
|
206
|
+
connection_class=JaaqlPGConnection)
|
|
207
|
+
the_pool.getconn(timeout=TIMEOUT)
|
|
208
|
+
user_pool[self.db_name] = the_pool
|
|
209
|
+
the_queue = queue.Queue()
|
|
210
|
+
DBPGInterface.HOST_POOLS_QUEUES[self.username][self.db_name] = the_queue
|
|
211
|
+
threading.Thread(target=DBPGInterface.put_conn_threaded, args=[self.username, self.db_name, the_queue], daemon=True).start()
|
|
212
|
+
except OperationalError as ex:
|
|
213
|
+
if the_pool is not None:
|
|
214
|
+
the_pool.close()
|
|
215
|
+
if "does not exist" in str(ex).split("\"")[-1] or "couldn't get a connection after" in str(ex):
|
|
216
|
+
raise HttpStatusException("Database \"" + self.db_name + "\" does not exist",
|
|
217
|
+
CustomHTTPStatus.DATABASE_NO_EXIST)
|
|
218
|
+
else:
|
|
219
|
+
raise HttpStatusException(str(ex))
|
|
220
|
+
|
|
221
|
+
def _get_conn(self):
|
|
222
|
+
conn = DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
|
|
223
|
+
conn.jaaql_reset_key = str(uuid.uuid4())
|
|
224
|
+
if isinstance(conn, JaaqlPGConnection):
|
|
225
|
+
# Discard any pending authorization left by a previous checkout that was returned
|
|
226
|
+
# without executing (defensive: every return path should already have cleared it).
|
|
227
|
+
# Without this a later checkout could execute a previous user's authorization
|
|
228
|
+
conn.jaaql_take_pending_auth()
|
|
229
|
+
if self.role is not None or self.sub_role is not None:
|
|
230
|
+
# Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
|
|
231
|
+
# JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
|
|
232
|
+
# raise are translated by _translate_session_auth_error at execution time; dead pool
|
|
233
|
+
# connections, previously detected here, are recycled by execute_query's
|
|
234
|
+
# OperationalError retry loop
|
|
235
|
+
pending = []
|
|
236
|
+
if self.role is not None:
|
|
237
|
+
pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
|
|
238
|
+
if self.sub_role is not None:
|
|
239
|
+
pending.append("SET ROLE \"" + self.sub_role + "\"")
|
|
240
|
+
conn.jaaql_set_pending_auth(pending)
|
|
241
|
+
return conn
|
|
242
|
+
|
|
243
|
+
def get_pool(self):
|
|
244
|
+
if self.db_name not in DBPGInterface.HOST_POOLS[self.username]:
|
|
245
|
+
DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
|
|
246
|
+
max_size=PGCONN__max_conns, max_lifetime=60 * 30,
|
|
247
|
+
connection_class=JaaqlPGConnection)
|
|
248
|
+
return DBPGInterface.HOST_POOLS[self.username][self.db_name]
|
|
249
|
+
|
|
250
|
+
def get_conn(self, retry_closed_pool: bool = True):
|
|
251
|
+
try:
|
|
252
|
+
conn = self._get_conn()
|
|
253
|
+
except PoolClosed as ex:
|
|
254
|
+
# Jaaql has likely been reinstalled, this pool has been forcibly closed
|
|
255
|
+
if retry_closed_pool:
|
|
256
|
+
DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
|
|
257
|
+
max_size=PGCONN__max_conns, max_lifetime=60 * 30,
|
|
258
|
+
connection_class=JaaqlPGConnection)
|
|
259
|
+
conn = self.get_conn(retry_closed_pool=False)
|
|
260
|
+
if not conn:
|
|
261
|
+
raise ex
|
|
262
|
+
else:
|
|
263
|
+
return False
|
|
264
|
+
except HttpStatusException as ex:
|
|
265
|
+
raise ex
|
|
266
|
+
except Exception:
|
|
267
|
+
traceback.print_exc()
|
|
268
|
+
raise HttpStatusException(ERR__connect_db, HTTPStatus.INTERNAL_SERVER_ERROR)
|
|
269
|
+
|
|
270
|
+
return conn
|
|
271
|
+
|
|
272
|
+
def put_conn(self, conn):
|
|
273
|
+
DBPGInterface.HOST_POOLS_QUEUES[self.username][self.db_name].put((conn, self.role is not None))
|
|
274
|
+
|
|
275
|
+
def close(self):
|
|
276
|
+
DBPGInterface.HOST_POOLS[self.username].pop(self.db_name).close()
|
|
277
|
+
DBPGInterface.HOST_POOLS_QUEUES[self.username].pop(self.db_name)
|
|
278
|
+
|
|
279
|
+
def check_dba(self, conn, wait_hook: queue.Queue = None):
|
|
280
|
+
columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook, prepare=True)
|
|
281
|
+
if not rows[0]:
|
|
282
|
+
raise HttpStatusException(ERR__must_use_canned_query)
|
|
283
|
+
|
|
284
|
+
def _translate_session_auth_error(self, ex):
|
|
285
|
+
"""
|
|
286
|
+
The deferred session-authorization statements execute batched with the first query, so
|
|
287
|
+
their errors surface during execute_query rather than at checkout. This translates them to
|
|
288
|
+
the same exceptions the eager checkout path used to raise. Returns None when the error is
|
|
289
|
+
not recognisably from the authorization statements (the caller re-raises it untouched, so
|
|
290
|
+
a query's own error keeps its normal handling)
|
|
291
|
+
"""
|
|
292
|
+
if isinstance(ex, InternalError):
|
|
293
|
+
if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
|
|
294
|
+
return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
|
|
295
|
+
if isinstance(ex, UndefinedFunction) and "jaaql__set_session_authorization" in str(ex):
|
|
296
|
+
return HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
|
|
297
|
+
if isinstance(ex, InvalidParameterValue) and "role" in str(ex).lower():
|
|
298
|
+
return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False):
|
|
302
|
+
x = 0
|
|
303
|
+
err = None
|
|
304
|
+
while x < PGCONN__max_conns:
|
|
305
|
+
x += 1
|
|
306
|
+
try:
|
|
307
|
+
cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
|
|
308
|
+
with cursor_factory() as cursor:
|
|
309
|
+
do_prepare = prepare and _statement_is_preparable(query)
|
|
310
|
+
|
|
311
|
+
if wait_hook:
|
|
312
|
+
res, err, code = wait_hook.get()
|
|
313
|
+
if not res:
|
|
314
|
+
if code == 500:
|
|
315
|
+
raise Exception(err)
|
|
316
|
+
raise UserUnauthorized()
|
|
317
|
+
|
|
318
|
+
def execute_main_query():
|
|
319
|
+
if parameters is None or len(parameters.keys()) == 0:
|
|
320
|
+
cursor.execute(query, prepare=do_prepare)
|
|
321
|
+
else:
|
|
322
|
+
cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
|
|
323
|
+
|
|
324
|
+
# Claimed only now, after the wait_hook: if verification rejects the request,
|
|
325
|
+
# the statements remain pending and the putback thread discards them unsent
|
|
326
|
+
pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
|
|
327
|
+
|
|
328
|
+
if pending_auth:
|
|
329
|
+
auth_cursor = conn.jaaql_raw_cursor()
|
|
330
|
+
try:
|
|
331
|
+
# Pipeline mode forces the extended protocol on every execute, and the
|
|
332
|
+
# extended protocol rejects a multi-command string ("cannot insert
|
|
333
|
+
# multiple commands into a prepared statement"). Only pipeline
|
|
334
|
+
# single-command queries; a multi-command query (install scripts,
|
|
335
|
+
# interpreted/canned SQL run during a bump) falls back to sequential
|
|
336
|
+
# execution, where the main query runs under the simple protocol that
|
|
337
|
+
# permits several commands.
|
|
338
|
+
if PIPELINE_SUPPORTED and _statement_is_preparable(query):
|
|
339
|
+
with conn.pipeline():
|
|
340
|
+
for statement in pending_auth:
|
|
341
|
+
auth_cursor.execute(statement)
|
|
342
|
+
execute_main_query()
|
|
343
|
+
else:
|
|
344
|
+
for statement in pending_auth:
|
|
345
|
+
auth_cursor.execute(statement)
|
|
346
|
+
execute_main_query()
|
|
347
|
+
except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
|
|
348
|
+
translated = self._translate_session_auth_error(ex)
|
|
349
|
+
if translated is not None:
|
|
350
|
+
raise translated
|
|
351
|
+
if isinstance(ex, InternalError):
|
|
352
|
+
traceback.print_exc()
|
|
353
|
+
raise ex
|
|
354
|
+
finally:
|
|
355
|
+
try:
|
|
356
|
+
auth_cursor.close()
|
|
357
|
+
except Exception:
|
|
358
|
+
pass
|
|
359
|
+
else:
|
|
360
|
+
execute_main_query()
|
|
361
|
+
|
|
362
|
+
if cursor.description is None:
|
|
363
|
+
return [], [], []
|
|
364
|
+
else:
|
|
365
|
+
return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
|
|
366
|
+
except OperationalError as ex:
|
|
367
|
+
if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
|
|
368
|
+
if isinstance(conn, JaaqlPGConnection):
|
|
369
|
+
# This putconn bypasses the reset queue, so make sure no unsent
|
|
370
|
+
# authorization statements ride back into the pool where check() or a
|
|
371
|
+
# later checkout could flush them as the wrong user
|
|
372
|
+
conn.jaaql_take_pending_auth()
|
|
373
|
+
DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
|
|
374
|
+
DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
|
|
375
|
+
conn = self.get_conn()
|
|
376
|
+
err = ex
|
|
377
|
+
else:
|
|
378
|
+
raise ex
|
|
379
|
+
except Exception as ex:
|
|
380
|
+
if self.output_query_exceptions:
|
|
381
|
+
traceback.print_exc()
|
|
382
|
+
raise ex
|
|
383
|
+
|
|
384
|
+
if err:
|
|
385
|
+
raise err
|
|
386
|
+
|
|
387
|
+
def commit(self, conn):
|
|
388
|
+
conn.commit()
|
|
389
|
+
|
|
390
|
+
def rollback(self, conn):
|
|
391
|
+
conn.rollback()
|
|
392
|
+
|
|
393
|
+
def handle_db_error(self, err, echo):
|
|
394
|
+
if isinstance(err, ProgrammingError) and hasattr(err, 'pgresult'):
|
|
395
|
+
err = err.pgresult.error_message.decode("UTF-8")
|
|
396
|
+
|
|
397
|
+
err = str(err)
|
|
398
|
+
if echo != ECHO__none:
|
|
399
|
+
err += CHAR__newline + echo
|
|
400
|
+
raise HttpStatusException(err, HTTPStatus.UNPROCESSABLE_ENTITY)
|
{jaaql_middleware_python-5.1.0 → jaaql_middleware_python-5.3.1}/jaaql/db/db_utils_no_circ.py
RENAMED
|
@@ -68,7 +68,7 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
|
|
|
68
68
|
|
|
69
69
|
def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
|
|
70
70
|
cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
|
|
71
|
-
conn=None, interface: DBInterface = None, db_cache=None):
|
|
71
|
+
conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False):
|
|
72
72
|
if not isinstance(inputs, dict):
|
|
73
73
|
raise HttpStatusException("Expected object or string input")
|
|
74
74
|
|
|
@@ -80,7 +80,7 @@ def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: d
|
|
|
80
80
|
).transform(inputs, skip_commit=inputs.get(KEY__read_only), wait_hook=verification_hook,
|
|
81
81
|
encryption_key=db_crypt_key, conn=conn,
|
|
82
82
|
canned_query_service=cached_canned_query_service, prevent_unused_parameters=prevent_unused,
|
|
83
|
-
and_return_connection_mid_transaction=keep_alive_conn)
|
|
83
|
+
and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements)
|
|
84
84
|
|
|
85
85
|
if as_objects:
|
|
86
86
|
ret = objectify(ret, singleton=singleton)
|
|
@@ -108,6 +108,17 @@ DOCUMENTATION__is_alive = SwaggerDocumentation(
|
|
|
108
108
|
)
|
|
109
109
|
)
|
|
110
110
|
|
|
111
|
+
DOCUMENTATION__deep_health = SwaggerDocumentation(
|
|
112
|
+
security=False,
|
|
113
|
+
tags="Deep Health",
|
|
114
|
+
methods=SwaggerMethod(
|
|
115
|
+
name="Deep health check",
|
|
116
|
+
description="Resolves a query through the compiled query cache and runs it, so a stale, partial or "
|
|
117
|
+
"unloaded cache fails the check (unlike is-alive, which only pings the database)",
|
|
118
|
+
method=REST__GET
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
|
|
111
122
|
DOCUMENTATION__clean = SwaggerDocumentation(
|
|
112
123
|
tags="Installation",
|
|
113
124
|
methods=SwaggerMethod(
|
|
@@ -605,4 +616,26 @@ DOCUMENTATION__security_event = SwaggerDocumentation(
|
|
|
605
616
|
],
|
|
606
617
|
response=RES__allow_all
|
|
607
618
|
)
|
|
608
|
-
)
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
|
|
622
|
+
tags="Reporting",
|
|
623
|
+
security=False, # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
|
|
624
|
+
methods=SwaggerMethod(
|
|
625
|
+
name="Report error",
|
|
626
|
+
description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
|
|
627
|
+
"alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
|
|
628
|
+
method=REST__POST,
|
|
629
|
+
body=[
|
|
630
|
+
SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
|
|
631
|
+
SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
|
|
632
|
+
SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
|
|
633
|
+
SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
|
|
634
|
+
SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
|
|
635
|
+
SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
|
|
636
|
+
SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
|
|
637
|
+
SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
|
|
638
|
+
SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
|
|
639
|
+
]
|
|
640
|
+
)
|
|
641
|
+
)
|