jaaql-middleware-python 5.3.21__tar.gz → 5.3.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/PKG-INFO +1 -1
  2. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/constants.py +56 -1
  3. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/db/db_interface.py +7 -3
  4. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/db/db_pg_interface.py +54 -23
  5. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/db/db_utils.py +13 -3
  6. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/db/db_utils_no_circ.py +25 -8
  7. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/documentation/documentation_internal.py +7 -1
  8. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/exceptions/http_status_exception.py +41 -2
  9. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/interpreter/interpret_jaaql.py +123 -75
  10. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/base_controller.py +39 -51
  11. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/controller.py +29 -3
  12. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/handmade_queries.py +14 -0
  13. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/model.py +305 -109
  14. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/03.install_super_user.handwritten.sql +7 -0
  15. jaaql_middleware_python-5.3.23/jaaql/test/test_call_proc.py +214 -0
  16. jaaql_middleware_python-5.3.23/jaaql/test/test_execute.py +61 -0
  17. jaaql_middleware_python-5.3.23/jaaql/test/test_oidc_cookies.py +83 -0
  18. jaaql_middleware_python-5.3.23/jaaql/test/test_password_login.py +68 -0
  19. jaaql_middleware_python-5.3.23/jaaql/test/test_requested_database.py +130 -0
  20. jaaql_middleware_python-5.3.23/jaaql/test/test_security_events.py +471 -0
  21. jaaql_middleware_python-5.3.23/jaaql/test/test_sentinel_ingest.py +843 -0
  22. jaaql_middleware_python-5.3.23/jaaql/test/test_server_errors.py +1709 -0
  23. jaaql_middleware_python-5.3.23/jaaql/test/test_slow_queries.py +1081 -0
  24. jaaql_middleware_python-5.3.23/jaaql/test/test_token_client_assertion.py +94 -0
  25. jaaql_middleware_python-5.3.23/jaaql/utilities/sentinel.py +149 -0
  26. jaaql_middleware_python-5.3.23/jaaql/utilities/sentinel_ingest.py +411 -0
  27. jaaql_middleware_python-5.3.23/jaaql/utilities/server_errors.py +777 -0
  28. jaaql_middleware_python-5.3.23/jaaql/utilities/slow_queries.py +598 -0
  29. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql_middleware_python.egg-info/PKG-INFO +1 -1
  30. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql_middleware_python.egg-info/SOURCES.txt +14 -0
  31. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/LICENSE.txt +0 -0
  32. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/README.md +0 -0
  33. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/__init__.py +0 -0
  34. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/config/__init__.py +0 -0
  35. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/config/config-docker.ini +0 -0
  36. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/config/config-test.ini +0 -0
  37. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/config/config.ini +0 -0
  38. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/config_constants.py +0 -0
  39. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/db/__init__.py +0 -0
  40. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/documentation/__init__.py +0 -0
  41. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/documentation/documentation_public.py +0 -0
  42. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/documentation/documentation_shared.py +0 -0
  43. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/email/__init__.py +0 -0
  44. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/email/email_manager.py +0 -0
  45. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/email/email_manager_service.py +0 -0
  46. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/email/patch_ems.py +0 -0
  47. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/exceptions/__init__.py +0 -0
  48. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/exceptions/custom_http_status.py +0 -0
  49. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/exceptions/jaaql_interpretable_handled_errors.py +0 -0
  50. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/exceptions/not_yet_implement_exception.py +0 -0
  51. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/generated_constants.py +0 -0
  52. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/interpreter/__init__.py +0 -0
  53. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/jaaql.py +0 -0
  54. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/migrations/__init__.py +0 -0
  55. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/migrations/migrations.py +0 -0
  56. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/__init__.py +0 -0
  57. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/base_model.py +0 -0
  58. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/controller_interface.py +0 -0
  59. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/exception_queries.py +0 -0
  60. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/generated_queries.py +0 -0
  61. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/model_interface.py +0 -0
  62. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/mvc/response.py +0 -0
  63. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/openapi/__init__.py +0 -0
  64. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/openapi/swagger_documentation.py +0 -0
  65. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/patch.py +0 -0
  66. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/01.install_domains.generated.sql +0 -0
  67. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/02.install_super_user.exceptions.sql +0 -0
  68. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/04.install_jaaql_data_structures.generated.sql +0 -0
  69. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/05.install_static_data.generated.sql +0 -0
  70. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/06.install_jaaql.exceptions.sql +0 -0
  71. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/ZZZZ.generated_functions_views_and_permissions.sql +0 -0
  72. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/ZZZZ.reset_references.sql +0 -0
  73. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/scripts/swagger_template.html +0 -0
  74. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/__init__.py +0 -0
  75. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/cached_canned_query_service.py +0 -0
  76. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/migrations_manager_service.py +0 -0
  77. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/patch_mms.py +0 -0
  78. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/patch_shared_var_service.py +0 -0
  79. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/services/shared_var_service.py +0 -0
  80. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/test/__init__.py +0 -0
  81. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/test/test_connection_lifecycle.py +0 -0
  82. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/test/test_timelines.py +0 -0
  83. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/__init__.py +0 -0
  84. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/bootstrap_secrets.py +0 -0
  85. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/cron.py +0 -0
  86. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/crypt_utils.py +0 -0
  87. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/options.py +0 -0
  88. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/utils.py +0 -0
  89. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/utils_no_project_imports.py +0 -0
  90. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql/utilities/vault.py +0 -0
  91. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql_middleware_python.egg-info/dependency_links.txt +0 -0
  92. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql_middleware_python.egg-info/requires.txt +0 -0
  93. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/jaaql_middleware_python.egg-info/top_level.txt +0 -0
  94. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/setup.cfg +0 -0
  95. {jaaql_middleware_python-5.3.21 → jaaql_middleware_python-5.3.23}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jaaql-middleware-python
3
- Version: 5.3.21
3
+ Version: 5.3.23
4
4
  Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
5
5
  Home-page: https://github.com/JAAQL/JAAQL-middleware-python
6
6
  Author: Software Quality Measurement and Improvement bv
@@ -1,4 +1,5 @@
1
1
  import os
2
+ import re
2
3
  import socket
3
4
 
4
5
  # Do not delete this as it's being used elsewhere
@@ -63,6 +64,9 @@ CRON_dayOfWeek = "dayOfWeek"
63
64
 
64
65
  REGEX__dmbs_object_name = r'^[0-9a-zA-Z_]{1,63}$'
65
66
  REGEX__dmbs_procedure_name = r'^[0-9a-zA-Z_$-.\+]{1,63}$'
67
+ # An explicit type is spliced into the SQL after '::', so it must be a type name as FIESTA writes one and nothing more (use
68
+ # fullmatch): a realm name, or a Postgres datatype with an optional modifier, such as integer, numeric(4,2) or character varying(64)
69
+ REGEX__dmbs_type_name = r'(?:[A-Za-z_][A-Za-z0-9_]{0,62}|character varying)(?:\(\d{1,8}(?:,\d{1,8})?\))?'
66
70
 
67
71
  # The optional request key KEY__timelines ({"<timeline>": "<ISO-8601 moment>" | null}) sets, for the request's transaction
68
72
  # only, the moment each timeline is viewed at. A moment becomes the transaction-local setting GUC__timeline_prefix + name,
@@ -116,8 +120,31 @@ ENVIRON__local_install = "JAAQL_LOCAL_INSTALL"
116
120
  ENVIRON__jaaql_profiling = "JAAQL_PROFILING"
117
121
  ENVIRON__install_path = "INSTALL_PATH"
118
122
  ENVIRON__sentinel_url = "SENTINEL_URL"
123
+ # The time in seconds over which a JAAQL query request (its statements and its COMMIT) is reported to Sentinel as a slow query; 0 or
124
+ # less turns slow-query reports off. Read once per process, see jaaql/utilities/slow_queries.py
125
+ ENVIRON__sentinel_slow_query_seconds = "SENTINEL_SLOW_QUERY_SECONDS"
119
126
  ENVIRON__canned_queries = "CANNED_QUERIES"
120
127
 
128
+ SLOW_QUERY__default_seconds = 3
129
+ # A slow query is reported at most once an hour per process; its slow runs in between are counted into its next report. A query that has
130
+ # not been slow for this long is forgotten
131
+ SLOW_QUERY__repeat_seconds = 3600
132
+ # At most this many slow-query reports per process in any hour, so a database in trouble cannot push real errors out of Sentinel's list
133
+ SLOW_QUERY__max_reports_per_hour = 10
134
+ # Of those, at most this many for the queries of one account, so one login cannot spend them all
135
+ SLOW_QUERY__max_reports_per_account_per_hour = 3
136
+ # The queries a process keeps track of for the above; beyond it the one longest not slow is forgotten
137
+ SLOW_QUERY__max_tracked_queries = 1000
138
+ # A parameter whose name matches, at any depth of its value, is shown in a slow-query report as <redacted>. api_key holds a reversibly
139
+ # encrypted password; a document_id fetches its rendered document without a login
140
+ SLOW_QUERY__redacted_key = re.compile(r"pass(word|wd)|pwd|secret|token|api_?key|private_?key|credential|authori[sz]ation|document_id",
141
+ re.IGNORECASE)
142
+
143
+ # Reports waiting for the per-process sender; a report that finds the queue full is dropped, never waited for
144
+ SENTINEL__queue_size = 50
145
+ SENTINEL__connect_timeout = 3.05
146
+ SENTINEL__read_timeout = 10
147
+
121
148
  EMAIL_PARAM__unlock_key = "JAAQL__UNLOCK_KEY"
122
149
  EMAIL_PARAM__unlock_code = "JAAQL__UNLOCK_CODE"
123
150
  EMAIL_PARAM__app_url = "JAAQL__APP_URL"
@@ -218,6 +245,34 @@ ENDPOINT__set_shared_var = "/set-shared-var"
218
245
  ENDPOINT__get_shared_var = "/get-shared-var"
219
246
  ENDPOINT__oidc_get_token = "/exchange-auth-code"
220
247
 
248
+ # Routes whose queries are never reported as slow. Sentinel's own ingest route first: a report of it would be a report of a report.
249
+ # The rest is deploy-time and development tooling, where slow statements are expected (/prepare type-checks every query of a build)
250
+ SLOW_QUERY__unreported_routes = frozenset({
251
+ ENDPOINT__report_sentinel_error, ENDPOINT__install, "/internal/clean", ENDPOINT__execute_migrations, "/internal/freeze", "/internal/defrost",
252
+ "/internal/set-web-config", "/internal/dispatchers", "/prepare", "/domains", "/procedures", "/accounts", "/accounts/batch", "/build-time"
253
+ })
254
+ # Routes whose queries are not reported as slow when the request names no application, however it logged in: jaaql-monitor posts every
255
+ # statement of a deploy script, a migration (migrate.sh), freeze, defrost and a procedure test run to /submit without one, logging in with
256
+ # a bypass key in development and with a password on a box, while BATON names its application in every request it sends
257
+ SLOW_QUERY__unreported_without_application = frozenset({"/submit"})
258
+
259
+ # Routes whose failures are never reported as server errors (jaaql/utilities/server_errors.py): Sentinel's own ingest route, so a report
260
+ # never reports itself, and the deploy-time ones
261
+ SERVER_ERROR__unreported_routes = frozenset({
262
+ ENDPOINT__report_sentinel_error, ENDPOINT__install, "/internal/clean", ENDPOINT__execute_migrations
263
+ })
264
+ # The routes whose inputs a server-error report shows with their values: the query routes, whose SQL and parameters a slow-query report
265
+ # shows too (values included, secrets redacted). Every other route's report names its inputs only: a login's email address, an
266
+ # authorization code, a webhook's payload are never sent
267
+ SERVER_ERROR__routes_with_inputs = frozenset({"/submit", "/execute", "/call-proc"})
268
+ # The same limits as slow queries, in a budget of their own: one report per error per hour per process, its repeats counted into its next
269
+ # report; at most this many reports an hour, this many of them for one account; this many errors kept track of
270
+ SERVER_ERROR__max_reports_per_hour = 10
271
+ SERVER_ERROR__max_reports_per_account_per_hour = 3
272
+ SERVER_ERROR__max_tracked_errors = 1000
273
+ # The server faults one request notes for its report at most
274
+ SERVER_ERROR__max_faults = 5
275
+
221
276
  CONFIG__default = "Default config"
222
277
  CONFIG__default_desc = "Default config description"
223
278
  DATASET__default = "Default dataset"
@@ -236,5 +291,5 @@ ROLE__dba = "dba"
236
291
 
237
292
  PROTOCOL__postgres = "postgresql://"
238
293
 
239
- VERSION = "5.3.21"
294
+ VERSION = "5.3.23"
240
295
 
@@ -144,6 +144,8 @@ class DBInterface(ABC):
144
144
  self.log_warning(ex)
145
145
  translated = None
146
146
  if translated is not None:
147
+ # Kept for jaaql/utilities/server_errors.py, which tells a refusal of JAAQL's own COMMIT from the client's by it
148
+ setattr(translated, ATTR__database_error, commit_err)
147
149
  return translated
148
150
  return HttpStatusException("Commit failed, transaction not persisted: " + str(commit_err),
149
151
  HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -188,7 +190,8 @@ class DBInterface(ABC):
188
190
  pass
189
191
 
190
192
  def execute_query_fetching_results(self, conn, query, parameters=None, echo=ECHO__none, as_objects=False, wait_hook: queue.Queue = None,
191
- requires_dba_check: bool = False, prepare: bool = False, capture_provenance: list = None):
193
+ requires_dba_check: bool = False, prepare: bool = False, capture_provenance: list = None,
194
+ capture_timing: list = None):
192
195
  if echo not in ECHO__allowed:
193
196
  allowed_echoes = ", ".join([str(allowed_echo) for allowed_echo in ECHO__allowed])
194
197
  raise HttpStatusException(ERR__unknown_echo % (str(echo), allowed_echoes), HTTPStatus.BAD_REQUEST)
@@ -213,7 +216,7 @@ class DBInterface(ABC):
213
216
  wait_hook = None
214
217
 
215
218
  columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook, prepare=prepare,
216
- capture_provenance=capture_provenance)
219
+ capture_provenance=capture_provenance, capture_timing=capture_timing)
217
220
 
218
221
  ret = {
219
222
  RET__columns: columns,
@@ -290,7 +293,8 @@ class DBInterface(ABC):
290
293
 
291
294
  @abstractmethod
292
295
  def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None, prepare: bool = False,
293
- capture_provenance: list = None):
296
+ capture_provenance: list = None, capture_timing: list = None):
297
+ # capture_timing, when given, is appended the seconds the statement took once the parallel verifier's verdict was in
294
298
  pass
295
299
 
296
300
  @abstractmethod
@@ -3,11 +3,13 @@ import uuid
3
3
 
4
4
  import psycopg
5
5
  from psycopg import OperationalError
6
+ from psycopg.conninfo import make_conninfo
6
7
  from psycopg.pq import TransactionStatus
7
8
  from psycopg_pool import ConnectionPool, PoolClosed
8
9
  import queue
9
10
  from psycopg.errors import ProgrammingError, InvalidParameterValue, UndefinedFunction, InternalError
10
11
  import threading
12
+ import time
11
13
  import traceback
12
14
  from jaaql.constants import ERR__invalid_token
13
15
 
@@ -17,6 +19,7 @@ from jaaql.exceptions.custom_http_status import CustomHTTPStatus
17
19
  from jaaql.exceptions.jaaql_interpretable_handled_errors import UserUnauthorized, DatabaseOperationalError, handled_error_from_database_error, \
18
20
  database_error_descriptor
19
21
  from jaaql.constants import KEY__database
22
+ from jaaql.utilities import server_errors
20
23
 
21
24
  ERR__connect_db = "Could not create connection to database!"
22
25
  ERR__commit_outcome_unknown = "The connection was lost during COMMIT, so whether the transaction was persisted is unknown: "
@@ -108,6 +111,31 @@ class JaaqlPGConnection(psycopg.Connection):
108
111
  return super().cursor(*args, **kwargs)
109
112
 
110
113
 
114
+ def await_verdict(wait_hook: queue.Queue):
115
+ """
116
+ Waits for the parallel verifier's verdict on the request, raising unless it accepted it
117
+ """
118
+ try:
119
+ verdict = wait_hook.get(timeout=WAIT_HOOK__timeout)
120
+ except queue.Empty:
121
+ # The parallel verifier never delivered a verdict (its single serial thread
122
+ # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
123
+ # the worker unwind - without this bound the worker blocks forever holding
124
+ # an open transaction, and enough of them starve the pool into nginx 504s.
125
+ # A server fault whatever SQL waited for it (jaaql/utilities/server_errors.py)
126
+ raise VerificationTimedOut(ERR__verification_timed_out)
127
+ # The verdict belongs to the request, not to this attempt at it: left on the hook, a
128
+ # request re-run on a fresh connection after losing its first one reads it again
129
+ # instead of waiting out WAIT_HOOK__timeout for a second verdict that never comes
130
+ wait_hook.put(verdict)
131
+ res, err, code = verdict
132
+ if not res:
133
+ if code == 500:
134
+ # The verifier failed, and reports that itself
135
+ raise VerificationFailed(err)
136
+ raise UserUnauthorized()
137
+
138
+
111
139
  def _statement_is_preparable(query: str) -> bool:
112
140
  # Server-side PREPARE accepts a single statement only. A ';' anywhere except trailing means the
113
141
  # text may hold multiple statements (a ';' inside a string literal merely skips the
@@ -286,15 +314,17 @@ class DBPGInterface(DBInterface):
286
314
 
287
315
  if password is not None:
288
316
  try:
289
- conn_str = "user=" + username + " password=" + password + " dbname=" + db_name
317
+ conn_params = {"user": username, "password": password, "dbname": db_name}
290
318
  # Important we don't list the host as this will force a unix socket
291
319
  if host is not None and host not in ['localhost', '127.0.0.1']:
292
- conn_str += " host=" + host
320
+ conn_params["host"] = host
293
321
 
294
322
  if str(port) != "5432":
295
- conn_str += " port=" + str(port)
323
+ conn_params["port"] = str(port)
296
324
 
297
- conn_str += CONN_STR__keepalives
325
+ # make_conninfo quotes every value, so no value can add a setting of its own (a database named
326
+ # "x host=elsewhere" would otherwise send the superuser credentials to that host)
327
+ conn_str = make_conninfo(CONN_STR__keepalives.strip(), **conn_params)
298
328
 
299
329
  self.conn_str = conn_str
300
330
 
@@ -313,6 +343,9 @@ class DBPGInterface(DBInterface):
313
343
  raise HttpStatusException("Database \"" + self.db_name + "\" does not exist",
314
344
  CustomHTTPStatus.DATABASE_NO_EXIST)
315
345
  else:
346
+ # Refused for another reason (the credentials, too many connections, a server not accepting): the server's, though
347
+ # answered 422, so noted for a report should the request fail (jaaql/utilities/server_errors.py)
348
+ server_errors.note_fault(ex, server_errors.FAULT__pool, database=self.db_name)
316
349
  raise HttpStatusException(str(ex))
317
350
 
318
351
  def _get_conn(self):
@@ -407,30 +440,20 @@ class DBPGInterface(DBInterface):
407
440
  return None
408
441
 
409
442
  def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False,
410
- capture_provenance: list = None):
443
+ capture_provenance: list = None, capture_timing: list = None):
444
+ # The statement's time, for capture_timing, runs from the verifier's verdict to its last row fetched: the pipelined authorization
445
+ # statements, the execution, the transfer. The wait for the verdict is the verifier's, not the statement's
446
+ started = None
411
447
  try:
412
448
  cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
413
449
  with cursor_factory() as cursor:
414
450
  do_prepare = prepare and _statement_is_preparable(query)
415
451
 
416
452
  if wait_hook:
417
- try:
418
- verdict = wait_hook.get(timeout=WAIT_HOOK__timeout)
419
- except queue.Empty:
420
- # The parallel verifier never delivered a verdict (its single serial thread
421
- # wedged, e.g. on a connection killed by a DB reboot). Fail closed and let
422
- # the worker unwind - without this bound the worker blocks forever holding
423
- # an open transaction, and enough of them starve the pool into nginx 504s.
424
- raise Exception(ERR__verification_timed_out)
425
- # The verdict belongs to the request, not to this attempt at it: left on the hook, a
426
- # request re-run on a fresh connection after losing its first one reads it again
427
- # instead of waiting out WAIT_HOOK__timeout for a second verdict that never comes
428
- wait_hook.put(verdict)
429
- res, err, code = verdict
430
- if not res:
431
- if code == 500:
432
- raise Exception(err)
433
- raise UserUnauthorized()
453
+ await_verdict(wait_hook)
454
+
455
+ if capture_timing is not None:
456
+ started = time.perf_counter()
434
457
 
435
458
  def execute_main_query():
436
459
  if parameters is None or len(parameters.keys()) == 0:
@@ -491,6 +514,10 @@ class DBPGInterface(DBInterface):
491
514
  for idx in range(pgresult.nfields))
492
515
  return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
493
516
  except Exception as ex:
517
+ if started is not None:
518
+ # Before the pool check below, which is not the statement's time
519
+ capture_timing.append(time.perf_counter() - started)
520
+ started = None
494
521
  if isinstance(ex, OperationalError) and conn.closed:
495
522
  # The connection is gone and the transaction on it with it. The query is not retried here on
496
523
  # another connection: the caller holds this one and would go on to commit and return it, so
@@ -504,6 +531,9 @@ class DBPGInterface(DBInterface):
504
531
  if self.output_query_exceptions:
505
532
  traceback.print_exc()
506
533
  raise ex
534
+ finally:
535
+ if started is not None:
536
+ capture_timing.append(time.perf_counter() - started)
507
537
 
508
538
  def commit(self, conn):
509
539
  conn.commit()
@@ -529,7 +559,8 @@ class DBPGInterface(DBInterface):
529
559
  # Lost while the COMMIT was in flight: the server may have committed before the connection went (a crash
530
560
  # after the commit record was flushed, a network loss before the reply). The outcome is unknown, so this
531
561
  # is reported as it stands and never re-run, which could apply the request twice
532
- return DatabaseOperationalError(message=ERR__commit_outcome_unknown + str(commit_err), descriptor=database_error_descriptor(commit_err))
562
+ return outcome_unknown(DatabaseOperationalError(message=ERR__commit_outcome_unknown + str(commit_err),
563
+ descriptor=database_error_descriptor(commit_err)))
533
564
  # The server refused the COMMIT (a deferred constraint, a serialization failure, ...), so the transaction
534
565
  # was rolled back: the error maps exactly as it does when a statement raises it
535
566
  return handled_error_from_database_error(commit_err, error_set)
@@ -1,7 +1,8 @@
1
1
  from jaaql.exceptions.http_status_exception import HttpStatusException, HttpSingletonStatusException, ConnectionLostError
2
2
  from jaaql.interpreter.interpret_jaaql import InterpretJAAQL
3
- from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials
3
+ from jaaql.constants import ENCODING__utf, VAULT_KEY__super_db_credentials, REGEX__dmbs_object_name
4
4
  from typing import Union
5
+ import re
5
6
  import jaaql.utilities.crypt_utils as crypt_utils
6
7
  from jaaql.db.db_pg_interface import DBPGInterface
7
8
  from jaaql.db.db_interface import DBInterface
@@ -15,6 +16,7 @@ ERR__duplicated_encryption_salt = "Duplicated value in encryption_salts list"
15
16
  ERR__expected_single_row = "Expected single row response but received '%d' rows"
16
17
  ERR__unsupported_interface = "Unsupported interface '%s'. We only support %s"
17
18
  ERR__schema_invalid = "Schema invalid!"
19
+ ERR__database_name_unsafe = "Unsafe database name"
18
20
 
19
21
  # A lost connection persisted nothing, so a self-contained statement is re-run on a fresh connection.
20
22
  # Bounded so a genuinely-down database still surfaces an error rather than looping.
@@ -64,7 +66,8 @@ def execute_supplied_statement(db_interface, query: str, parameters: dict = None
64
66
  encryption_salts: dict = None, skip_commit: bool = False,
65
67
  autocommit: bool = False, do_prepare_only: str = None,
66
68
  attempt_fetch_domain_types: bool = False, psql: list = None,
67
- pre_psql: str = None):
69
+ pre_psql: str = None, client_sql: bool = False):
70
+ # client_sql: the query is the request's own SQL, whose errors are never JAAQL's server errors (jaaql/utilities/server_errors.py)
68
71
  if parameters is None:
69
72
  parameters = {}
70
73
 
@@ -120,7 +123,7 @@ def execute_supplied_statement(db_interface, query: str, parameters: dict = None
120
123
  try:
121
124
  data = InterpretJAAQL(db_interface).transform(statement, skip_commit=skip_commit, encryption_key=encryption_key, autocommit=autocommit,
122
125
  do_prepare_only=do_prepare_only, attempt_fetch_domain_types=attempt_fetch_domain_types,
123
- psql=psql, pre_psql=pre_psql)
126
+ psql=psql, pre_psql=pre_psql, client_sql=client_sql)
124
127
  break
125
128
  except ConnectionLostError:
126
129
  if attempts >= CONN_LOST__max_attempts:
@@ -194,6 +197,13 @@ def execute_supplied_statements(db_interface, queries: Union[str, list],
194
197
  return data
195
198
 
196
199
 
200
+ def requested_database(database) -> str:
201
+ # A database named by the request itself: a plain name, as every database JAAQL uses has one
202
+ if not isinstance(database, str) or re.fullmatch(REGEX__dmbs_object_name, database) is None:
203
+ raise HttpStatusException(ERR__database_name_unsafe)
204
+ return database
205
+
206
+
197
207
  def create_interface_for_db(vault, config, user_id: str, database: str, sub_role: str = None, session_settings: dict = None):
198
208
  jaaql_uri = vault.get_obj(VAULT_KEY__super_db_credentials)
199
209
  address, port, _, username, password = DBInterface.fracture_uri(jaaql_uri)
@@ -4,7 +4,8 @@ from http import HTTPStatus
4
4
  from jaaql.mvc.exception_queries import QUERY__fetch_application_schemas, KG__application_schema__application, KG__application__is_live, \
5
5
  KG__application_schema__name, KEY__is_default
6
6
  from queue import Queue
7
- from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts
7
+ from jaaql.db.db_utils import execute_supplied_statement, create_interface_for_db, ERR__schema_invalid, CONN_LOST__max_attempts, \
8
+ requested_database
8
9
  from jaaql.exceptions.http_status_exception import HttpStatusException, ConnectionLostError
9
10
  from jaaql.interpreter.interpret_jaaql import InterpretJAAQL, KEY_autocommit
10
11
  from jaaql.constants import KEY__application, KEY__database, KEY__schema, KEY__role, DB__jaaql, \
@@ -14,6 +15,8 @@ from jaaql.constants import KEY__application, KEY__database, KEY__schema, KEY__r
14
15
  from jaaql.db.db_interface import DBInterface
15
16
  from jaaql.utilities.utils_no_project_imports import objectify
16
17
  from jaaql.mvc.generated_queries import application__select
18
+ from jaaql.utilities.slow_queries import Label
19
+ from jaaql.utilities import server_errors
17
20
 
18
21
  ERROR_VALUE__max_length = 80
19
22
 
@@ -84,7 +87,10 @@ def pop_timeline_settings(inputs: dict):
84
87
  return settings
85
88
 
86
89
 
87
- def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict, account_id: str, conn=None, interface: DBInterface = None, db_cache=None):
90
+ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict, account_id: str, conn=None, interface: DBInterface = None, db_cache=None,
91
+ clients: bool = False):
92
+ # clients: the inputs are the request's own, so a schema it names that the application lacks, or a value of a type no application or role
93
+ # name has, is the client's even where JAAQL's own code or query meets it (jaaql/utilities/server_errors.py)
88
94
  if not isinstance(inputs, dict):
89
95
  raise HttpStatusException("Expected object or string input")
90
96
 
@@ -97,7 +103,7 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
97
103
  if schemas is None:
98
104
  schemas = execute_supplied_statement(jaaql_connection, QUERY__fetch_application_schemas, {
99
105
  KG__application_schema__application: inputs[KEY__application]
100
- }, as_objects=True)
106
+ }, as_objects=True, client_sql=clients and not isinstance(inputs[KEY__application], str))
101
107
  if len(schemas) == 0:
102
108
  application__select(jaaql_connection, inputs[KEY__application],
103
109
  singleton_message=f"Application '{inputs[KEY__application]}' does not exist. Are you sure you have installed it?")
@@ -108,7 +114,8 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
108
114
 
109
115
  found_db = None
110
116
  if KEY__schema in inputs and inputs[KEY__schema] is not None:
111
- found_db = schemas[inputs[KEY__schema]][KEY__database]
117
+ with server_errors.reading_request(clients):
118
+ found_db = schemas[inputs[KEY__schema]][KEY__database]
112
119
  inputs.pop(KEY__schema)
113
120
  else:
114
121
  if len(schemas) == 1:
@@ -122,13 +129,17 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
122
129
  raise HttpStatusException(ERR__schema_invalid)
123
130
 
124
131
  inputs[KEY__database] = found_db
132
+ elif KEY__database in inputs:
133
+ requested_database(inputs[KEY__database])
125
134
 
126
135
  if KEY__database not in inputs:
127
136
  inputs[KEY__database] = DB__jaaql
128
137
 
129
138
  sub_role = inputs.pop(KEY__role) if KEY__role in inputs else None
130
139
 
131
- required_db = create_interface_for_db(vault, config, account_id, inputs[KEY__database], sub_role, session_settings=session_settings)
140
+ # A role of another type than a name is the client's, wherever making the interface trips over it
141
+ with server_errors.reading_request(clients and not isinstance(sub_role, (str, type(None)))):
142
+ required_db = create_interface_for_db(vault, config, account_id, inputs[KEY__database], sub_role, session_settings=session_settings)
132
143
  else:
133
144
  if interface is None:
134
145
  raise Exception("Must supply interface is connection is supplied!")
@@ -140,11 +151,16 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
140
151
 
141
152
  def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
142
153
  cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
143
- conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False):
154
+ conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False, slow_query_label: Label = None,
155
+ client_sql: bool = False):
156
+ # slow_query_label names the request's queries in a slow-query report (jaaql/utilities/slow_queries.py); without one they are named
157
+ # from their SQL. client_sql says they are the request's own SQL, whose errors are never JAAQL's server errors
158
+ # (jaaql/utilities/server_errors.py). Both are given here rather than set around the call, so the application lookup below, JAAQL's
159
+ # own query, is never taken for the request's
144
160
  if not isinstance(inputs, dict):
145
161
  raise HttpStatusException("Expected object or string input")
146
162
 
147
- required_db = get_required_db(vault, config, jaaql_connection, inputs, account_id, conn, interface, db_cache=db_cache)
163
+ required_db = get_required_db(vault, config, jaaql_connection, inputs, account_id, conn, interface, db_cache=db_cache, clients=client_sql)
148
164
 
149
165
  prevent_unused = inputs.pop(KEY__prevent_unused_parameters) if KEY__prevent_unused_parameters in inputs else True
150
166
 
@@ -160,7 +176,8 @@ def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: d
160
176
  ).transform(inputs, skip_commit=inputs.get(KEY__read_only), wait_hook=verification_hook,
161
177
  encryption_key=db_crypt_key, conn=conn,
162
178
  canned_query_service=cached_canned_query_service, prevent_unused_parameters=prevent_unused,
163
- and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements)
179
+ and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements,
180
+ slow_query_label=slow_query_label, client_sql=client_sql)
164
181
  break
165
182
  except ConnectionLostError:
166
183
  if conn is not None or attempts >= CONN_LOST__max_attempts:
@@ -624,7 +624,13 @@ DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
624
624
  methods=SwaggerMethod(
625
625
  name="Report error",
626
626
  description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
627
- "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
627
+ "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively). "
628
+ "Any JSON object holding a report key is stored, whatever its Content-Type: a value that does not fit "
629
+ "Sentinel's error table is adapted (cut, NULL for a line or column out of range, defaults for missing keys, "
630
+ "unknown keys ignored), with each adaptation listed under 'Ingest adjustments:' in the stored stacktrace; a "
631
+ "body over 2 MB keeps the start and end of its stacktrace. Refused are only a body holding no JSON object "
632
+ "(400), an object with none of the report keys (400) and a body over 16 MiB (413). A stored report is "
633
+ "answered 200 even if its alert processing fails.",
628
634
  method=REST__POST,
629
635
  body=[
630
636
  SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
@@ -44,6 +44,43 @@ class ConnectionLostError(HttpStatusException):
44
44
  super().__init__("Connection lost, transaction not persisted: " + message, HTTPStatus.INTERNAL_SERVER_ERROR)
45
45
 
46
46
 
47
+ class VerificationTimedOut(Exception):
48
+ # The parallel verifier gave no verdict in time. Answered as the plain Exception it replaces; a server fault whatever SQL waited for it
49
+ # (jaaql/utilities/server_errors.py)
50
+ pass
51
+
52
+
53
+ class VerificationFailed(Exception):
54
+ # The parallel verifier failed with a server error. Answered as the plain Exception it replaces; the verifier reports it itself, the
55
+ # requests waiting for its verdict do not
56
+ pass
57
+
58
+
59
+ class AuthorizationResponseError(Exception):
60
+ # The identity provider answered a login's authorization request with an error of its own (a JARM response carrying error=...). Never
61
+ # raised: the login is answered with its redirect, and the error reported unless the user caused it (jaaql/utilities/server_errors.py)
62
+ pass
63
+
64
+
65
+ # An exception carrying this attribute set to True is the client's, whatever its status: it is never reported as a server error
66
+ ATTR__client_fault = "jaaql_client_fault"
67
+ # The database error a refused COMMIT raised, on the error it is answered with (translated as a statement's would be)
68
+ ATTR__database_error = "jaaql_database_error"
69
+ # Set to True on the error a request is answered with when its connection was lost after its SQL may have committed (a COMMIT lost in
70
+ # flight, a connection gone after the request's own COMMIT): the outcome is unknown, whoever wrote the SQL, which is the server's to answer for
71
+ ATTR__outcome_unknown = "jaaql_outcome_unknown"
72
+
73
+
74
+ def outcome_unknown(ex):
75
+ setattr(ex, ATTR__outcome_unknown, True)
76
+ return ex
77
+
78
+
79
+ def client_fault(ex):
80
+ setattr(ex, ATTR__client_fault, True)
81
+ return ex
82
+
83
+
47
84
  class JaaqlInterpretableHandledError(Exception):
48
85
  def __init__(self, error_code: int, http_response_code: int,
49
86
  table_name: str | None, index: int | None, message: str,
@@ -61,8 +98,10 @@ class JaaqlInterpretableHandledError(Exception):
61
98
 
62
99
  @staticmethod
63
100
  def deserialize_from_json(obj):
64
- return JaaqlInterpretableHandledError(
101
+ # An error a cloud procedure relays from its output, whatever its code: the procedure's own requests to JAAQL report their own
102
+ # server faults
103
+ return client_fault(JaaqlInterpretableHandledError(
65
104
  obj.get("error_code"), RESPONSE_CODE_LOOKUP.get(obj.get("error_code"), 422), obj.get("table_name"),
66
105
  obj.get("index"), obj.get("message"), obj.get("column_name"),
67
106
  obj.get("set"), obj.get("descriptor")
68
- )
107
+ ))