jaaql-middleware-python 5.2.7__py3-none-any.whl → 5.3.2__py3-none-any.whl

jaaql/constants.py

@@ -187,6 +187,13 @@ ENDPOINT__internal_applications = "/internal/applications"
 ENDPOINT__internal_templates = "/internal/emails/templates"
 ENDPOINT__internal_accounts = "/internal/emails/accounts"
 ENDPOINT__is_alive = "/internal/is-alive"
+ENDPOINT__deep_health = "/internal/deep-health"
+
+# Synthetic query the BATON microcompiler injects into every queries.json so
+# deep_health() can resolve a query through the cache. Must match the microcompiler
+# constant HEALTH_QUERY_CACHE_KEY (index is always 0).
+QUERY_CACHE_KEY__health = "__health__"
+QUERY_CACHE_REF__health = QUERY_CACHE_KEY__health + ":0"
 ENDPOINT__report_sentinel_error = "/sentinel/reporting/error"
 ENDPOINT__install = "/internal/install"
 ENDPOINT__set_shared_var = "/set-shared-var"
@@ -211,5 +218,5 @@ ROLE__dba = "dba"
 
 PROTOCOL__postgres = "postgresql://"
 
-VERSION = "5.2.7"
+VERSION = "5.3.2"
 

jaaql/db/db_interface.py

@@ -125,7 +125,7 @@ class DBInterface(ABC):
         pass
 
     def execute_query_fetching_results(self, conn, query, parameters=None, echo=ECHO__none, as_objects=False, wait_hook: queue.Queue = None,
-                                       requires_dba_check: bool = False):
+                                       requires_dba_check: bool = False, prepare: bool = False):
         if echo not in ECHO__allowed:
             allowed_echoes = ", ".join([str(allowed_echo) for allowed_echo in ECHO__allowed])
             raise HttpStatusException(ERR__unknown_echo % (str(echo), allowed_echoes), HTTPStatus.BAD_REQUEST)
@@ -149,7 +149,7 @@ class DBInterface(ABC):
                 if wait_hook:
                     wait_hook = None
 
-            columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook)
+            columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook, prepare=prepare)
 
             ret = {
                 RET__columns: columns,
@@ -225,7 +225,7 @@ class DBInterface(ABC):
         pass
 
     @abstractmethod
-    def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None):
+    def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None, prepare: bool = False):
         pass
 
     @abstractmethod

jaaql/db/db_pg_interface.py

@@ -1,5 +1,6 @@
 import uuid
 
+import psycopg
 from psycopg import OperationalError
 from psycopg_pool import ConnectionPool, PoolClosed
 import queue
@@ -27,6 +28,65 @@ ERR__must_use_canned_query = "Must use canned query as you are not an admin!"
 QUERY__dba_query = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = %(database)s;"
 QUERY__dba_query_external = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = current_database();"
 
+try:
+    # Pipeline mode needs libpq >= 14. create_app refuses to boot when this is False so a server
+    # can never silently degrade; the sequential fallback in execute_query exists only for
+    # non-server tooling that imports this module directly
+    PIPELINE_SUPPORTED = psycopg.Pipeline.is_supported()
+except Exception:
+    PIPELINE_SUPPORTED = False
+
+
+class JaaqlPGConnection(psycopg.Connection):
+    """
+    Connection whose per-checkout session-authorization statements (jaaql__set_session_authorization
+    / SET ROLE) are deferred at checkout and sent pipelined with the first real query, saving a
+    network round trip per checkout.
+
+    Safety property: authorization must never be skippable. Any cursor obtained through the normal
+    cursor() call while statements are still pending executes them eagerly first, so raw connection
+    usage cannot run as the pool user. execute_query claims the statements via
+    jaaql_take_pending_auth and is then responsible for sending them before (or batched with) the
+    query it executes.
+    """
+
+    _jaaql_pending_auth = None
+    _jaaql_flushing = False
+
+    def jaaql_set_pending_auth(self, statements: list):
+        self._jaaql_pending_auth = statements
+
+    def jaaql_has_pending_auth(self) -> bool:
+        return self._jaaql_pending_auth is not None
+
+    def jaaql_take_pending_auth(self):
+        pending, self._jaaql_pending_auth = self._jaaql_pending_auth, None
+        return pending
+
+    def jaaql_raw_cursor(self, *args, **kwargs):
+        # Bypasses the pending-authorization flush; the caller has taken responsibility for
+        # executing the pending statements itself
+        return super().cursor(*args, **kwargs)
+
+    def cursor(self, *args, **kwargs):
+        if self._jaaql_pending_auth is not None and not self._jaaql_flushing:
+            self._jaaql_flushing = True
+            try:
+                pending = self.jaaql_take_pending_auth()
+                with super().cursor() as flush_cursor:
+                    for statement in pending:
+                        flush_cursor.execute(statement)
+            finally:
+                self._jaaql_flushing = False
+        return super().cursor(*args, **kwargs)
+
+
+def _statement_is_preparable(query: str) -> bool:
+    # Server-side PREPARE accepts a single statement only. A ';' anywhere except trailing means the
+    # text may hold multiple statements (a ';' inside a string literal merely skips the
+    # optimisation, which is the safe direction)
+    return ";" not in query.rstrip().rstrip(";")
+
 
 def _escape_unescaped_percent(query: str) -> str:
     # psycopg3 scans every '%' in the SQL when parameters are supplied and
@@ -75,25 +135,35 @@ class DBPGInterface(DBInterface):
         DBPGInterface.HOST_POOLS = {}
         DBPGInterface.HOST_POOLS_QUEUES = {}
 
+    @staticmethod
+    def _process_returned_conn(username: str, db_name: str, conn, do_reset: bool):
+        if isinstance(conn, JaaqlPGConnection) and conn.jaaql_has_pending_auth():
+            # The deferred authorization statements were never sent, so the session still runs as
+            # the pool user and there is nothing to reset. Clearing them here also stops the
+            # cursor() below from flushing them
+            conn.jaaql_take_pending_auth()
+            do_reset = False
+        if do_reset:
+            with conn.cursor() as cursor:
+                cursor.execute("RESET ROLE;")
+                did_close = False
+                if hasattr(conn, "jaaql_reset_key"):
+                    try:
+                        cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
+                    except:
+                        did_close = True
+                        conn.close()
+                if not did_close:
+                    cursor.execute("RESET ALL;")
+                    conn.commit()
+        DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
+
     @staticmethod
     def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
         while True:
             try:
                 conn, do_reset = the_queue.get()
-                if do_reset:
-                    with conn.cursor() as cursor:
-                        cursor.execute("RESET ROLE;")
-                        did_close = False
-                        if hasattr(conn, "jaaql_reset_key"):
-                            try:
-                                cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
-                            except:
-                                did_close = True
-                                conn.close()
-                        if not did_close:
-                            cursor.execute("RESET ALL;")
-                            conn.commit()
-                DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
+                DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
             except Exception:
                 pass  # Ignore, pool has likely been wiped
 
@@ -132,7 +202,8 @@ class DBPGInterface(DBInterface):
                 self.conn_str = conn_str
 
                 if self.db_name not in user_pool:
-                    the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                    the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                              connection_class=JaaqlPGConnection)
                     the_pool.getconn(timeout=TIMEOUT)
                     user_pool[self.db_name] = the_pool
                     the_queue = queue.Queue()
@@ -147,49 +218,33 @@ class DBPGInterface(DBInterface):
                 else:
                     raise HttpStatusException(str(ex))
 
-    def _get_conn(self, allow_operational: bool = True):
+    def _get_conn(self):
         conn = DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
         conn.jaaql_reset_key = str(uuid.uuid4())
+        if isinstance(conn, JaaqlPGConnection):
+            # Discard any pending authorization left by a previous checkout that was returned
+            # without executing (defensive: every return path should already have cleared it).
+            # Without this a later checkout could execute a previous user's authorization
+            conn.jaaql_take_pending_auth()
         if self.role is not None or self.sub_role is not None:
-            try:
-                with conn.cursor() as cursor:
-                    if self.role is not None:
-                        try:
-                            cursor.execute("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
-                        except InternalError as ex:
-                            if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
-                                raise HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
-                            else:
-                                traceback.print_exc()
-                                raise ex
-                        except UndefinedFunction:
-                            raise HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
-                    if self.sub_role is not None:
-                        cursor.execute("SET ROLE \"" + self.sub_role + "\"")
-            except InvalidParameterValue:
-                raise HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
-            except OperationalError as ex:
-                if allow_operational:
-                    DBPGInterface.HOST_POOLS[self.username][self.db_name].close()
-                    DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                           max_size=PGCONN__max_conns, max_lifetime=60 * 30)
-                    try:
-                        DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
-                    except OperationalError:
-                        self.close()
-                        raise HttpStatusException("Database \"" + self.db_name + "\" does not exist",
-                                                  CustomHTTPStatus.DATABASE_NO_EXIST)
-
-                    conn = self._get_conn(False)
-                else:
-                    raise ex
-
+            # Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
+            # JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
+            # raise are translated by _translate_session_auth_error at execution time; dead pool
+            # connections, previously detected here, are recycled by execute_query's
+            # OperationalError retry loop
+            pending = []
+            if self.role is not None:
+                pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
+            if self.sub_role is not None:
+                pending.append("SET ROLE \"" + self.sub_role + "\"")
+            conn.jaaql_set_pending_auth(pending)
         return conn
 
     def get_pool(self):
         if self.db_name not in DBPGInterface.HOST_POOLS[self.username]:
             DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                   max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                                                                                   max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                                                                   connection_class=JaaqlPGConnection)
         return DBPGInterface.HOST_POOLS[self.username][self.db_name]
 
     def get_conn(self, retry_closed_pool: bool = True):
@@ -199,7 +254,8 @@ class DBPGInterface(DBInterface):
             # Jaaql has likely been reinstalled, this pool has been forcibly closed
             if retry_closed_pool:
                 DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                       max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                                                                                       max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                                                                       connection_class=JaaqlPGConnection)
                 conn = self.get_conn(retry_closed_pool=False)
                 if not conn:
                     raise ex
@@ -221,18 +277,36 @@ class DBPGInterface(DBInterface):
         DBPGInterface.HOST_POOLS_QUEUES[self.username].pop(self.db_name)
 
     def check_dba(self, conn, wait_hook: queue.Queue = None):
-        columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook)
+        columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook, prepare=True)
         if not rows[0]:
             raise HttpStatusException(ERR__must_use_canned_query)
 
-    def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None):
+    def _translate_session_auth_error(self, ex):
+        """
+        The deferred session-authorization statements execute batched with the first query, so
+        their errors surface during execute_query rather than at checkout. This translates them to
+        the same exceptions the eager checkout path used to raise. Returns None when the error is
+        not recognisably from the authorization statements (the caller re-raises it untouched, so
+        a query's own error keeps its normal handling)
+        """
+        if isinstance(ex, InternalError):
+            if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
+                return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
+        if isinstance(ex, UndefinedFunction) and "jaaql__set_session_authorization" in str(ex):
+            return HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
+        if isinstance(ex, InvalidParameterValue) and "role" in str(ex).lower():
+            return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
+        return None
+
+    def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False):
         x = 0
         err = None
         while x < PGCONN__max_conns:
             x += 1
             try:
-                with conn.cursor() as cursor:
-                    do_prepare = False
+                cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
+                with cursor_factory() as cursor:
+                    do_prepare = prepare and _statement_is_preparable(query)
 
                     if wait_hook:
                         res, err, code = wait_hook.get()
@@ -241,16 +315,65 @@ class DBPGInterface(DBInterface):
                                 raise Exception(err)
                             raise UserUnauthorized()
 
-                    if parameters is None or len(parameters.keys()) == 0:
-                        cursor.execute(query, prepare=do_prepare)
+                    def execute_main_query():
+                        if parameters is None or len(parameters.keys()) == 0:
+                            cursor.execute(query, prepare=do_prepare)
+                        else:
+                            cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
+
+                    # Claimed only now, after the wait_hook: if verification rejects the request,
+                    # the statements remain pending and the putback thread discards them unsent
+                    pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
+
+                    if pending_auth:
+                        auth_cursor = conn.jaaql_raw_cursor()
+                        try:
+                            # Pipeline mode is unsafe for two kinds of statement, so restrict it to
+                            # single-command queries on transactional (non-autocommit) connections;
+                            # everything else falls back to sequential execution:
+                            #  - it forces the extended protocol on every execute, which rejects a
+                            #    multi-command string ("cannot insert multiple commands into a
+                            #    prepared statement"); the sequential fallback runs the main query
+                            #    under the simple protocol that permits several commands.
+                            #  - it wraps statements in an implicit transaction, which statements
+                            #    that cannot run in a transaction block (CREATE DATABASE, VACUUM,
+                            #    CREATE INDEX CONCURRENTLY, ...) reject. JAAQL runs those with
+                            #    autocommit=True, so skipping the pipeline there keeps them standalone.
+                            if PIPELINE_SUPPORTED and _statement_is_preparable(query) and not conn.autocommit:
+                                with conn.pipeline():
+                                    for statement in pending_auth:
+                                        auth_cursor.execute(statement)
+                                    execute_main_query()
+                            else:
+                                for statement in pending_auth:
+                                    auth_cursor.execute(statement)
+                                execute_main_query()
+                        except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
+                            translated = self._translate_session_auth_error(ex)
+                            if translated is not None:
+                                raise translated
+                            if isinstance(ex, InternalError):
+                                traceback.print_exc()
+                            raise ex
+                        finally:
+                            try:
+                                auth_cursor.close()
+                            except Exception:
+                                pass
                     else:
-                        cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
+                        execute_main_query()
+
                     if cursor.description is None:
                         return [], [], []
                     else:
                         return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
             except OperationalError as ex:
                 if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
+                    if isinstance(conn, JaaqlPGConnection):
+                        # This putconn bypasses the reset queue, so make sure no unsent
+                        # authorization statements ride back into the pool where check() or a
+                        # later checkout could flush them as the wrong user
+                        conn.jaaql_take_pending_auth()
                     DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
                     DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
                     conn = self.get_conn()

jaaql/db/db_utils_no_circ.py

@@ -68,7 +68,7 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
 
 def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
            cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
-           conn=None, interface: DBInterface = None, db_cache=None):
+           conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False):
     if not isinstance(inputs, dict):
         raise HttpStatusException("Expected object or string input")
 
@@ -80,7 +80,7 @@ def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: d
                          ).transform(inputs, skip_commit=inputs.get(KEY__read_only), wait_hook=verification_hook,
                                      encryption_key=db_crypt_key, conn=conn,
                                      canned_query_service=cached_canned_query_service, prevent_unused_parameters=prevent_unused,
-                                     and_return_connection_mid_transaction=keep_alive_conn)
+                                     and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements)
 
     if as_objects:
         ret = objectify(ret, singleton=singleton)

jaaql/documentation/documentation_internal.py

@@ -108,6 +108,17 @@ DOCUMENTATION__is_alive = SwaggerDocumentation(
     )
 )
 
+DOCUMENTATION__deep_health = SwaggerDocumentation(
+    security=False,
+    tags="Deep Health",
+    methods=SwaggerMethod(
+        name="Deep health check",
+        description="Resolves a query through the compiled query cache and runs it, so a stale, partial or "
+                    "unloaded cache fails the check (unlike is-alive, which only pings the database)",
+        method=REST__GET
+    )
+)
+
 DOCUMENTATION__clean = SwaggerDocumentation(
     tags="Installation",
     methods=SwaggerMethod(
@@ -605,26 +616,26 @@ DOCUMENTATION__security_event = SwaggerDocumentation(
         ],
         response=RES__allow_all
     )
-)
-
-DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
-    tags="Reporting",
-    security=False,  # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
-    methods=SwaggerMethod(
-        name="Report error",
-        description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
-                    "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
-        method=REST__POST,
-        body=[
-            SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
-            SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
-            SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
-            SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
-            SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
-            SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
-            SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
-            SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
-            SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
-        ]
-    )
-)
+)
+
+DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
+    tags="Reporting",
+    security=False,  # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
+    methods=SwaggerMethod(
+        name="Report error",
+        description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
+                    "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
+        method=REST__POST,
+        body=[
+            SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
+            SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
+            SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
+            SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
+            SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
+            SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
+            SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
+            SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
+            SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
+        ]
+    )
+)

jaaql/interpreter/interpret_jaaql.py

@@ -18,6 +18,7 @@ from jaaql.constants import KEY__position, KEY__file, KEY__application, KEY__err
 from jaaql.exceptions.jaaql_interpretable_handled_errors import DatabaseOperationalError, HandledProcedureError, UnhandledQueryError, UnhandledProcedureError, \
     SingletonExpected
 from typing import Union
+from functools import lru_cache
 
 
 ERR_malformed_statement = "Malformed query, expecting string or dictionary"
@@ -112,6 +113,42 @@ class MarkerPrepare:
         self.count = count
 
 
+# Statements larger than this are tokenized on every call rather than cached, bounding the
+# memory the cache can hold (canned/compiled queries are far below this)
+TOKEN_CACHE__max_statement_len = 16384
+
+
+def _tokenize_statement_uncached(query: str, match_regex: str):
+    literals = []
+    names = []
+    last_index = 0
+    for match in re.finditer(match_regex, query):
+        start_pos = match.regs[0][0]
+        if start_pos != 0 and query[start_pos - 1] == REGEX_query_argument[0]:
+            continue
+        end_pos = match.regs[0][1]
+        literals.append(query[last_index:start_pos])
+        names.append(query[start_pos + 1:end_pos])
+        last_index = end_pos
+    literals.append(query[last_index:])
+    return tuple(literals), tuple(names)
+
+
+_tokenize_statement_cached = lru_cache(maxsize=1024)(_tokenize_statement_uncached)
+
+
+def tokenize_statement(query: str, match_regex: str):
+    """
+    Splits a statement into literal SQL fragments and the argument names found between them, so the
+    (backtracking-heavy) not-in-quotes argument regex runs once per distinct statement text instead
+    of on every request. The skip of matches directly preceded by ':' (postgres '::' casts) is part
+    of tokenization, mirroring the previous inline check in pre_prepare_statement.
+    """
+    if len(query) <= TOKEN_CACHE__max_statement_len:
+        return _tokenize_statement_cached(query, match_regex)
+    return _tokenize_statement_uncached(query, match_regex)
+
+
 class InterpretJAAQL:
 
     def __init__(self, db_interface: DBInterface, jaaql_db_interface: DBInterface = None):
@@ -121,7 +158,7 @@ class InterpretJAAQL:
     def transform(self, operation: Union[dict, str], conn=None, skip_commit: bool = False, wait_hook: queue.Queue = None,
                   encryption_key: bytes = None, autocommit: bool = False, canned_query_service=None, prevent_unused_parameters: bool = True,
                   do_prepare_only: str = False, and_return_connection_mid_transaction: bool = False, attempt_fetch_domain_types: bool = False,
-                  psql: list = None, pre_psql: str = None):
+                  psql: list = None, pre_psql: str = None, prepare_statements: bool = False):
         if (not isinstance(operation, dict)) and (not isinstance(operation, str)):
             raise HttpStatusException(ERR_malformed_operation_type, HTTPStatus.BAD_REQUEST)
 
@@ -143,6 +180,9 @@ class InterpretJAAQL:
         ret = {}
         err = None
         query_key = None
+        # Keys whose statements are server-authored (canned) and single: safe and worthwhile to
+        # execute as postgres prepared statements so parse/plan is skipped on hot connections
+        canned_keys = set()
 
         if autocommit:
             conn.autocommit = autocommit
@@ -174,6 +214,7 @@ class InterpretJAAQL:
                 query = {"query": {KEY_query: canned_query, KEY_assert: operation.get(KEY_assert), KEY_decrypt: operation.get(KEY_decrypt),
                                    KEY_parameters: {}}}
                 check_required = False
+                canned_keys.add("query")
             else:
                 is_dict_query = isinstance(query, dict)
                 if not is_dict_query:
@@ -187,15 +228,19 @@ class InterpretJAAQL:
                             query[key] = {KEY_query: val, KEY_assert: None, KEY_decrypt: None, KEY_parameters: {}}
                         else:
                             if KEY_store in val:
+                                store_all_canned = True
                                 for sub_key, sub_val in val.items():
                                     if sub_key in [KEY_parameters, KEY_decrypt, KEY_store]:
                                         continue
                                     if isinstance(sub_val, str):
                                         all_replaced = False
+                                        store_all_canned = False
                                     else:
                                         canned_query = canned_query_service.get_canned_query(operation[KEY__application],
                                                                                              sub_val[KEY__file], sub_val[KEY__position])
                                         val[sub_key] = canned_query
+                                if store_all_canned:
+                                    canned_keys.add(key)
                                 if KEY_parameters not in val:
                                     val[KEY_parameters] = {}
                                 if KEY_decrypt not in val:
@@ -208,6 +253,7 @@ class InterpretJAAQL:
                                 else:
                                     fetched_query = canned_query_service.get_canned_query(operation[KEY__application],
                                                                                           val[KEY_query][KEY__file], val[KEY_query][KEY__position])
+                                    canned_keys.add(key)
                                 query[key] = {KEY_query: fetched_query, KEY_assert: val.get(KEY_assert), KEY_decrypt: val.get(KEY_decrypt),
                                               KEY_parameters: val.get(KEY_parameters, {})}
                     check_required = not all_replaced
@@ -465,7 +511,8 @@ ORDER BY d.column_name;
                         }
                     else:
                         res = self.db_interface.execute_query_fetching_results(conn, last_query, found_params, wait_hook=wait_hook,
-                                                                               requires_dba_check=check_required and canned_query_service is not None)
+                                                                               requires_dba_check=check_required and canned_query_service is not None,
+                                                                               prepare=prepare_statements or query_key in canned_keys)
 
                     if do_prepare_only and not attempt_fetch_domain_types and not psql:
                         self.db_interface.execute_query_fetching_results(conn, "DEALLOCATE _jaaql_query_check_" + do_prepare_only, found_params,
@@ -771,6 +818,11 @@ ORDER BY d.column_name;
             return [element]
 
     def encrypt_literals(self, query, encryption_key: bytes = None, match_regex: str = REGEX_enc_literals):
+        # The default pattern can only match if the literal prefix #' is present; the substring
+        # check is C-speed and skips the regex scan for the common case of no encrypted literals
+        if match_regex is REGEX_enc_literals and "#'" not in query:
+            return query
+
         new_query = ""
         last_end_match = 0
 
@@ -786,22 +838,15 @@ ORDER BY d.column_name;
     def pre_prepare_statement(self, query, parameters, match_regex: str = REGEX_query_argument, encryption_key: bytes = None,
                               require_presence: bool = True, for_prepare: bool = False, skipped_singletons: list[str] = None,
                               replace_with_nulls: bool = False):
-        prepared = ""
-        last_index = 0
+        literals, names = tokenize_statement(query, match_regex)
+        prepared_parts = []
         found_parameters = []
         found_parameter_dictionary = {}
 
-        for match in re.finditer(match_regex, query):
+        for token_idx, match_str in enumerate(names):
             is_skipped_default = False
-            start_pos = match.regs[0][0]
-            if start_pos != 0:
-                if query[start_pos - 1] == REGEX_query_argument[0]:
-                    continue
-            end_pos = match.regs[0][1]
-            match_str = query[start_pos + 1:end_pos]
 
-            prepared += query[last_index:start_pos]
-            last_index = end_pos
+            prepared_parts.append(literals[token_idx])
 
             if match_str not in parameters and for_prepare:
                 parameters[match_str] = MarkerPrepare(len(parameters))
@@ -826,20 +871,20 @@ ORDER BY d.column_name;
                     found_parameter_dictionary[match_str] = param
 
             if parameters[match_str] is None and not isinstance(parameters[match_str], MarkerPrepare):
-                prepared += STATEMENT_null
+                prepared_parts.append(STATEMENT_null)
             else:
                 if isinstance(parameters[match_str], MarkerPrepare):
                     if replace_with_nulls:
-                        prepared += "NULL"
+                        prepared_parts.append("NULL")
                     else:
-                        prepared += "$" + str(int(parameters[match_str].count) + 1)
+                        prepared_parts.append("$" + str(int(parameters[match_str].count) + 1))
                 else:
                     the_type = self.get_format_type(match_str, parameters[match_str])
                     if is_skipped_default:
-                        prepared += MARKER_DEFAULT
+                        prepared_parts.append(MARKER_DEFAULT)
                     else:
-                        prepared += STATEMENT_argument_begin + STATEMENT_paren_open + match_str + STATEMENT_paren_close + the_type
+                        prepared_parts.append(STATEMENT_argument_begin + STATEMENT_paren_open + match_str + STATEMENT_paren_close + the_type)
 
-        prepared += query[last_index:]
+        prepared_parts.append(literals[-1])
 
-        return prepared, found_parameter_dictionary
+        return STATEMENT_empty.join(prepared_parts), found_parameter_dictionary

jaaql/jaaql.py

@@ -93,6 +93,19 @@ def create_app(override_config_path: str = None, controllers: [JAAQLControllerIn
 
     logging.getLogger("werkzeug").handlers.append(SensitiveHandler())
 
+    import psycopg
+    print("psycopg implementation: " + psycopg.pq.__impl__ + " (libpq " + str(psycopg.pq.version()) + ")")
+    if psycopg.pq.__impl__ == "python":
+        print("WARNING: psycopg is running the pure-python protocol implementation. Install psycopg[c] "
+              "(or psycopg[binary]) so the C implementation is used - it is significantly faster")
+
+    from jaaql.db.db_pg_interface import PIPELINE_SUPPORTED
+    if not PIPELINE_SUPPORTED:
+        print("FATAL: psycopg pipeline mode is unavailable (it requires libpq >= 14, found libpq " +
+              str(psycopg.pq.version()) + "). Session-authorization batching depends on it and JAAQL "
+              "refuses to boot degraded rather than silently fall back to per-statement round trips")
+        exit(1)
+
     config = load_config(is_gunicorn, override_config_path)
 
     mfa_name = config[CONFIG_KEY__security][CONFIG_KEY_SECURITY__mfa_label]

jaaql/mvc/base_controller.py

@@ -9,6 +9,11 @@ from werkzeug.exceptions import InternalServerError, HTTPException
 import inspect
 import json
 import requests
+
+try:
+    import orjson
+except ImportError:
+    orjson = None
 from datetime import datetime, date, time
 from jaaql.exceptions.custom_http_status import CustomHTTPStatus
 from jaaql.exceptions.jaaql_interpretable_handled_errors import UnhandledJaaqlServerError, NotYetInstalled
@@ -126,6 +131,27 @@ class JAAQLJSONProvider(DefaultJSONProvider):
         json_serial
     )  # type: ignore[assignment]
 
+    # Flask removed the JSON_SORT_KEYS config in 2.3, so the app-level setting of it below no
+    # longer has any effect; the provider attribute is what controls sorting now. Sorting is both
+    # unwanted (per that config) and slower, and orjson does not sort either
+    sort_keys = False
+
+    if orjson is not None:
+        # OPT_PASSTHROUGH_DATETIME routes datetime/date/time through json_serial so the wire format
+        # (e.g. microseconds stripped from datetimes) stays identical to the stdlib provider.
+        # orjson natively serializes UUIDs and dataclasses to the same output json_serial produces;
+        # Decimal is not natively supported so it still reaches json_serial via default=.
+        _ORJSON_OPTS = orjson.OPT_PASSTHROUGH_DATETIME | orjson.OPT_NON_STR_KEYS
+
+        def dumps(self, obj: t.Any, **kwargs: t.Any) -> str:
+            opts = self._ORJSON_OPTS
+            if kwargs.get("indent"):
+                opts |= orjson.OPT_INDENT_2
+            return orjson.dumps(obj, default=json_serial, option=opts).decode("utf-8")
+
+        def loads(self, s: t.Union[str, bytes], **kwargs: t.Any) -> t.Any:
+            return orjson.loads(s)
+
 
 class BaseJAAQLController:
 
@@ -460,6 +486,10 @@ class BaseJAAQLController:
             swagger_documentation = documentation_as_lists[0]
 
         def wrap_func(view_func):
+            # getfullargspec is expensive and the result never changes for a given view function, so
+            # resolve it once at registration instead of ~20 times per request
+            view_func_args = inspect.getfullargspec(view_func).args
+
             @wraps(view_func)
             def routed_function(view_func_local, **kwargs):
                 if not self.model.has_installed and not route.startswith('/internal'):
@@ -540,12 +570,12 @@ class BaseJAAQLController:
                     throw_ex = None
                     ex_msg = None
                     try:
-                        if ARG__http_inputs in inspect.getfullargspec(view_func_local).args or any([key in inspect.getfullargspec(view_func_local).args for key in kwargs.keys()]):
+                        if ARG__http_inputs in view_func_args or any([key in view_func_args for key in kwargs.keys()]):
                             validated = BaseJAAQLController.get_input_as_dictionary(the_method, self.is_prod, **kwargs)
-                            if ARG__http_inputs in inspect.getfullargspec(view_func_local).args:
+                            if ARG__http_inputs in view_func_args:
                                 supply_dict[ARG__http_inputs] = validated
 
-                        if ARG__account_id in inspect.getfullargspec(view_func_local).args:
+                        if ARG__account_id in view_func_args:
                             if not swagger_documentation.security:
                                 raise Exception(ERR__method_required_account_id)
                             supply_dict[ARG__account_id] = account_id
@@ -554,7 +584,7 @@ class BaseJAAQLController:
                             supply_dict[ARG__verification_hook] = verification_hook
 
                         # Do not close created interfaces as they are re-used
-                        for arg in inspect.getfullargspec(view_func_local).args:
+                        for arg in view_func_args:
                             if arg.startswith(ARG_START__connection):
                                 if not swagger_documentation.security:
                                     raise Exception(ERR__method_required_user_connection)
@@ -562,66 +592,66 @@ class BaseJAAQLController:
                                 supply_dict[arg] = create_interface_for_db(self.model.vault, self.model.config, account_id, connect_db,
                                                                            sub_role=account_id)
 
-                        for arg in inspect.getfullargspec(view_func_local).args:
+                        for arg in view_func_args:
                             if arg.startswith(ARG_START__jaaql_connection):
                                 connect_db = arg.split(ARG_START__jaaql_connection)[1]
                                 supply_dict[arg] = create_interface_for_db(self.model.vault, self.model.config, account_id, connect_db,
                                                                            sub_role=ROLE__jaaql)
 
-                        if ARG__username in inspect.getfullargspec(view_func_local).args:
+                        if ARG__username in view_func_args:
                             if not swagger_documentation.security:
                                 raise Exception(ERR__method_required_username)
                             supply_dict[ARG__username] = username
 
-                        if ARG__ip_address in inspect.getfullargspec(view_func_local).args:
+                        if ARG__ip_address in view_func_args:
                             supply_dict[ARG__ip_address] = ip_addr
 
-                        if ARG__request_headers in inspect.getfullargspec(view_func_local).args:
+                        if ARG__request_headers in view_func_args:
                             supply_dict[ARG__request_headers] = request.headers
 
-                        if ARG__response in inspect.getfullargspec(view_func_local).args:
+                        if ARG__response in view_func_args:
                             supply_dict[ARG__response] = jaaql_resp
 
-                        if ARG__auth_token in inspect.getfullargspec(view_func_local).args:
+                        if ARG__auth_token in view_func_args:
                             if not swagger_documentation.security:
                                 raise Exception(ERR__method_required_token)
                             supply_dict[ARG__auth_token] = request.headers.get(HEADER__security)
                             if supply_dict[ARG__auth_token] is None:
                                 supply_dict[ARG__auth_token] = auth_cookie
 
-                        if ARG__auth_token_for_refresh in inspect.getfullargspec(view_func_local).args:
+                        if ARG__auth_token_for_refresh in view_func_args:
                             supply_dict[ARG__auth_token_for_refresh] = request.headers.get(HEADER__security)
                             if supply_dict[ARG__auth_token_for_refresh] is None:
                                 supply_dict[ARG__auth_token_for_refresh] = auth_cookie
 
-                        if ARG__connection in inspect.getfullargspec(view_func_local).args:
+                        if ARG__connection in view_func_args:
                             if not swagger_documentation.security:
                                 raise Exception(ERR__method_required_connection)
                             supply_dict[ARG__connection] = create_interface_for_db(self.model.vault, self.model.config, account_id, DB__jaaql)
 
-                        if ARG__is_the_anonymous_user in inspect.getfullargspec(view_func_local).args:
+                        if ARG__is_the_anonymous_user in view_func_args:
                             if not swagger_documentation.security:
                                 raise Exception(ERR__method_required_is_the_anonymous_user)
                             supply_dict[ARG__is_the_anonymous_user] = is_public
 
                         self.perform_profile(request_id, "Fetch args")
-                        if ARG__profiler in inspect.getfullargspec(view_func_local).args:
+                        if ARG__profiler in view_func_args:
                             supply_dict[ARG__profiler] = Profiler(request_id, self.do_profiling)
 
-                        if ARG__headers in inspect.getfullargspec(view_func_local).args:
+                        if ARG__headers in view_func_args:
                             supply_dict[ARG__headers] = dict(request.headers)
 
-                        if ARG__body in inspect.getfullargspec(view_func_local).args:
+                        if ARG__body in view_func_args:
                             supply_dict[ARG__body] = request.get_data()
 
-                        if ARG__args in inspect.getfullargspec(view_func_local).args:
+                        if ARG__args in view_func_args:
                             supply_dict[ARG__args] = dict(request.args)
 
-                        if ARG__flask_request in inspect.getfullargspec(view_func_local).args:
+                        if ARG__flask_request in view_func_args:
                             supply_dict[ARG__flask_request] = request
 
                         for k, v in kwargs.items():
-                            if k in inspect.getfullargspec(view_func_local).args:
+                            if k in view_func_args:
                                 supply_dict[k] = v
 
                         resp = view_func_local(**supply_dict)

jaaql/mvc/base_model.py

@@ -27,6 +27,11 @@ import time
 import os
 from os.path import join
 
+# Absolute path (inside the container) of the compiled query cache written by the
+# BATON microcompiler / reset_app.sh at deploy and at boot. Re-read on flush and on
+# the cache-miss self-heal in Model.execute.
+PATH__query_cache = "/queries/queries.json"
+
 
 CONFIG_KEY_security = "SECURITY"
 CONFIG_KEY_SECURITY__mfa_label = "mfa_label"
@@ -187,18 +192,38 @@ class BaseJAAQLModel:
 
     def reload_cache(self):
         print("Received cache flush instruction")
-        if os.path.exists("/queries/queries.json"):
-            try:
-                self.query_caches = json.loads(open("/queries/queries.json", "r").read())
-                for key, encoded_list in self.query_caches["queries"].items():
-                    # Replace each encoded string in the list with its decoded version.
-                    self.query_caches["queries"][key] = [
-                        base64.b64decode(encoded).decode('utf-8') for encoded in encoded_list
-                    ]
-                self.db_cache = 1
-                print("Loaded query cache")
-            except:
-                traceback.print_exc()
+        if not os.path.exists(PATH__query_cache):
+            return
+        try:
+            mtime = os.path.getmtime(PATH__query_cache)
+            with open(PATH__query_cache, "r") as cache_file:
+                loaded = json.loads(cache_file.read())
+            # Decode the whole cache into a fresh dict and only publish it once
+            # every entry has parsed and decoded. A queries.json that is truncated
+            # or partial (a deploy/boot still mid-write - see BATON writeAllSQL) or
+            # otherwise malformed must never replace an already-good cache;
+            # otherwise the process serves a broken cache until it is restarted.
+            decoded_queries = {
+                key: [base64.b64decode(encoded).decode("utf-8") for encoded in encoded_list]
+                for key, encoded_list in loaded["queries"].items()
+            }
+            loaded["queries"] = decoded_queries
+            self.query_caches = loaded
+            self.query_caches_mtime = mtime
+            self.db_cache = 1
+            print("Loaded query cache (%d files)" % len(decoded_queries))
+        except Exception:
+            # Keep the previous good cache. A later flush, or the cache-miss
+            # self-heal in Model.execute, will retry once queries.json is whole.
+            traceback.print_exc()
+
+    def query_cache_is_stale(self):
+        # True when queries.json on disk differs from the version currently loaded,
+        # i.e. a newer (or first) cache is available to (re)load.
+        try:
+            return os.path.getmtime(PATH__query_cache) != self.query_caches_mtime
+        except OSError:
+            return False
 
     def set_jaaql_lookup_connection(self):
         if self.vault.has_obj(VAULT_KEY__jaaql_lookup_connection):
@@ -216,6 +241,7 @@ class BaseJAAQLModel:
         self.is_container = is_container
 
         self.query_caches = {}
+        self.query_caches_mtime = None
         self.db_cache = None
 
         self.prevent_arbitrary_queries = os.environ.get("PREVENT_ARBITRARY_QUERIES", "false") == "true"

jaaql/mvc/controller.py

@@ -71,6 +71,10 @@ class JAAQLController(BaseJAAQLController):
         def is_alive():
             self.model.is_alive()
 
+        @self.publish_route(ENDPOINT__deep_health, DOCUMENTATION__deep_health)
+        def deep_health():
+            self.model.deep_health()
+
         @self.publish_route('/accounts', DOCUMENTATION__create_account)
         def accounts(connection: DBInterface, http_inputs: dict):
             registered = http_inputs.get(KEY__registered, True)
@@ -158,8 +162,10 @@ class JAAQLController(BaseJAAQLController):
 
         @self.publish_route('/secure/<application>/<name>', DOCUMENTATION__secure_webhook)
         def handle_secure_webhook(application: str, name: str, body: bytes, headers: dict, args: dict,
-                                  response: JAAQLResponse, account_id: str):
-            self.model.handle_webhook(application, name, body, headers, args, response, account_id)
+                                  response: JAAQLResponse, username: str):
+            # Pass the username, not the account id: the proc runtime forwards this value as the bypass
+            # emulation user (dbms_user), which the server resolves via account.username
+            self.model.handle_webhook(application, name, body, headers, args, response, username)
 
         @self.publish_route('/remote_procedure', DOCUMENTATION__remote_procedures)
         def handle_remote_procedure(http_inputs: dict, is_the_anonymous_user: bool, auth_token: str, username: str, ip_address: str, account_id: str):
@@ -238,7 +244,7 @@ class JAAQLController(BaseJAAQLController):
                 http_inputs["ip_address"] = ip_address  # plaintext -> #ip_address (field-level encrypted)
                 inserted = self.model.submit(
                     {KEY_query: ins_error, KEY_parameters: http_inputs, KEY__application: "sentinel"},
-                    ROLE__dba, as_objects=True, singleton=True
+                    ROLE__dba, as_objects=True, singleton=True, server_authored_query=True
                 )
                 self.model.submit(
                     {
@@ -246,7 +252,7 @@ class JAAQLController(BaseJAAQLController):
                         KEY_parameters: {"id": inserted["error_id"], "raw_ip_address": ip_address},
                         KEY__application: "sentinel"
                     },
-                    ROLE__dba
+                    ROLE__dba, server_authored_query=True
                 )
             except Exception as ex:
                 import traceback

jaaql/mvc/model.py

@@ -1472,6 +1472,16 @@ WHERE
         if "PostgreSQL " not in version:
             raise HttpStatusException(ERR__keep_alive_failed)
 
+    def deep_health(self):
+        # Deep, unauthenticated health check. is_alive() only pings the DB and so
+        # stayed green while a stale/partial query cache broke every real request.
+        # This resolves a query THROUGH the query cache - the same lookup execute()
+        # performs, including the cache-miss self-heal - and runs it, so an empty,
+        # partial or stale cache fails the check. The health query is injected into
+        # every compiled queries.json by the BATON microcompiler.
+        sql = self._lookup_cached_query(QUERY_CACHE_REF__health)
+        execute_supplied_statement_singleton(self.jaaql_lookup_connection, sql, as_objects=True)
+
     def install(self, db_connection_string: str, jaaql_password: str, super_db_password: str, install_key: str, allow_uninstall: bool,
                 do_reboot: bool = True, jaaql_db_password: str = None):
         if self.jaaql_lookup_connection is None:
@@ -2254,7 +2264,7 @@ WHERE
             raise HttpStatusException("Could not intepret remote procedure result", HTTPStatus.INTERNAL_SERVER_ERROR)
 
     def handle_webhook(self, application: str, name: str, body: bytes, headers: dict, args: dict,
-                       response: JAAQLResponse, account_id: str | None):
+                       response: JAAQLResponse, username: str | None):
         rpc = remote_procedure__select(self.jaaql_lookup_connection, application, name)
         if rpc[KG__remote_procedure__access] != RPC_ACCESS__webhook:
             raise HttpStatusException("Procedure type is not type webhook")
@@ -2275,17 +2285,19 @@ WHERE
 
         argv = shlex.split(base_cmd)  # ["node","--import","./__imports__.js","RemoteProcedures/...entrypoint.js"]
         argv += [encoded_headers, encoded_args, encoded_body]
-        if account_id is not None:
-            argv += [account_id]
+        if username is not None:
+            argv += [username]  # consumed as webhook.dbms_user: the user the proc emulates via the bypass key
 
         # No shell; pass argv list. Capture output as text.
+        # Webhooks are implicit system access on the BATON side, so they need the super bypass key
         result = subprocess.run(
             argv,
             cwd=cwd,  # None if no "cd … &&" in the command
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
-            check=False
+            check=False,
+            env=self.env_with_super_bypass_key()
         )
 
         try:
@@ -2333,6 +2345,13 @@ WHERE
                 { **proc, KG__remote_procedure__application: inputs[KEY__application] })
 
 
+    def env_with_super_bypass_key(self):
+        # System crons and webhooks authenticate back to JAAQL with the super bypass key, which the BATON runtime
+        # reads from this env var. get_or_seed_vault_secret scrubs the var from os.environ at startup, so spawned
+        # children no longer inherit it - it must be re-injected here. Never inject it for private/public remote
+        # procedures (handle_procedure): those run code on behalf of a normal user and must not hold the master key
+        return {**os.environ, ENVIRON__JAAQL__SUPER_BYPASS_KEY: self.local_super_access_key}
+
     def execute_cron_jobs(self, ip_address: str):
         if ip_address not in IPS__local:
             raise UserUnauthorized()
@@ -2349,7 +2368,8 @@ WHERE
                     cron[KEY__command],
                     shell=True,
                     start_new_session=True,  # detach from gunicorn worker
-                    text=True
+                    text=True,
+                    env=self.env_with_super_bypass_key()
                 )
 
     def get_last_successful_build_time(self):
@@ -2371,6 +2391,27 @@ WHERE
 
         os.kill(int(open("app.pid", "r").read()), signal.SIGUSR1)
 
+    def _lookup_cached_query(self, trimmed: str):
+        name = trimmed.split(":")[0]
+        index = int(trimmed.split(":")[1])
+        try:
+            return self.query_caches["queries"][name][index]
+        except (KeyError, IndexError):
+            # The compiled query is absent from the in-memory cache. This happens
+            # when the container loaded queries.json before / while a deploy or
+            # boot was (re)writing it, leaving a stale or partial cache that then
+            # serves KeyErrors on the missing queries until the process restarts.
+            # If a newer queries.json is on disk, reload once and retry so the
+            # process self-heals in place instead of staying wedged.
+            if self.query_cache_is_stale():
+                self.reload_cache()
+                try:
+                    return self.query_caches["queries"][name][index]
+                except (KeyError, IndexError):
+                    pass
+            raise HttpStatusException("Compiled query '%s' is not present in the query cache" % name,
+                                      response_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+
     def execute(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False):
         if not self.query_caches:
             self.reload_cache()
@@ -2385,16 +2426,22 @@ WHERE
             self.db_cache = {itm[KG__application_schema__name]: itm for itm in schemas}
             print("Loaded DB Cache")
 
+        # Snapshot the schema cache before resolving queries: a cache-miss self-heal
+        # inside _lookup_cached_query calls reload_cache(), which resets self.db_cache
+        # to 1. submit() expects the materialised schema dict (or None), so pass the
+        # snapshot - the reload still forces a schema refetch on the next execute.
+        db_cache = self.db_cache
+
         for key, val in inputs["query"].items():
             if isinstance(val, dict):
-                trimmed = val["query"].strip()
-                val["query"] = self.query_caches["queries"][trimmed.split(":")[0]][int(trimmed.split(":")[1])]
+                val["query"] = self._lookup_cached_query(val["query"].strip())
             else:
-                trimmed = val.strip()
-                inputs["query"][key] = self.query_caches["queries"][trimmed.split(":")[0]][int(trimmed.split(":")[1])]
+                inputs["query"][key] = self._lookup_cached_query(val.strip())
 
+        # Queries here come from the compiled application query cache (server-authored, single
+        # statements), so let postgres execute them as prepared statements
         return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,
-                      self.cached_canned_query_service, as_objects=as_objects, singleton=singleton, db_cache=self.db_cache)
+                      self.cached_canned_query_service, as_objects=as_objects, singleton=singleton, db_cache=db_cache, prepare_statements=True)
 
     def call_proc(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False):
         parameters = inputs["parameters"]
@@ -2434,8 +2481,11 @@ WHERE
         return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,
                       self.cached_canned_query_service, as_objects=as_objects, singleton=singleton, db_cache=None)
 
-    def submit(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False, ip_address: str = None):
-        if ip_address not in IPS__local and self.prevent_arbitrary_queries:
+    def submit(self, inputs: dict, account_id: str, verification_hook: Queue = None, as_objects: bool = False, singleton: bool = False, ip_address: str = None,
+               server_authored_query: bool = False):
+        # server_authored_query asserts the query text is a code literal, not client input. It exempts the caller from the
+        # PREVENT_ARBITRARY_QUERIES guard, which exists to stop remote clients supplying their own query text
+        if not server_authored_query and ip_address not in IPS__local and self.prevent_arbitrary_queries:
             raise UnhandledJaaqlServerError("Not allowed to send queries to server!")
 
         return submit(self.vault, self.config, self.get_db_crypt_key(), self.jaaql_lookup_connection, inputs, account_id, verification_hook,

{jaaql_middleware_python-5.2.7.dist-info → jaaql_middleware_python-5.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jaaql-middleware-python
-Version: 5.2.7
+Version: 5.3.2
 Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
 Home-page: https://github.com/JAAQL/JAAQL-middleware-python
 Author: Software Quality Measurement and Improvement bv
@@ -10,19 +10,20 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: jaaql-monitor~=1.6.3
 Requires-Dist: psycopg[binary]~=3.2.13
-Requires-Dist: Pillow~=11.2.1
+Requires-Dist: Pillow~=11.3.0
 Requires-Dist: cryptography~=44.0.2
 Requires-Dist: flask~=3.1.0
-Requires-Dist: coverage~=7.4.3
+Requires-Dist: orjson~=3.11.9
+Requires-Dist: coverage~=7.15.2
 Requires-Dist: psycopg-pool~=3.2.1
 Requires-Dist: pyjwt~=2.10.1
-Requires-Dist: pyyaml~=6.0.1
+Requires-Dist: pyyaml~=6.0.3
 Requires-Dist: werkzeug~=3.1.3
 Requires-Dist: argon2-cffi~=23.1.0
 Requires-Dist: pyotp~=2.9.0
 Requires-Dist: qrcode~=7.4.2
 Requires-Dist: gunicorn~=23.0.0
-Requires-Dist: gevent~=24.11.1
+Requires-Dist: gevent~=25.9.1
 Requires-Dist: requests~=2.32.3
 Requires-Dist: pyzbar~=0.1.9
 Requires-Dist: parsys-requests-unixsocket~=0.3.2

{jaaql_middleware_python-5.2.7.dist-info → jaaql_middleware_python-5.3.2.dist-info}/RECORD

@@ -1,20 +1,20 @@
 jaaql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/config_constants.py,sha256=puoR6jfbXw8oXXEfbYNn2ysGwgiGTyuDjOj7F8acsTE,470
-jaaql/constants.py,sha256=Me_r4pBC0TxJZsC4uCYdI23WIgs5YnTkuvqLWPOfUu8,6994
+jaaql/constants.py,sha256=sfY4dvRV-sRuMwo6UcqD37XKFLPYNSzRD2VjM8CQNAs,7355
 jaaql/generated_constants.py,sha256=pxmpTux7rrjJH6iqLb-R8RKJ4t0CeoGdnspjH5DiWgU,11327
-jaaql/jaaql.py,sha256=0jTYtpDYgKRNKU889wMI495k_h-EzQuGEQqrqHNhNrg,5829
+jaaql/jaaql.py,sha256=bfiEW9A8PlZIVjDJd_LOhCLBL8flc_f7w0umR6EWbU4,6614
 jaaql/patch.py,sha256=VE4B6jA492NCIxNIewbynh92jxVulmRFtB9hNEe6q94,149
 jaaql/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/config/config-docker.ini,sha256=2UNtdwHQWZhANgxnrHvtxnDW0C0MuX042IZITMXNsZc,354
 jaaql/config/config-test.ini,sha256=ap9OFqCCbYdWqmUKq0p9GD1IkM4CzeTXDV5gbQGQZoQ,283
 jaaql/config/config.ini,sha256=o1orXNYRFgf09MKzZBQ3v0oMzWn7N6KwvX5YV0WJtns,321
 jaaql/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jaaql/db/db_interface.py,sha256=A_daGWC6hP3M1EewNuUU5o5PL_ab2EIzrm3ZDoFxddQ,7639
-jaaql/db/db_pg_interface.py,sha256=rzoJw_XTxIqGtuQjawNxfgRAaxQs1m1MoKMOibEK0tY,12643
+jaaql/db/db_interface.py,sha256=n4pwRHeGpkbmTOvZDe8VvqFf2l4QsE9GbhUBzW335m0,7702
+jaaql/db/db_pg_interface.py,sha256=3w7_uMUmZ7RedqIQ_hZry5HDtfUK9kCdazHra3TYuDQ,19597
 jaaql/db/db_utils.py,sha256=xZCdtWT9semzLL-NAiwMFx_38Wv96BBv60pfPYNDnCc,7761
-jaaql/db/db_utils_no_circ.py,sha256=E3jxF4O0KXUnK9yxLQYxfqDTfkAwBFJSS3sX12A1row,4593
+jaaql/db/db_utils_no_circ.py,sha256=WLhk6EwsnTdLGUAjq3RZBH3FzgwEumh45A24WWysWmQ,4666
 jaaql/documentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jaaql/documentation/documentation_internal.py,sha256=ecqvo-lN1Kl0_ce-Ohj75-nZFS6GmIActJjBC-sg1pM,22404
+jaaql/documentation/documentation_internal.py,sha256=_tU92TI4Sd8AXj3b7-UOKCN1h4VR7SaakIv2ATC45Bo,22783
 jaaql/documentation/documentation_public.py,sha256=XU1Smw-1heWhCs-FDqTCTVCkAI2COHEy6fk2fhrwFN4,10932
 jaaql/documentation/documentation_shared.py,sha256=gTw3rT2hC6LTlJMXnwmxhSTh3xUetIJPtIhldptXF84,6654
 jaaql/email/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,18 +27,18 @@ jaaql/exceptions/http_status_exception.py,sha256=rTFegFibZ9C_G3BzdLCACOkZc3k1VNS
 jaaql/exceptions/jaaql_interpretable_handled_errors.py,sha256=846c3MLeQ9VQjs9f4WM6H4rszh4tEGDLgBRd8nEMCc8,11180
 jaaql/exceptions/not_yet_implement_exception.py,sha256=T7QC-I7ciWuFFp7C_jIJRMDlpshVdy_01bVuEnu-Txs,321
 jaaql/interpreter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jaaql/interpreter/interpret_jaaql.py,sha256=MezWeC76eaQrVLkpQlN93xFfxeaIebvVfom6-r9Dl-o,39156
+jaaql/interpreter/interpret_jaaql.py,sha256=UQUukf5xMyN_IWNVpZtO058QW5a0VPuH25TVnEEgGUk,41510
 jaaql/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/migrations/migrations.py,sha256=XE46AzNmfAg9qiatIL_6ctjqiiuVMHUNEcVHhmuh4hA,4195
 jaaql/mvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jaaql/mvc/base_controller.py,sha256=FCBX6pVkUh3yeLHWbw_2ht2G0L2L3vTdk2T_qcC47cI,39102
-jaaql/mvc/base_model.py,sha256=ovNPYpd6TmJfC7qSAAxVXeejv5qVBC2Ht8qAoNR5pxE,15609
-jaaql/mvc/controller.py,sha256=lnZBYmwzJd81bdiCkic1asOa_2veEr0U-M_vNFFTS9w,14405
+jaaql/mvc/base_controller.py,sha256=I2EzDz3ace2f0Kjp9GNJk4A2cupKV4xINMZKbTJyYjI,39976
+jaaql/mvc/base_model.py,sha256=SJ2HOFNIqxhmNVIkt_zVtomVcwPBT3mxU8xCRbc7aVs,16916
+jaaql/mvc/controller.py,sha256=UV4hWAFeTrpRjLgPQSKTRxrGMIrycwM6hi7ulfqk03o,14794
 jaaql/mvc/controller_interface.py,sha256=Wx8fA4lrqb_PPwkLNjJDxDloHB5JxmNt1Gv3SsDFqT4,274
 jaaql/mvc/exception_queries.py,sha256=lF65cj9KgJgRvyzxDjPDM05rHFSHF0jy1V8B77H9Rvo,7899
 jaaql/mvc/generated_queries.py,sha256=z1Jsyu-Y6D-Xd8LYvefzwv8C01NaBjHaHI7mcvv-WEk,72232
 jaaql/mvc/handmade_queries.py,sha256=kyXZ7L_AoxfcPR4u3r5So8khlRD6_sGQ39JnSAHDsEA,938
-jaaql/mvc/model.py,sha256=p_gAw5_0GAb14PPKs1O99mRfYjtL7f3BVqyaMLcgKT4,129446
+jaaql/mvc/model.py,sha256=NAKI651kg8HdMY-4IaxkZbussKRo0b-M34cT3Bz2l-U,132860
 jaaql/mvc/model_interface.py,sha256=YgAjaAiIQL1UtIJYgvkl_C3r6PVo-8Z3MXtwzyIwKnc,257
 jaaql/mvc/response.py,sha256=F4WvR9h1JcWF3Y6cj3n5HpVIqnwiMpNqfE8D29ZHbSE,860
 jaaql/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -66,8 +66,8 @@ jaaql/utilities/options.py,sha256=75M7-oSeY4wmHGz3-su_c36yf_Csv9HxvTG7aRlGR0A,49
 jaaql/utilities/utils.py,sha256=nVYVhuHtWSjdaJaXe9eC1Usja56NO-V9SoBIlcHAK8Q,5166
 jaaql/utilities/utils_no_project_imports.py,sha256=YwcskR15dAqKdEiuF8zGx-tzwWkclSGil_zm_qZ8ylU,4626
 jaaql/utilities/vault.py,sha256=DudyQNtMtaRLL71IGQX0AI16Tg1id98BHTCQPyUNkmE,2243
-jaaql_middleware_python-5.2.7.dist-info/licenses/LICENSE.txt,sha256=wr50eN0TnkrCSuNwyiI81FLZqmrW3q0JqRlVVQIrGA0,18134
-jaaql_middleware_python-5.2.7.dist-info/METADATA,sha256=kRCQ9esMuc6bK2pFTKu3optqYTDsgwnkFxDGn00_gLI,1945
-jaaql_middleware_python-5.2.7.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-jaaql_middleware_python-5.2.7.dist-info/top_level.txt,sha256=k5aKVQy77bPyfyNHITvo5m3FqdSAiiXbjT5S9D9KzgU,6
-jaaql_middleware_python-5.2.7.dist-info/RECORD,,
+jaaql_middleware_python-5.3.2.dist-info/licenses/LICENSE.txt,sha256=wr50eN0TnkrCSuNwyiI81FLZqmrW3q0JqRlVVQIrGA0,18134
+jaaql_middleware_python-5.3.2.dist-info/METADATA,sha256=cN4zk-t4FHTjS1N16a6CldkX4G5BqynXlvWUp2qjRUU,1976
+jaaql_middleware_python-5.3.2.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+jaaql_middleware_python-5.3.2.dist-info/top_level.txt,sha256=k5aKVQy77bPyfyNHITvo5m3FqdSAiiXbjT5S9D9KzgU,6
+jaaql_middleware_python-5.3.2.dist-info/RECORD,,