jaaql-middleware-python 5.1.0__py3-none-any.whl → 5.3.1__py3-none-any.whl

jaaql/constants.py

@@ -187,6 +187,13 @@ ENDPOINT__internal_applications = "/internal/applications"
 ENDPOINT__internal_templates = "/internal/emails/templates"
 ENDPOINT__internal_accounts = "/internal/emails/accounts"
 ENDPOINT__is_alive = "/internal/is-alive"
+ENDPOINT__deep_health = "/internal/deep-health"
+
+# Synthetic query the BATON microcompiler injects into every queries.json so
+# deep_health() can resolve a query through the cache. Must match the microcompiler
+# constant HEALTH_QUERY_CACHE_KEY (index is always 0).
+QUERY_CACHE_KEY__health = "__health__"
+QUERY_CACHE_REF__health = QUERY_CACHE_KEY__health + ":0"
 ENDPOINT__report_sentinel_error = "/sentinel/reporting/error"
 ENDPOINT__install = "/internal/install"
 ENDPOINT__set_shared_var = "/set-shared-var"
@@ -207,8 +214,9 @@ USERNAME__anonymous = "anonymous"
 PASSWORD__anonymous = "jaaql_public_password"
 ROLE__jaaql = "jaaql"
 ROLE__postgres = "postgres"
+ROLE__dba = "dba"
 
 PROTOCOL__postgres = "postgresql://"
 
-VERSION = "5.1.0"
+VERSION = "5.3.1"
 

jaaql/db/db_interface.py

@@ -125,7 +125,7 @@ class DBInterface(ABC):
         pass
 
     def execute_query_fetching_results(self, conn, query, parameters=None, echo=ECHO__none, as_objects=False, wait_hook: queue.Queue = None,
-                                       requires_dba_check: bool = False):
+                                       requires_dba_check: bool = False, prepare: bool = False):
         if echo not in ECHO__allowed:
             allowed_echoes = ", ".join([str(allowed_echo) for allowed_echo in ECHO__allowed])
             raise HttpStatusException(ERR__unknown_echo % (str(echo), allowed_echoes), HTTPStatus.BAD_REQUEST)
@@ -149,7 +149,7 @@ class DBInterface(ABC):
                 if wait_hook:
                     wait_hook = None
 
-            columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook)
+            columns, type_codes, rows = self.execute_query(conn, query, new_parameters, wait_hook, prepare=prepare)
 
             ret = {
                 RET__columns: columns,
@@ -225,7 +225,7 @@ class DBInterface(ABC):
         pass
 
     @abstractmethod
-    def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None):
+    def execute_query(self, conn, query, parameters: Optional[dict] = None, wait_hook: queue.Queue = None, prepare: bool = False):
         pass
 
     @abstractmethod

jaaql/db/db_pg_interface.py

@@ -1,5 +1,6 @@
 import uuid
 
+import psycopg
 from psycopg import OperationalError
 from psycopg_pool import ConnectionPool, PoolClosed
 import queue
@@ -27,6 +28,99 @@ ERR__must_use_canned_query = "Must use canned query as you are not an admin!"
 QUERY__dba_query = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = %(database)s;"
 QUERY__dba_query_external = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = current_database();"
 
+try:
+    # Pipeline mode needs libpq >= 14. create_app refuses to boot when this is False so a server
+    # can never silently degrade; the sequential fallback in execute_query exists only for
+    # non-server tooling that imports this module directly
+    PIPELINE_SUPPORTED = psycopg.Pipeline.is_supported()
+except Exception:
+    PIPELINE_SUPPORTED = False
+
+
+class JaaqlPGConnection(psycopg.Connection):
+    """
+    Connection whose per-checkout session-authorization statements (jaaql__set_session_authorization
+    / SET ROLE) are deferred at checkout and sent pipelined with the first real query, saving a
+    network round trip per checkout.
+
+    Safety property: authorization must never be skippable. Any cursor obtained through the normal
+    cursor() call while statements are still pending executes them eagerly first, so raw connection
+    usage cannot run as the pool user. execute_query claims the statements via
+    jaaql_take_pending_auth and is then responsible for sending them before (or batched with) the
+    query it executes.
+    """
+
+    _jaaql_pending_auth = None
+    _jaaql_flushing = False
+
+    def jaaql_set_pending_auth(self, statements: list):
+        self._jaaql_pending_auth = statements
+
+    def jaaql_has_pending_auth(self) -> bool:
+        return self._jaaql_pending_auth is not None
+
+    def jaaql_take_pending_auth(self):
+        pending, self._jaaql_pending_auth = self._jaaql_pending_auth, None
+        return pending
+
+    def jaaql_raw_cursor(self, *args, **kwargs):
+        # Bypasses the pending-authorization flush; the caller has taken responsibility for
+        # executing the pending statements itself
+        return super().cursor(*args, **kwargs)
+
+    def cursor(self, *args, **kwargs):
+        if self._jaaql_pending_auth is not None and not self._jaaql_flushing:
+            self._jaaql_flushing = True
+            try:
+                pending = self.jaaql_take_pending_auth()
+                with super().cursor() as flush_cursor:
+                    for statement in pending:
+                        flush_cursor.execute(statement)
+            finally:
+                self._jaaql_flushing = False
+        return super().cursor(*args, **kwargs)
+
+
+def _statement_is_preparable(query: str) -> bool:
+    # Server-side PREPARE accepts a single statement only. A ';' anywhere except trailing means the
+    # text may hold multiple statements (a ';' inside a string literal merely skips the
+    # optimisation, which is the safe direction)
+    return ";" not in query.rstrip().rstrip(";")
+
+
+def _escape_unescaped_percent(query: str) -> str:
+    # psycopg3 scans every '%' in the SQL when parameters are supplied and
+    # rejects anything that isn't a placeholder or '%%'. JAAQL uses named
+    # parameters only (:parameter -> %(name)s), so positional %s/%b/%t must
+    # never survive: any '%s' in author SQL is a literal inside a string
+    # (LIKE '%saas%', LIKE '%twynstra%', etc.), not a placeholder. Escape
+    # everything except '%%' and '%(name)X'.
+    out = []
+    i = 0
+    n = len(query)
+    while i < n:
+        if query[i] != '%':
+            out.append(query[i])
+            i += 1
+            continue
+
+        nxt = query[i + 1] if i + 1 < n else ''
+        if nxt == '%':
+            out.append('%%')
+            i += 2
+        elif nxt == '(':
+            close = query.find(')', i + 2)
+            if close == -1 or close + 1 >= n:
+                out.append('%%')
+                i += 1
+            else:
+                out.append(query[i:close + 2])
+                i = close + 2
+        else:
+            out.append('%%')
+            i += 1
+    return ''.join(out)
+
 
 class DBPGInterface(DBInterface):
 
@@ -41,25 +135,35 @@ class DBPGInterface(DBInterface):
         DBPGInterface.HOST_POOLS = {}
         DBPGInterface.HOST_POOLS_QUEUES = {}
 
+    @staticmethod
+    def _process_returned_conn(username: str, db_name: str, conn, do_reset: bool):
+        if isinstance(conn, JaaqlPGConnection) and conn.jaaql_has_pending_auth():
+            # The deferred authorization statements were never sent, so the session still runs as
+            # the pool user and there is nothing to reset. Clearing them here also stops the
+            # cursor() below from flushing them
+            conn.jaaql_take_pending_auth()
+            do_reset = False
+        if do_reset:
+            with conn.cursor() as cursor:
+                cursor.execute("RESET ROLE;")
+                did_close = False
+                if hasattr(conn, "jaaql_reset_key"):
+                    try:
+                        cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
+                    except:
+                        did_close = True
+                        conn.close()
+                if not did_close:
+                    cursor.execute("RESET ALL;")
+                    conn.commit()
+        DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
+
     @staticmethod
     def put_conn_threaded(username: str, db_name: str, the_queue: queue.Queue):
         while True:
             try:
                 conn, do_reset = the_queue.get()
-                if do_reset:
-                    with conn.cursor() as cursor:
-                        cursor.execute("RESET ROLE;")
-                        did_close = False
-                        if hasattr(conn, "jaaql_reset_key"):
-                            try:
-                                cursor.execute("SELECT jaaql_extension.jaaql__reset_session_authorization('" + str(conn.jaaql_reset_key) + "');")
-                            except:
-                                did_close = True
-                                conn.close()
-                        if not did_close:
-                            cursor.execute("RESET ALL;")
-                            conn.commit()
-                DBPGInterface.HOST_POOLS[username][db_name].putconn(conn)
+                DBPGInterface._process_returned_conn(username, db_name, conn, do_reset)
             except Exception:
                 pass  # Ignore, pool has likely been wiped
 
@@ -98,7 +202,8 @@ class DBPGInterface(DBInterface):
                 self.conn_str = conn_str
 
                 if self.db_name not in user_pool:
-                    the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                    the_pool = ConnectionPool(conn_str, min_size=PGCONN__min_conns, max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                              connection_class=JaaqlPGConnection)
                     the_pool.getconn(timeout=TIMEOUT)
                     user_pool[self.db_name] = the_pool
                     the_queue = queue.Queue()
@@ -113,49 +218,33 @@ class DBPGInterface(DBInterface):
                 else:
                     raise HttpStatusException(str(ex))
 
-    def _get_conn(self, allow_operational: bool = True):
+    def _get_conn(self):
         conn = DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
         conn.jaaql_reset_key = str(uuid.uuid4())
+        if isinstance(conn, JaaqlPGConnection):
+            # Discard any pending authorization left by a previous checkout that was returned
+            # without executing (defensive: every return path should already have cleared it).
+            # Without this a later checkout could execute a previous user's authorization
+            conn.jaaql_take_pending_auth()
         if self.role is not None or self.sub_role is not None:
-            try:
-                with conn.cursor() as cursor:
-                    if self.role is not None:
-                        try:
-                            cursor.execute("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
-                        except InternalError as ex:
-                            if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
-                                raise HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
-                            else:
-                                traceback.print_exc()
-                                raise ex
-                        except UndefinedFunction:
-                            raise HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
-                    if self.sub_role is not None:
-                        cursor.execute("SET ROLE \"" + self.sub_role + "\"")
-            except InvalidParameterValue:
-                raise HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
-            except OperationalError as ex:
-                if allow_operational:
-                    DBPGInterface.HOST_POOLS[self.username][self.db_name].close()
-                    DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                           max_size=PGCONN__max_conns, max_lifetime=60 * 30)
-                    try:
-                        DBPGInterface.HOST_POOLS[self.username][self.db_name].getconn(timeout=TIMEOUT)
-                    except OperationalError:
-                        self.close()
-                        raise HttpStatusException("Database \"" + self.db_name + "\" does not exist",
-                                                  CustomHTTPStatus.DATABASE_NO_EXIST)
-
-                    conn = self._get_conn(False)
-                else:
-                    raise ex
-
+            # Deferred: sent pipelined with the first query by execute_query, or flushed eagerly by
+            # JaaqlPGConnection.cursor() if the connection is used rawly. Errors these statements
+            # raise are translated by _translate_session_auth_error at execution time; dead pool
+            # connections, previously detected here, are recycled by execute_query's
+            # OperationalError retry loop
+            pending = []
+            if self.role is not None:
+                pending.append("SELECT jaaql_extension.jaaql__set_session_authorization('" + self.role + "', '" + conn.jaaql_reset_key + "');")
+            if self.sub_role is not None:
+                pending.append("SET ROLE \"" + self.sub_role + "\"")
+            conn.jaaql_set_pending_auth(pending)
         return conn
 
     def get_pool(self):
         if self.db_name not in DBPGInterface.HOST_POOLS[self.username]:
             DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                   max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                                                                                   max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                                                                   connection_class=JaaqlPGConnection)
         return DBPGInterface.HOST_POOLS[self.username][self.db_name]
 
     def get_conn(self, retry_closed_pool: bool = True):
@@ -165,7 +254,8 @@ class DBPGInterface(DBInterface):
             # Jaaql has likely been reinstalled, this pool has been forcibly closed
             if retry_closed_pool:
                 DBPGInterface.HOST_POOLS[self.username][self.db_name] = ConnectionPool(self.conn_str, min_size=PGCONN__min_conns,
-                                                                                       max_size=PGCONN__max_conns, max_lifetime=60 * 30)
+                                                                                       max_size=PGCONN__max_conns, max_lifetime=60 * 30,
+                                                                                       connection_class=JaaqlPGConnection)
                 conn = self.get_conn(retry_closed_pool=False)
                 if not conn:
                     raise ex
@@ -187,18 +277,36 @@ class DBPGInterface(DBInterface):
         DBPGInterface.HOST_POOLS_QUEUES[self.username].pop(self.db_name)
 
     def check_dba(self, conn, wait_hook: queue.Queue = None):
-        columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook)
+        columns, _, rows = self.execute_query(conn, QUERY__dba_query, parameters={KEY__database: self.db_name}, wait_hook=wait_hook, prepare=True)
         if not rows[0]:
             raise HttpStatusException(ERR__must_use_canned_query)
 
-    def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None):
+    def _translate_session_auth_error(self, ex):
+        """
+        The deferred session-authorization statements execute batched with the first query, so
+        their errors surface during execute_query rather than at checkout. This translates them to
+        the same exceptions the eager checkout path used to raise. Returns None when the error is
+        not recognisably from the authorization statements (the caller re-raises it untouched, so
+        a query's own error keeps its normal handling)
+        """
+        if isinstance(ex, InternalError):
+            if str(ex).startswith("role \"") and str(ex).endswith("\" does not exist"):
+                return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
+        if isinstance(ex, UndefinedFunction) and "jaaql__set_session_authorization" in str(ex):
+            return HttpStatusException("Database '%s' has not been configured for usage with JAAQL. Please ask the dba to run 'configure_database_for_use_with_jaaql' from the jaaql database" % self.db_name)
+        if isinstance(ex, InvalidParameterValue) and "role" in str(ex).lower():
+            return HttpStatusException(ERR__invalid_token, HTTPStatus.UNAUTHORIZED)
+        return None
+
+    def execute_query(self, conn, query, parameters=None, wait_hook: queue.Queue = None, prepare: bool = False):
         x = 0
         err = None
         while x < PGCONN__max_conns:
             x += 1
             try:
-                with conn.cursor() as cursor:
-                    do_prepare = False
+                cursor_factory = conn.jaaql_raw_cursor if isinstance(conn, JaaqlPGConnection) else conn.cursor
+                with cursor_factory() as cursor:
+                    do_prepare = prepare and _statement_is_preparable(query)
 
                     if wait_hook:
                         res, err, code = wait_hook.get()
@@ -207,16 +315,61 @@ class DBPGInterface(DBInterface):
                                 raise Exception(err)
                             raise UserUnauthorized()
 
-                    if parameters is None or len(parameters.keys()) == 0:
-                        cursor.execute(query, prepare=do_prepare)
+                    def execute_main_query():
+                        if parameters is None or len(parameters.keys()) == 0:
+                            cursor.execute(query, prepare=do_prepare)
+                        else:
+                            cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
+
+                    # Claimed only now, after the wait_hook: if verification rejects the request,
+                    # the statements remain pending and the putback thread discards them unsent
+                    pending_auth = conn.jaaql_take_pending_auth() if isinstance(conn, JaaqlPGConnection) else None
+
+                    if pending_auth:
+                        auth_cursor = conn.jaaql_raw_cursor()
+                        try:
+                            # Pipeline mode forces the extended protocol on every execute, and the
+                            # extended protocol rejects a multi-command string ("cannot insert
+                            # multiple commands into a prepared statement"). Only pipeline
+                            # single-command queries; a multi-command query (install scripts,
+                            # interpreted/canned SQL run during a bump) falls back to sequential
+                            # execution, where the main query runs under the simple protocol that
+                            # permits several commands.
+                            if PIPELINE_SUPPORTED and _statement_is_preparable(query):
+                                with conn.pipeline():
+                                    for statement in pending_auth:
+                                        auth_cursor.execute(statement)
+                                    execute_main_query()
+                            else:
+                                for statement in pending_auth:
+                                    auth_cursor.execute(statement)
+                                execute_main_query()
+                        except (InternalError, UndefinedFunction, InvalidParameterValue) as ex:
+                            translated = self._translate_session_auth_error(ex)
+                            if translated is not None:
+                                raise translated
+                            if isinstance(ex, InternalError):
+                                traceback.print_exc()
+                            raise ex
+                        finally:
+                            try:
+                                auth_cursor.close()
+                            except Exception:
+                                pass
                     else:
-                        cursor.execute(query, parameters, prepare=do_prepare)
+                        execute_main_query()
+
                     if cursor.description is None:
                         return [], [], []
                     else:
                         return [desc[0] for desc in cursor.description], [desc.type_code for desc in cursor.description], cursor.fetchall()
             except OperationalError as ex:
                 if ex.sqlstate is None or ex.sqlstate.startswith("08") or ex.sqlstate.startswith("57"):
+                    if isinstance(conn, JaaqlPGConnection):
+                        # This putconn bypasses the reset queue, so make sure no unsent
+                        # authorization statements ride back into the pool where check() or a
+                        # later checkout could flush them as the wrong user
+                        conn.jaaql_take_pending_auth()
                     DBPGInterface.HOST_POOLS[self.username][self.db_name].putconn(conn)
                     DBPGInterface.HOST_POOLS[self.username][self.db_name].check()
                     conn = self.get_conn()

jaaql/db/db_utils_no_circ.py

@@ -68,7 +68,7 @@ def get_required_db(vault, config, jaaql_connection: DBInterface, inputs: dict,
 
 def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: dict, account_id: str, verification_hook: Queue = None,
            cached_canned_query_service=None, as_objects: bool = False, singleton: bool = False, keep_alive_conn: bool = False,
-           conn=None, interface: DBInterface = None, db_cache=None):
+           conn=None, interface: DBInterface = None, db_cache=None, prepare_statements: bool = False):
     if not isinstance(inputs, dict):
         raise HttpStatusException("Expected object or string input")
 
@@ -80,7 +80,7 @@ def submit(vault, config, db_crypt_key, jaaql_connection: DBInterface, inputs: d
                          ).transform(inputs, skip_commit=inputs.get(KEY__read_only), wait_hook=verification_hook,
                                      encryption_key=db_crypt_key, conn=conn,
                                      canned_query_service=cached_canned_query_service, prevent_unused_parameters=prevent_unused,
-                                     and_return_connection_mid_transaction=keep_alive_conn)
+                                     and_return_connection_mid_transaction=keep_alive_conn, prepare_statements=prepare_statements)
 
     if as_objects:
         ret = objectify(ret, singleton=singleton)

jaaql/documentation/documentation_internal.py

@@ -108,6 +108,17 @@ DOCUMENTATION__is_alive = SwaggerDocumentation(
     )
 )
 
+DOCUMENTATION__deep_health = SwaggerDocumentation(
+    security=False,
+    tags="Deep Health",
+    methods=SwaggerMethod(
+        name="Deep health check",
+        description="Resolves a query through the compiled query cache and runs it, so a stale, partial or "
+                    "unloaded cache fails the check (unlike is-alive, which only pings the database)",
+        method=REST__GET
+    )
+)
+
 DOCUMENTATION__clean = SwaggerDocumentation(
     tags="Installation",
     methods=SwaggerMethod(
@@ -605,4 +616,26 @@ DOCUMENTATION__security_event = SwaggerDocumentation(
         ],
         response=RES__allow_all
     )
-)
+)
+
+DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
+    tags="Reporting",
+    security=False,  # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
+    methods=SwaggerMethod(
+        name="Report error",
+        description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
+                    "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
+        method=REST__POST,
+        body=[
+            SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
+            SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
+            SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
+            SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
+            SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
+            SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
+            SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
+            SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
+            SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
+        ]
+    )
+)

jaaql/interpreter/interpret_jaaql.py

@@ -18,6 +18,7 @@ from jaaql.constants import KEY__position, KEY__file, KEY__application, KEY__err
 from jaaql.exceptions.jaaql_interpretable_handled_errors import DatabaseOperationalError, HandledProcedureError, UnhandledQueryError, UnhandledProcedureError, \
     SingletonExpected
 from typing import Union
+from functools import lru_cache
 
 
 ERR_malformed_statement = "Malformed query, expecting string or dictionary"
@@ -112,6 +113,42 @@ class MarkerPrepare:
         self.count = count
 
 
+# Statements larger than this are tokenized on every call rather than cached, bounding the
+# memory the cache can hold (canned/compiled queries are far below this)
+TOKEN_CACHE__max_statement_len = 16384
+
+
+def _tokenize_statement_uncached(query: str, match_regex: str):
+    literals = []
+    names = []
+    last_index = 0
+    for match in re.finditer(match_regex, query):
+        start_pos = match.regs[0][0]
+        if start_pos != 0 and query[start_pos - 1] == REGEX_query_argument[0]:
+            continue
+        end_pos = match.regs[0][1]
+        literals.append(query[last_index:start_pos])
+        names.append(query[start_pos + 1:end_pos])
+        last_index = end_pos
+    literals.append(query[last_index:])
+    return tuple(literals), tuple(names)
+
+
+_tokenize_statement_cached = lru_cache(maxsize=1024)(_tokenize_statement_uncached)
+
+
+def tokenize_statement(query: str, match_regex: str):
+    """
+    Splits a statement into literal SQL fragments and the argument names found between them, so the
+    (backtracking-heavy) not-in-quotes argument regex runs once per distinct statement text instead
+    of on every request. The skip of matches directly preceded by ':' (postgres '::' casts) is part
+    of tokenization, mirroring the previous inline check in pre_prepare_statement.
+    """
+    if len(query) <= TOKEN_CACHE__max_statement_len:
+        return _tokenize_statement_cached(query, match_regex)
+    return _tokenize_statement_uncached(query, match_regex)
+
+
 class InterpretJAAQL:
 
     def __init__(self, db_interface: DBInterface, jaaql_db_interface: DBInterface = None):
@@ -121,7 +158,7 @@ class InterpretJAAQL:
     def transform(self, operation: Union[dict, str], conn=None, skip_commit: bool = False, wait_hook: queue.Queue = None,
                   encryption_key: bytes = None, autocommit: bool = False, canned_query_service=None, prevent_unused_parameters: bool = True,
                   do_prepare_only: str = False, and_return_connection_mid_transaction: bool = False, attempt_fetch_domain_types: bool = False,
-                  psql: list = None, pre_psql: str = None):
+                  psql: list = None, pre_psql: str = None, prepare_statements: bool = False):
         if (not isinstance(operation, dict)) and (not isinstance(operation, str)):
             raise HttpStatusException(ERR_malformed_operation_type, HTTPStatus.BAD_REQUEST)
 
@@ -143,6 +180,9 @@ class InterpretJAAQL:
         ret = {}
         err = None
         query_key = None
+        # Keys whose statements are server-authored (canned) and single: safe and worthwhile to
+        # execute as postgres prepared statements so parse/plan is skipped on hot connections
+        canned_keys = set()
 
         if autocommit:
             conn.autocommit = autocommit
@@ -174,6 +214,7 @@ class InterpretJAAQL:
                 query = {"query": {KEY_query: canned_query, KEY_assert: operation.get(KEY_assert), KEY_decrypt: operation.get(KEY_decrypt),
                                    KEY_parameters: {}}}
                 check_required = False
+                canned_keys.add("query")
             else:
                 is_dict_query = isinstance(query, dict)
                 if not is_dict_query:
@@ -187,15 +228,19 @@ class InterpretJAAQL:
                             query[key] = {KEY_query: val, KEY_assert: None, KEY_decrypt: None, KEY_parameters: {}}
                         else:
                             if KEY_store in val:
+                                store_all_canned = True
                                 for sub_key, sub_val in val.items():
                                     if sub_key in [KEY_parameters, KEY_decrypt, KEY_store]:
                                         continue
                                     if isinstance(sub_val, str):
                                         all_replaced = False
+                                        store_all_canned = False
                                     else:
                                         canned_query = canned_query_service.get_canned_query(operation[KEY__application],
                                                                                              sub_val[KEY__file], sub_val[KEY__position])
                                         val[sub_key] = canned_query
+                                if store_all_canned:
+                                    canned_keys.add(key)
                                 if KEY_parameters not in val:
                                     val[KEY_parameters] = {}
                                 if KEY_decrypt not in val:
@@ -208,6 +253,7 @@ class InterpretJAAQL:
                                 else:
                                     fetched_query = canned_query_service.get_canned_query(operation[KEY__application],
                                                                                           val[KEY_query][KEY__file], val[KEY_query][KEY__position])
+                                    canned_keys.add(key)
                                 query[key] = {KEY_query: fetched_query, KEY_assert: val.get(KEY_assert), KEY_decrypt: val.get(KEY_decrypt),
                                               KEY_parameters: val.get(KEY_parameters, {})}
                     check_required = not all_replaced
@@ -465,7 +511,8 @@ ORDER BY d.column_name;
                         }
                     else:
                         res = self.db_interface.execute_query_fetching_results(conn, last_query, found_params, wait_hook=wait_hook,
-                                                                               requires_dba_check=check_required and canned_query_service is not None)
+                                                                               requires_dba_check=check_required and canned_query_service is not None,
+                                                                               prepare=prepare_statements or query_key in canned_keys)
 
                     if do_prepare_only and not attempt_fetch_domain_types and not psql:
                         self.db_interface.execute_query_fetching_results(conn, "DEALLOCATE _jaaql_query_check_" + do_prepare_only, found_params,
@@ -771,6 +818,11 @@ ORDER BY d.column_name;
             return [element]
 
     def encrypt_literals(self, query, encryption_key: bytes = None, match_regex: str = REGEX_enc_literals):
+        # The default pattern can only match if the literal prefix #' is present; the substring
+        # check is C-speed and skips the regex scan for the common case of no encrypted literals
+        if match_regex is REGEX_enc_literals and "#'" not in query:
+            return query
+
         new_query = ""
         last_end_match = 0
 
@@ -786,22 +838,15 @@ ORDER BY d.column_name;
     def pre_prepare_statement(self, query, parameters, match_regex: str = REGEX_query_argument, encryption_key: bytes = None,
                               require_presence: bool = True, for_prepare: bool = False, skipped_singletons: list[str] = None,
                               replace_with_nulls: bool = False):
-        prepared = ""
-        last_index = 0
+        literals, names = tokenize_statement(query, match_regex)
+        prepared_parts = []
         found_parameters = []
         found_parameter_dictionary = {}
 
-        for match in re.finditer(match_regex, query):
+        for token_idx, match_str in enumerate(names):
             is_skipped_default = False
-            start_pos = match.regs[0][0]
-            if start_pos != 0:
-                if query[start_pos - 1] == REGEX_query_argument[0]:
-                    continue
-            end_pos = match.regs[0][1]
-            match_str = query[start_pos + 1:end_pos]
 
-            prepared += query[last_index:start_pos]
-            last_index = end_pos
+            prepared_parts.append(literals[token_idx])
 
             if match_str not in parameters and for_prepare:
                 parameters[match_str] = MarkerPrepare(len(parameters))
@@ -826,20 +871,20 @@ ORDER BY d.column_name;
                     found_parameter_dictionary[match_str] = param
 
             if parameters[match_str] is None and not isinstance(parameters[match_str], MarkerPrepare):
-                prepared += STATEMENT_null
+                prepared_parts.append(STATEMENT_null)
             else:
                 if isinstance(parameters[match_str], MarkerPrepare):
                     if replace_with_nulls:
-                        prepared += "NULL"
+                        prepared_parts.append("NULL")
                     else:
-                        prepared += "$" + str(int(parameters[match_str].count) + 1)
+                        prepared_parts.append("$" + str(int(parameters[match_str].count) + 1))
                 else:
                     the_type = self.get_format_type(match_str, parameters[match_str])
                     if is_skipped_default:
-                        prepared += MARKER_DEFAULT
+                        prepared_parts.append(MARKER_DEFAULT)
                     else:
-                        prepared += STATEMENT_argument_begin + STATEMENT_paren_open + match_str + STATEMENT_paren_close + the_type
+                        prepared_parts.append(STATEMENT_argument_begin + STATEMENT_paren_open + match_str + STATEMENT_paren_close + the_type)
 
-        prepared += query[last_index:]
+        prepared_parts.append(literals[-1])
 
-        return prepared, found_parameter_dictionary
+        return STATEMENT_empty.join(prepared_parts), found_parameter_dictionary