jaaql-middleware-python 5.1.0__py3-none-any.whl → 5.2.7__py3-none-any.whl

jaaql/constants.py

@@ -207,8 +207,9 @@ USERNAME__anonymous = "anonymous"
 PASSWORD__anonymous = "jaaql_public_password"
 ROLE__jaaql = "jaaql"
 ROLE__postgres = "postgres"
+ROLE__dba = "dba"
 
 PROTOCOL__postgres = "postgresql://"
 
-VERSION = "5.1.0"
+VERSION = "5.2.7"
 

jaaql/db/db_pg_interface.py

@@ -28,6 +28,40 @@ QUERY__dba_query = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_databa
 QUERY__dba_query_external = "SELECT pg_has_role(datdba::regrole, 'MEMBER') FROM pg_database WHERE datname = current_database();"
 
 
+def _escape_unescaped_percent(query: str) -> str:
+    # psycopg3 scans every '%' in the SQL when parameters are supplied and
+    # rejects anything that isn't a placeholder or '%%'. JAAQL uses named
+    # parameters only (:parameter -> %(name)s), so positional %s/%b/%t must
+    # never survive: any '%s' in author SQL is a literal inside a string
+    # (LIKE '%saas%', LIKE '%twynstra%', etc.), not a placeholder. Escape
+    # everything except '%%' and '%(name)X'.
+    out = []
+    i = 0
+    n = len(query)
+    while i < n:
+        if query[i] != '%':
+            out.append(query[i])
+            i += 1
+            continue
+
+        nxt = query[i + 1] if i + 1 < n else ''
+        if nxt == '%':
+            out.append('%%')
+            i += 2
+        elif nxt == '(':
+            close = query.find(')', i + 2)
+            if close == -1 or close + 1 >= n:
+                out.append('%%')
+                i += 1
+            else:
+                out.append(query[i:close + 2])
+                i = close + 2
+        else:
+            out.append('%%')
+            i += 1
+    return ''.join(out)
+
+
 class DBPGInterface(DBInterface):
 
     HOST_POOLS = {}
@@ -210,7 +244,7 @@ class DBPGInterface(DBInterface):
                     if parameters is None or len(parameters.keys()) == 0:
                         cursor.execute(query, prepare=do_prepare)
                     else:
-                        cursor.execute(query, parameters, prepare=do_prepare)
+                        cursor.execute(_escape_unescaped_percent(query), parameters, prepare=do_prepare)
                     if cursor.description is None:
                         return [], [], []
                     else:

jaaql/documentation/documentation_internal.py

@@ -605,4 +605,26 @@ DOCUMENTATION__security_event = SwaggerDocumentation(
         ],
         response=RES__allow_all
     )
-)
+)
+
+DOCUMENTATION__report_sentinel_error = SwaggerDocumentation(
+    tags="Reporting",
+    security=False,  # PUBLIC: also receives self-posts from the in-core sentinel reporter (SENTINEL_URL=_)
+    methods=SwaggerMethod(
+        name="Report error",
+        description="Logs an application error (user_agent + ip_address encrypted at rest) and triggers Sentinel "
+                    "alert processing. Never returns 500 (would otherwise make JAAQL report itself recursively).",
+        method=REST__POST,
+        body=[
+            SwaggerArgumentResponse(name="location", description="URL/location where the error occurred", arg_type=str, example=["https://app/index.html"]),
+            SwaggerArgumentResponse(name="source_file", description="Source file the error occurred in", arg_type=str, example=["common.js"]),
+            SwaggerArgumentResponse(name="error_condensed", description="Condensed error (truncated to 200 chars for grouping)", arg_type=str, example=["Uncaught TypeError"]),
+            SwaggerArgumentResponse(name="stacktrace", description="Full stacktrace", arg_type=str, example=["at <anonymous>:1:16"]),
+            SwaggerArgumentResponse(name="version", description="Product version", arg_type=str, example=["1.2.3"]),
+            SwaggerArgumentResponse(name="source_system", description="Originating system/application", arg_type=str, example=["JAAQL"]),
+            SwaggerArgumentResponse(name="file_line_number", description="Line number", arg_type=int, required=False, condition="If available", example=[123]),
+            SwaggerArgumentResponse(name="file_col_number", description="Column number", arg_type=int, required=False, condition="If available", example=[21]),
+            SwaggerArgumentResponse(name="user_agent", description="Browser user agent (encrypted at rest). Absent on internal self-posts.", arg_type=str, required=False, condition="If present", example=["Mozilla/5.0"]),
+        ]
+    )
+)

jaaql/mvc/controller.py

@@ -13,6 +13,7 @@ from flask import request
 import queue
 
 from jaaql.utilities.utils_no_project_imports import COOKIE_OIDC, COOKIE_OIDC_RETURN
+from jaaql.interpreter.interpret_jaaql import KEY_query, KEY_parameters
 
 
 class JAAQLController(BaseJAAQLController):
@@ -221,3 +222,33 @@ class JAAQLController(BaseJAAQLController):
                 {"state": http_inputs["state"], "oidc_cookie": request.cookies.get(COOKIE_OIDC)},
                 response
             )
+
+        @self.publish_route(ENDPOINT__report_sentinel_error, DOCUMENTATION__report_sentinel_error, True)
+        def report_sentinel_error(http_inputs: dict, ip_address: str):
+            # Public error-reporting endpoint, moved here from the deprecated jaaql-sentinel-middleware.
+            # MUST NEVER return a 500: a 500 makes JAAQL self-report (SENTINEL_URL=_) and recurse.
+            ins_error = (
+                "insert into error (location, source_file, user_agent, ip_address, error_condensed, stacktrace, "
+                "file_line_number, file_col_number, version, source_system) "
+                "VALUES (:location, :source_file, #user_agent, #ip_address, left(:error_condensed, 200), :stacktrace, "
+                ":file_line_number, :file_col_number, :version, :source_system) "
+                "RETURNING id::text as error_id"
+            )
+            try:
+                http_inputs["ip_address"] = ip_address  # plaintext -> #ip_address (field-level encrypted)
+                inserted = self.model.submit(
+                    {KEY_query: ins_error, KEY_parameters: http_inputs, KEY__application: "sentinel"},
+                    ROLE__dba, as_objects=True, singleton=True
+                )
+                self.model.submit(
+                    {
+                        KEY_query: 'SELECT "error.process_alert"(:id, :raw_ip_address)',
+                        KEY_parameters: {"id": inserted["error_id"], "raw_ip_address": ip_address},
+                        KEY__application: "sentinel"
+                    },
+                    ROLE__dba
+                )
+            except Exception as ex:
+                import traceback
+                traceback.print_exc()
+                raise HttpStatusException(str(ex))

jaaql/mvc/model.py

@@ -124,6 +124,51 @@ QUERY__purge_rendered_document = "DELETE FROM document_request WHERE completed i
 QUERY__fetch_rendered_document = "SELECT app.base_url, rd.uuid as document_id, 'pdf' as render_as, rd.file_name, rd.create_file, rd.completed, rd.encrypted_access_token as oauth_token FROM document_request rd INNER JOIN document_template able ON rd.template = able.name INNER JOIN application app ON app.name = rd.application WHERE rd.uuid = :document_id"
 QUERY__fetch_cron_jobs = "SELECT * FROM remote_procedure WHERE access = 'S'";
 
+
+def _rewrite_endpoint_to_origin(endpoint_url: str, target_origin_url: str) -> str:
+    # Swap scheme+host of `endpoint_url` for those of `target_origin_url`,
+    # preserving path/query/fragment. Returns endpoint_url unchanged if either
+    # input is missing a scheme or host. Used to keep the OIDC auth iframe
+    # same-origin with the parent app — see fetch_redirect_uri.
+    if not endpoint_url or not target_origin_url:
+        return endpoint_url
+    parsed_endpoint = urllib.parse.urlparse(endpoint_url)
+    parsed_origin = urllib.parse.urlparse(target_origin_url)
+    if not parsed_origin.scheme or not parsed_origin.netloc:
+        return endpoint_url
+    return urllib.parse.urlunparse((
+        parsed_origin.scheme,
+        parsed_origin.netloc,
+        parsed_endpoint.path,
+        parsed_endpoint.params,
+        parsed_endpoint.query,
+        parsed_endpoint.fragment,
+    ))
+
+
+def _keycloak_backchannel_origin(discovery_url: str) -> str:
+    # Canonical scheme+host that JAAQL uses for SERVER-side Keycloak calls
+    # (PAR, token, JWKS). With per-realm frontendUrl set, the discovery doc
+    # advertises endpoint URLs against the BROWSER host (the app's domain),
+    # which routes through the same-origin nginx proxy. That proxy terminates
+    # the middleware's mTLS handshake at the app's nginx and opens a fresh
+    # upstream TLS connection — the FAPI client cert never reaches Keycloak.
+    # So for backchannel we must skip the proxy.
+    #
+    # Source of truth: KEYCLOAK_URL env var (already set per-deployment).
+    # Fallback: the origin of user_registry.discovery_url, which is also the
+    # canonical server-side host for the deployment.
+    env_url = os.environ.get("KEYCLOAK_URL", "").strip().rstrip("/")
+    if env_url:
+        return env_url
+    if not discovery_url:
+        return ""
+    parsed = urllib.parse.urlparse(discovery_url)
+    if not parsed.scheme or not parsed.netloc:
+        return ""
+    return f"{parsed.scheme}://{parsed.netloc}"
+
+
 class JAAQLModel(BaseJAAQLModel):
     VERIFICATION_QUEUE = None
 
@@ -648,9 +693,16 @@ WHERE
         return discovery
 
     def fetch_jwks_client(self, database: str, provider: str, tenant: str, discovery):
-        jwks_url = discovery.get("jwks_uri").replace("localhost", "host.docker.internal")
+        jwks_url = discovery.get("jwks_uri")
         if not jwks_url:
             raise Exception(f"Discovery document for {provider}, {tenant} did not have JWKS url")
+        # Force backchannel: skip the app same-origin proxy and hit Keycloak directly.
+        # JWKS doesn't require mTLS, but keeping all backchannel calls on the same
+        # canonical host avoids surprise routing and an extra hop. KEYCLOAK_URL env
+        # is the source of truth; if absent we leave the discovery URL untouched.
+        backchannel_origin = os.environ.get("KEYCLOAK_URL", "").strip().rstrip("/")
+        if backchannel_origin:
+            jwks_url = _rewrite_endpoint_to_origin(jwks_url, backchannel_origin)
         lookup = database + ":" + provider + ":" + tenant
         jwks = self.JWKS_CACHE.get(lookup)
         if not jwks:
@@ -697,13 +749,25 @@ WHERE
         discovery = self.fetch_discovery_content(database, provider, tenant, user_registry[KG__user_registry__discovery_url])
         jwk_client = self.fetch_jwks_client(database, provider, tenant, discovery)
 
-        expected_issuer = os.environ.get("OIDC_ISSUER", "").strip() or discovery.get("issuer")
-        if not expected_issuer:
-            raise Exception(f"Failed to fetch issuer in discovery document for {provider}, {tenant}")
+        oidc_issuer_override = os.environ.get("OIDC_ISSUER", "").strip()
+        if oidc_issuer_override:
+            expected_issuer = oidc_issuer_override
+        else:
+            expected_issuer = discovery.get("issuer")
+            if not expected_issuer:
+                raise Exception(f"Failed to fetch issuer in discovery document for {provider}, {tenant}")
+            # JARMS and id_tokens echo Keycloak's view of the host the BROWSER
+            # hit — which, with the same-origin proxy in place, is the app's
+            # base_url, not the server-side discovery host. Rewrite the
+            # discovery issuer to match so jwt.decode(...issuer=...) lines up.
+            expected_issuer = _rewrite_endpoint_to_origin(expected_issuer, application_tuple[KG__application__base_url])
 
         token_endpoint = discovery.get("token_endpoint")
         if not token_endpoint:
             raise Exception(f"No token endpoint for {provider}, {tenant}")
+        # Force backchannel: hit Keycloak directly (skip the app same-origin proxy)
+        # so the FAPI client cert presented at TLS handshake actually reaches Keycloak.
+        token_endpoint = _rewrite_endpoint_to_origin(token_endpoint, _keycloak_backchannel_origin(user_registry[KG__user_registry__discovery_url]))
         allowed_algs = discovery.get("id_token_signing_alg_values_supported")
         if not allowed_algs:
             raise Exception(f"No allowed algs for {provider}, {tenant}")
@@ -782,7 +846,7 @@ WHERE
             })
 
         token_response = self.idp_session.post(
-            token_endpoint.replace("localhost", "host.docker.internal"),
+            token_endpoint,
             data=token_request_payload,
             **kwargs
         )
@@ -850,6 +914,47 @@ WHERE
         response.response_code = HTTPStatus.FOUND
         response.raw_headers["Location"] = oidc_state[KEY__redirect_uri]
 
+    def _run_federation_procedure(self, database_user_registry, application, schema,
+                                  account_id, provider, tenant, email, sub, id_payload):
+        # Runs the app database's federation procedure (e.g. linking the JAAQL account to an
+        # app user row). Safe to call on every login: the procedure is expected to be idempotent.
+        if database_user_registry is None or application is None or schema is None:
+            return
+
+        db_params = {"tenant": tenant, "application": application, "account_id": account_id, "provider": provider,
+                     "email": email, "sub": sub}
+        parameters = fetch_parameters_for_federation_procedure(self.jaaql_lookup_connection,
+                                                               database_user_registry[KG__database_user_registry__federation_procedure])
+        for claim in parameters:
+            claim_name = claim[KG__federation_procedure_parameter__name]
+            db_params[claim_name] = id_payload.get(claim_name, "")
+
+        procedure_name = database_user_registry[KG__database_user_registry__federation_procedure]
+        if re.match(REGEX__dmbs_procedure_name, procedure_name) is None:
+            raise HttpStatusException("Unsafe data federation procedure")
+
+        procedure_params = []
+        for key, _ in db_params.items():
+            if re.match(REGEX__dmbs_object_name, key) is None:
+                raise HttpStatusException("Unsafe data federation parameter " + key)
+            procedure_params.append(f"{key} => :{key}")
+
+        submit_data = {
+            KEY__application: application,
+            KEY_query: f"SELECT * FROM \"{procedure_name}\"({', '.join(procedure_params)});",
+            KEY__schema: schema,
+            KEY__parameters: db_params
+        }
+
+        print("Preparing federating procedure")
+
+        submit(self.vault, self.config, self.get_db_crypt_key(),
+               self.jaaql_lookup_connection, submit_data, ROLE__jaaql,
+               None, self.cached_canned_query_service, as_objects=True, singleton=True)
+
+        print("Federated user")
+        print(submit_data)
+
     def _federate_and_issue_jwt(self, sub, provider, tenant, id_payload, ip_address, response,
                                 application=None, schema=None, database_user_registry=None):
         account = None
@@ -862,6 +967,18 @@ WHERE
                 email_verified = id_payload.get('email_verified') if require_email_verification else True
                 if email_verified:
                     mark_account_registered(self.jaaql_lookup_connection, account[KG__account__id])
+                    account[KG__account__email_verified] = True
+
+            # Re-run the federation procedure for already-known accounts so the app-side
+            # person<->account linkage self-heals when the app user was created after the
+            # account was first federated. Best-effort: a federation procedure failure (e.g. the
+            # app user does not exist yet) must not break an otherwise valid login.
+            try:
+                self._run_federation_procedure(database_user_registry, application, schema,
+                                               account[KG__account__id], provider, tenant,
+                                               id_payload.get('email'), sub, id_payload)
+            except Exception as e:
+                print(f"Federation procedure (existing-account re-link) skipped: {e}")
 
         except HttpSingletonStatusException:
             # User does not exist, federate it
@@ -874,40 +991,8 @@ WHERE
             print("new federated user with account id " + account_id)
             account = account__select(self.jaaql_lookup_connection, self.get_db_crypt_key(), account_id)
 
-            if database_user_registry is not None and application is not None and schema is not None:
-                db_params = {"tenant": tenant, "application": application, "account_id": account_id, "provider": provider,
-                             "email": email, "sub": sub}
-                parameters = fetch_parameters_for_federation_procedure(self.jaaql_lookup_connection,
-                                                                       database_user_registry[KG__database_user_registry__federation_procedure])
-                for claim in parameters:
-                    claim_name = claim[KG__federation_procedure_parameter__name]
-                    db_params[claim_name] = id_payload.get(claim_name, "")
-
-                procedure_name = database_user_registry[KG__database_user_registry__federation_procedure]
-                if re.match(REGEX__dmbs_procedure_name, procedure_name) is None:
-                    raise HttpStatusException("Unsafe data federation procedure")
-
-                procedure_params = []
-                for key, _ in db_params.items():
-                    if re.match(REGEX__dmbs_object_name, key) is None:
-                        raise HttpStatusException("Unsafe data federation parameter " + key)
-                    procedure_params.append(f"{key} => :{key}")
-
-                submit_data = {
-                    KEY__application: application,
-                    KEY_query: f"SELECT * FROM \"{procedure_name}\"({', '.join(procedure_params)});",
-                    KEY__schema: schema,
-                    KEY__parameters: db_params
-                }
-
-                print("Preparing federating procedure")
-
-                submit(self.vault, self.config, self.get_db_crypt_key(),
-                       self.jaaql_lookup_connection, submit_data, ROLE__jaaql,
-                       None, self.cached_canned_query_service, as_objects=True, singleton=True)
-
-                print("Federated user")
-                print(submit_data)
+            self._run_federation_procedure(database_user_registry, application, schema,
+                                           account_id, provider, tenant, email, sub, id_payload)
 
             # Insert into federation.federated_user on the app database
             if database_user_registry is not None:
@@ -987,7 +1072,7 @@ WHERE
                             attributes=get_cookie_attrs(self.vigilant_sessions, False, self.is_container),
                             is_https=self.is_https)
 
-        response.set_cookie(COOKIE_LOGIN_MARKER, value="true", is_https=True, attributes=get_sloppy_cookie_attrs())
+        response.set_cookie(COOKIE_LOGIN_MARKER, value="true", is_https=self.is_https, attributes=get_sloppy_cookie_attrs())
 
         return account, address
 
@@ -1125,6 +1210,15 @@ WHERE
         if not auth_endpoint:
             raise Exception("Authorization endpoint not found in discovery document")
 
+        # Rewrite host to the application's own origin so the OIDC login iframe
+        # stays same-origin with the parent page (password managers refuse to
+        # autofill cross-origin iframes). The nginx proxy in the app container
+        # forwards /realms/* and /resources/* back to the identity host.
+        # Only the host changes; the path (/realms/<R>/protocol/openid-connect/auth)
+        # is preserved verbatim so the proxy can route it as-is. PAR, token,
+        # JWKS and discovery endpoints are server-to-server and stay on identity.
+        auth_endpoint = _rewrite_endpoint_to_origin(auth_endpoint, application[KG__application__base_url])
+
         parameters = fetch_parameters_for_federation_procedure(self.jaaql_lookup_connection,
                                                                database_user_registry[KG__database_user_registry__federation_procedure])
         scope_list = [parameter[KG__federation_procedure_parameter__name] for parameter in parameters]
@@ -1177,7 +1271,11 @@ WHERE
             par_endpoint = discovery.get("pushed_authorization_request_endpoint")
             if not par_endpoint:
                 raise Exception("Pushed Authorization Request endpoint not found in discovery document.")
-            par_endpoint = par_endpoint.replace("localhost", "host.docker.internal")
+            # Force backchannel calls to go DIRECT to Keycloak (skip the app's
+            # same-origin proxy, which would terminate mTLS at the app's nginx
+            # and never forward the FAPI client cert to identity). See
+            # _keycloak_backchannel_origin.
+            par_endpoint = _rewrite_endpoint_to_origin(par_endpoint, _keycloak_backchannel_origin(user_registry[KG__user_registry__discovery_url]))
 
             par_payload = {
                 "client_id": database_user_registry[KG__database_user_registry__client_id],
@@ -1274,15 +1372,18 @@ WHERE
             with open(file_path, 'r') as file:
                 lines = file.readlines()
 
+            # Strip existing QUIC/HTTP3 lines so they can be cleanly re-added
+            # in the second pass without preventing the config update.
+            lines = [l for l in lines if "listen 443 quic" not in l
+                     and "listen [::]:443 quic" not in l
+                     and "http3 on;" not in l.strip()
+                     and not (l.strip().startswith("add_header Alt-Svc") and "h3=" in l)]
+
             # Initialize variables to track the current state and to store updated lines
             in_section = False
             updated_lines = []
 
-            do_write = True
-
             for line in lines:
-                if "listen 443 quic" in line:
-                    do_write = False
                 if line.strip().startswith('charset'):
                     in_section = True
                     continue  # Skip to the next iteration
@@ -1312,10 +1413,18 @@ WHERE
                 if not in_localhost or not line.strip().startswith("limit_req"):  # Ignore limit req for local requests
                     second_iteration_updated_lines.append(line)
 
+            # Defensive: if the input had `listen 443 ssl` (Certbot installed it) but
+            # our output dropped it, refuse to write. Better to leave Certbot's working
+            # config in place than overwrite with a broken one.
+            input_had_ssl = any('listen 443 ssl' in l for l in open(file_path).readlines())
+            output_has_ssl = any('listen 443 ssl' in l for l in second_iteration_updated_lines)
+            if input_had_ssl and not output_has_ssl:
+                print('set_web_config aborting: would have lost listen 443 ssl', file=sys.stderr)
+                return
+
             # Write the updated content back to the file
-            if do_write:
-                with open(file_path, 'w') as file:
-                    file.writelines(second_iteration_updated_lines)
+            with open(file_path, 'w') as file:
+                file.writelines(second_iteration_updated_lines)
 
             subprocess.call(['nginx', '-s', 'reload'])
         else:
@@ -1687,6 +1796,14 @@ WHERE
                                         submit_data, account_id, None, self.cached_canned_query_service,
                                         as_objects=True, singleton=True)
 
+        # Inject the JAAQL system email params so templates can deep-link back to
+        # the app via {{JAAQL__APP_URL}} (and {{JAAQL__APP_NAME}} / {{JAAQL__EMAIL_ADDRESS}}).
+        # The replacement set was otherwise the data-view row only, so these tokens
+        # went unresolved on every send path (e.g. cloud-proc alert mail).
+        email_replacement_data[EMAIL_PARAM__app_url] = self.replace_default_app_url(app[KG__application__base_url])
+        email_replacement_data[EMAIL_PARAM__app_name] = app[KG__application__name]
+        email_replacement_data[EMAIL_PARAM__email_address] = username
+
         document_templates = fetch_document_templates_for_email_template(self.jaaql_lookup_connection, inputs[KEY__application],
                                                                          inputs[KEY__template])
         attachments = [
@@ -2074,6 +2191,17 @@ WHERE
                 finally:
                     app_connection.close()
 
+                # Link the freshly-created app user (the person row created by the gate procedure
+                # above) to this account now, instead of waiting for a first login that will never
+                # trigger federation (the account already exists by then). Best-effort: do not fail
+                # user creation if the federation procedure errors.
+                try:
+                    self._run_federation_procedure(registry, application, "default",
+                                                   new_account_id, provider, tenant,
+                                                   username, user_id, {"email": username})
+                except Exception as e:
+                    print(f"Federation procedure (admin create user) skipped: {e}")
+
         return {
             "temporary_password": temp_pw,
             "response": response_obj,
@@ -2329,7 +2457,7 @@ WHERE
             application = application__select(self.jaaql_lookup_connection, inputs[KEY__application])
             redirect_uri = inputs.get(KEY__redirect_uri, "")
             post_logout_url = application[KG__application__base_url] + "/" + redirect_uri
-            response.set_cookie(COOKIE_LOGIN_MARKER, value="", is_https=True, attributes=get_sloppy_cookie_attrs())
+            response.set_cookie(COOKIE_LOGIN_MARKER, value="", is_https=self.is_https, attributes=get_sloppy_cookie_attrs())
             response.response_code = HTTPStatus.FOUND
             response.raw_headers["Location"] = "/.auth/logout?post_logout_redirect_uri=" + urllib.parse.quote(post_logout_url)
             return
@@ -2393,7 +2521,7 @@ WHERE
         # Kill your own app session immediately
         response.delete_cookie(COOKIE_JAAQL_AUTH, self.is_https)
         # Optional: clear the UI helper marker
-        response.set_cookie(COOKIE_LOGIN_MARKER, value="", is_https=True, attributes=get_sloppy_cookie_attrs())
+        response.set_cookie(COOKIE_LOGIN_MARKER, value="", is_https=self.is_https, attributes=get_sloppy_cookie_attrs())
 
         # Build front-channel logout URL (Keycloak requires either id_token_hint OR client_id with post_logout_redirect_uri)
         url = urllib.parse.urlsplit(end_session)

jaaql/scripts/03.install_super_user.handwritten.sql

@@ -27,26 +27,36 @@ BEGIN
 
         -- Federation schema: stores OIDC identity linkage for federated/invited users
         PERFORM dblink_exec('dbname=' || database, 'CREATE SCHEMA IF NOT EXISTS federation;');
+        -- The public-schema copies are kept for backwards compatibility with code that
+        -- references the domains unqualified. The federation-schema copies are what the
+        -- federation tables themselves type against, so they survive a `DROP SCHEMA public
+        -- CASCADE` (e.g. from JAAQL's own folder-migration step in migrations_manager_service.py).
         PERFORM dblink_exec('dbname=' || database, '
             DO $d$ BEGIN CREATE DOMAIN postgres_user_id AS character varying(63); EXCEPTION WHEN duplicate_object THEN NULL; END $d$;
         ');
         PERFORM dblink_exec('dbname=' || database, '
             DO $d$ BEGIN CREATE DOMAIN oidc_subject_id AS character varying(255); EXCEPTION WHEN duplicate_object THEN NULL; END $d$;
         ');
+        PERFORM dblink_exec('dbname=' || database, '
+            DO $d$ BEGIN CREATE DOMAIN federation.postgres_user_id AS character varying(63); EXCEPTION WHEN duplicate_object THEN NULL; END $d$;
+        ');
+        PERFORM dblink_exec('dbname=' || database, '
+            DO $d$ BEGIN CREATE DOMAIN federation.oidc_subject_id AS character varying(255); EXCEPTION WHEN duplicate_object THEN NULL; END $d$;
+        ');
         PERFORM dblink_exec('dbname=' || database, '
             CREATE TABLE IF NOT EXISTS federation.account (
-                account_id      postgres_user_id       NOT NULL,
+                account_id      federation.postgres_user_id       NOT NULL,
                 PRIMARY KEY (account_id)
             );
         ');
         PERFORM dblink_exec('dbname=' || database, '
             CREATE TABLE IF NOT EXISTS federation.federated_user (
-                account_id      postgres_user_id       NOT NULL,
-                sub             oidc_subject_id        NOT NULL,
-                registered_at   timestamptz            NOT NULL DEFAULT NOW(),
-                tenant          character varying(256)  NOT NULL,
-                provider        character varying(256)  NOT NULL,
-                is_active       boolean                NOT NULL DEFAULT TRUE,
+                account_id      federation.postgres_user_id       NOT NULL,
+                sub             federation.oidc_subject_id        NOT NULL,
+                registered_at   timestamptz                       NOT NULL DEFAULT NOW(),
+                tenant          character varying(256)            NOT NULL,
+                provider        character varying(256)            NOT NULL,
+                is_active       boolean                           NOT NULL DEFAULT TRUE,
                 encrypted_email character varying(512),
                 PRIMARY KEY (tenant, provider, sub),
                 FOREIGN KEY (account_id) REFERENCES federation.account (account_id)

jaaql/services/migrations_manager_service.py

@@ -393,16 +393,64 @@ def _run_folder_migration(folder_path: str, folder_name: str,
     _run_psql_file(pre_wipe, "dba", app_database)
 
     # Step 3: Wipe public schema.
+    #
+    # DROP SCHEMA public CASCADE wipes any extension installed in public (unaccent,
+    # pg_trgm, pgcrypto, ...) AND any column typed against a public-schema domain.
+    # JAAQL's federation tables historically typed account_id/sub against
+    # public.postgres_user_id / public.oidc_subject_id, so the cascade silently
+    # removed those columns. We snapshot extensions before the drop and restore
+    # them afterwards, and migrate the federation column types onto the federation-
+    # schema copies of the domains so they survive the wipe in future.
     print(f"  Wiping public schema", file=sys.stderr)
+
+    public_extensions_raw = _run_psql_query(
+        "SELECT extname FROM pg_extension "
+        "WHERE extnamespace = 'public'::regnamespace AND extname <> 'plpgsql_check' "
+        "ORDER BY extname;",
+        "postgres", app_database,
+    )
+    public_extensions = [e.strip() for e in public_extensions_raw.split("\n") if e.strip()]
+    print(f"  Preserving public-schema extensions: {public_extensions or '(none)'}", file=sys.stderr)
+
+    # Re-house federation domains and any federation columns that still type against
+    # the public-schema versions. Idempotent: healthy / already-migrated systems no-op.
     _run_psql_query(
+        "CREATE SCHEMA IF NOT EXISTS federation; "
+        "DO $d$ BEGIN CREATE DOMAIN federation.postgres_user_id AS character varying(63);  EXCEPTION WHEN duplicate_object THEN NULL; END $d$; "
+        "DO $d$ BEGIN CREATE DOMAIN federation.oidc_subject_id  AS character varying(255); EXCEPTION WHEN duplicate_object THEN NULL; END $d$; "
+        "DO $alter$ BEGIN "
+        "  IF EXISTS (SELECT 1 FROM information_schema.columns "
+        "             WHERE table_schema='federation' AND table_name='account' "
+        "             AND column_name='account_id' AND udt_schema='public' AND udt_name='postgres_user_id') THEN "
+        "    ALTER TABLE federation.account ALTER COLUMN account_id TYPE federation.postgres_user_id; "
+        "  END IF; "
+        "  IF EXISTS (SELECT 1 FROM information_schema.columns "
+        "             WHERE table_schema='federation' AND table_name='federated_user' "
+        "             AND column_name='account_id' AND udt_schema='public' AND udt_name='postgres_user_id') THEN "
+        "    ALTER TABLE federation.federated_user ALTER COLUMN account_id TYPE federation.postgres_user_id; "
+        "  END IF; "
+        "  IF EXISTS (SELECT 1 FROM information_schema.columns "
+        "             WHERE table_schema='federation' AND table_name='federated_user' "
+        "             AND column_name='sub' AND udt_schema='public' AND udt_name='oidc_subject_id') THEN "
+        "    ALTER TABLE federation.federated_user ALTER COLUMN sub TYPE federation.oidc_subject_id; "
+        "  END IF; "
+        "END $alter$;",
+        "postgres", app_database,
+    )
+
+    drop_recreate = (
         "SET search_path = public, pg_catalog; "
         "DROP EXTENSION IF EXISTS plpgsql_check; "
         "DROP SCHEMA IF EXISTS public CASCADE; "
         "CREATE SCHEMA public; "
         "GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC; "
-        "CREATE EXTENSION IF NOT EXISTS plpgsql_check SCHEMA public;",
-        "postgres", app_database,
+        "CREATE EXTENSION IF NOT EXISTS plpgsql_check SCHEMA public; "
     )
+    for ext in public_extensions:
+        # extname comes from pg_extension; safe to interpolate as a quoted identifier.
+        drop_recreate += f'CREATE EXTENSION IF NOT EXISTS "{ext}" SCHEMA public; '
+
+    _run_psql_query(drop_recreate, "postgres", app_database)
 
     # Step 4: Rebuild schema from data model (migration pt1).
     print(f"  Rebuilding schema via migration pt1: {pt1}", file=sys.stderr)

{jaaql_middleware_python-5.1.0.dist-info → jaaql_middleware_python-5.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jaaql-middleware-python
-Version: 5.1.0
+Version: 5.2.7
 Summary: The jaaql package, allowing for rapid development and deployment of RESTful HTTP applications
 Home-page: https://github.com/JAAQL/JAAQL-middleware-python
 Author: Software Quality Measurement and Improvement bv

{jaaql_middleware_python-5.1.0.dist-info → jaaql_middleware_python-5.2.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 jaaql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/config_constants.py,sha256=puoR6jfbXw8oXXEfbYNn2ysGwgiGTyuDjOj7F8acsTE,470
-jaaql/constants.py,sha256=XmSmSc2EHXh08i4RO0-iOD56QpjUTdrAxFtyN_-Mzzc,6976
+jaaql/constants.py,sha256=Me_r4pBC0TxJZsC4uCYdI23WIgs5YnTkuvqLWPOfUu8,6994
 jaaql/generated_constants.py,sha256=pxmpTux7rrjJH6iqLb-R8RKJ4t0CeoGdnspjH5DiWgU,11327
 jaaql/jaaql.py,sha256=0jTYtpDYgKRNKU889wMI495k_h-EzQuGEQqrqHNhNrg,5829
 jaaql/patch.py,sha256=VE4B6jA492NCIxNIewbynh92jxVulmRFtB9hNEe6q94,149
@@ -10,11 +10,11 @@ jaaql/config/config-test.ini,sha256=ap9OFqCCbYdWqmUKq0p9GD1IkM4CzeTXDV5gbQGQZoQ,
 jaaql/config/config.ini,sha256=o1orXNYRFgf09MKzZBQ3v0oMzWn7N6KwvX5YV0WJtns,321
 jaaql/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/db/db_interface.py,sha256=A_daGWC6hP3M1EewNuUU5o5PL_ab2EIzrm3ZDoFxddQ,7639
-jaaql/db/db_pg_interface.py,sha256=bxHnEg0X6VgpWwhbYYk9YHMWsAMryLlRu9-84tNiZu8,11516
+jaaql/db/db_pg_interface.py,sha256=rzoJw_XTxIqGtuQjawNxfgRAaxQs1m1MoKMOibEK0tY,12643
 jaaql/db/db_utils.py,sha256=xZCdtWT9semzLL-NAiwMFx_38Wv96BBv60pfPYNDnCc,7761
 jaaql/db/db_utils_no_circ.py,sha256=E3jxF4O0KXUnK9yxLQYxfqDTfkAwBFJSS3sX12A1row,4593
 jaaql/documentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jaaql/documentation/documentation_internal.py,sha256=cT_AI5z1_wgZb_Ml5C3DU-F5cnZVjATfv_RyewKCDVk,20436
+jaaql/documentation/documentation_internal.py,sha256=ecqvo-lN1Kl0_ce-Ohj75-nZFS6GmIActJjBC-sg1pM,22404
 jaaql/documentation/documentation_public.py,sha256=XU1Smw-1heWhCs-FDqTCTVCkAI2COHEy6fk2fhrwFN4,10932
 jaaql/documentation/documentation_shared.py,sha256=gTw3rT2hC6LTlJMXnwmxhSTh3xUetIJPtIhldptXF84,6654
 jaaql/email/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -33,19 +33,19 @@ jaaql/migrations/migrations.py,sha256=XE46AzNmfAg9qiatIL_6ctjqiiuVMHUNEcVHhmuh4h
 jaaql/mvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/mvc/base_controller.py,sha256=FCBX6pVkUh3yeLHWbw_2ht2G0L2L3vTdk2T_qcC47cI,39102
 jaaql/mvc/base_model.py,sha256=ovNPYpd6TmJfC7qSAAxVXeejv5qVBC2Ht8qAoNR5pxE,15609
-jaaql/mvc/controller.py,sha256=jKiUn9r6neuchKGhlrHqa8cvI1rG7G5B_Eh7OJOVXS0,12587
+jaaql/mvc/controller.py,sha256=lnZBYmwzJd81bdiCkic1asOa_2veEr0U-M_vNFFTS9w,14405
 jaaql/mvc/controller_interface.py,sha256=Wx8fA4lrqb_PPwkLNjJDxDloHB5JxmNt1Gv3SsDFqT4,274
 jaaql/mvc/exception_queries.py,sha256=lF65cj9KgJgRvyzxDjPDM05rHFSHF0jy1V8B77H9Rvo,7899
 jaaql/mvc/generated_queries.py,sha256=z1Jsyu-Y6D-Xd8LYvefzwv8C01NaBjHaHI7mcvv-WEk,72232
 jaaql/mvc/handmade_queries.py,sha256=kyXZ7L_AoxfcPR4u3r5So8khlRD6_sGQ39JnSAHDsEA,938
-jaaql/mvc/model.py,sha256=Vo4fvMbhslqw2I6kbTNktZLV8I2DJhdAlkMnEu3uNxA,121672
+jaaql/mvc/model.py,sha256=p_gAw5_0GAb14PPKs1O99mRfYjtL7f3BVqyaMLcgKT4,129446
 jaaql/mvc/model_interface.py,sha256=YgAjaAiIQL1UtIJYgvkl_C3r6PVo-8Z3MXtwzyIwKnc,257
 jaaql/mvc/response.py,sha256=F4WvR9h1JcWF3Y6cj3n5HpVIqnwiMpNqfE8D29ZHbSE,860
 jaaql/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/openapi/swagger_documentation.py,sha256=J1YwR36JJGNJGqPEUMraDoeqgldRLy1_2xR-i_ZPtmQ,29021
 jaaql/scripts/01.install_domains.generated.sql,sha256=9queSyDfgMEwy7G3GV4orWlnt9VCZ6jWa_ZbYOR_w-0,3005
 jaaql/scripts/02.install_super_user.exceptions.sql,sha256=XAHnq_3hVyLn0IZm5j_BNn-Ea2NBDkRqo-Wonk7SNuU,3198
-jaaql/scripts/03.install_super_user.handwritten.sql,sha256=VjpogeW1Vor1WmY3ONGEcea920X6xi-3QLyQI_nyRl4,5488
+jaaql/scripts/03.install_super_user.handwritten.sql,sha256=OrBRxh_QLk8FQnG9KvgFqe_hY1A0qhko2mGav7lu1qM,6352
 jaaql/scripts/04.install_jaaql_data_structures.generated.sql,sha256=isnGN4otZNPn1FwCM-W-r3HBj1tLgnp7jx47tF_maq8,7847
 jaaql/scripts/05.install_static_data.generated.sql,sha256=v6b6at8Ju5S750-uFwdvL4TvIrUAOmCutJ7eNTMreP8,18989
 jaaql/scripts/06.install_jaaql.exceptions.sql,sha256=A_N1KoTmPFcQ2KwGgkD4AcAFrs8nELcQGDm96sINBSs,324
@@ -54,7 +54,7 @@ jaaql/scripts/ZZZZ.reset_references.sql,sha256=Va0v04IBRUa7x_eE43jLO2xUQMHwYM4NP
 jaaql/scripts/swagger_template.html,sha256=Os7U11wiww5sk1J_wINbyC2id6yvvV6rbxASd5iUWiA,4198
 jaaql/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jaaql/services/cached_canned_query_service.py,sha256=Kq_0WwtUXBGhYJB4uhSKBn8KXuX0LHrfsBP90NZQRuc,2688
-jaaql/services/migrations_manager_service.py,sha256=ReXS2kN49OeVZWK3krzhXsrL1MTG2uyofZYGNCswIho,22039
+jaaql/services/migrations_manager_service.py,sha256=GkzX8traQcV0Q6TBp9uf8VPM8hMldlAnqXWp0FkwSIg,25072
 jaaql/services/patch_mms.py,sha256=chae_LNafHuOhewgJrSRe-FnxBi1kgs4kK6lyn-V59A,296
 jaaql/services/patch_shared_var_service.py,sha256=qZw6Kl86Tvz3_YK-Y1VZUx8T28-lg80qiwOEkXmlCFw,117
 jaaql/services/shared_var_service.py,sha256=gcaj1x5C5m3DOcV85jrkIe4Uc_C8mW9nNdqvdHb0eXs,1109
@@ -66,8 +66,8 @@ jaaql/utilities/options.py,sha256=75M7-oSeY4wmHGz3-su_c36yf_Csv9HxvTG7aRlGR0A,49
 jaaql/utilities/utils.py,sha256=nVYVhuHtWSjdaJaXe9eC1Usja56NO-V9SoBIlcHAK8Q,5166
 jaaql/utilities/utils_no_project_imports.py,sha256=YwcskR15dAqKdEiuF8zGx-tzwWkclSGil_zm_qZ8ylU,4626
 jaaql/utilities/vault.py,sha256=DudyQNtMtaRLL71IGQX0AI16Tg1id98BHTCQPyUNkmE,2243
-jaaql_middleware_python-5.1.0.dist-info/licenses/LICENSE.txt,sha256=wr50eN0TnkrCSuNwyiI81FLZqmrW3q0JqRlVVQIrGA0,18134
-jaaql_middleware_python-5.1.0.dist-info/METADATA,sha256=Sx9SWssXTsTpEkmqqBr2V6j1SjasGRtpdh5bGeTqKd4,1945
-jaaql_middleware_python-5.1.0.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-jaaql_middleware_python-5.1.0.dist-info/top_level.txt,sha256=k5aKVQy77bPyfyNHITvo5m3FqdSAiiXbjT5S9D9KzgU,6
-jaaql_middleware_python-5.1.0.dist-info/RECORD,,
+jaaql_middleware_python-5.2.7.dist-info/licenses/LICENSE.txt,sha256=wr50eN0TnkrCSuNwyiI81FLZqmrW3q0JqRlVVQIrGA0,18134
+jaaql_middleware_python-5.2.7.dist-info/METADATA,sha256=kRCQ9esMuc6bK2pFTKu3optqYTDsgwnkFxDGn00_gLI,1945
+jaaql_middleware_python-5.2.7.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+jaaql_middleware_python-5.2.7.dist-info/top_level.txt,sha256=k5aKVQy77bPyfyNHITvo5m3FqdSAiiXbjT5S9D9KzgU,6
+jaaql_middleware_python-5.2.7.dist-info/RECORD,,