c2cgeoportal-geoportal 2.5.0.100__py2.py3-none-any.whl → 2.7.1.83__py2.py3-none-any.whl

c2cgeoportal_geoportal/lib/lingua_extractor.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2011-2020, Camptocamp SA
+# Copyright (c) 2011-2023, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -33,11 +31,13 @@ import os
 import re
 import subprocess
 import traceback
-from typing import Dict, List, Optional, Set, cast
-from urllib.parse import urlsplit
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, cast
 from xml.dom import Node
 from xml.parsers.expat import ExpatError
 
+import pyramid.threadlocal
+import requests
+import yaml
 from bottle import MakoTemplate, template
 from c2c.template.config import config
 from defusedxml.minidom import parseString
@@ -46,66 +46,84 @@ from lingua.extractors import Extractor, Message
 from mako.lookup import TemplateLookup
 from mako.template import Template
 from owslib.wms import WebMapService
-import requests
-import sqlalchemy
 from sqlalchemy.exc import NoSuchTableError, OperationalError, ProgrammingError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.properties import ColumnProperty
+from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.util import class_mapper
-import yaml
 
-import c2cgeoportal_commons.models
 import c2cgeoportal_geoportal
-from c2cgeoportal_geoportal.lib import add_url_params, get_url2
-from c2cgeoportal_geoportal.lib.bashcolor import RED, colorize
+from c2cgeoportal_commons.lib.url import Url, get_url2
+from c2cgeoportal_geoportal.lib.bashcolor import Color, colorize
 from c2cgeoportal_geoportal.lib.caching import init_region
 from c2cgeoportal_geoportal.views.layers import Layers, get_layer_class
 
+if TYPE_CHECKING:
+    from c2cgeoportal_commons.models import main  # pylint: disable=ungrouped-imports,useless-suppression
+
+
+def _get_config(key: str, default: Optional[str] = None) -> Optional[str]:
+    """
+    Return the config value for passed key.
+
+    Passed throw environment variable for the command line,
+    or throw the query string on HTTP request.
+    """
+    request = pyramid.threadlocal.get_current_request()
+    if request is not None:
+        return cast(Optional[str], request.params.get(key.lower(), default))
+
+    return os.environ.get(key, default)
+
+
+def _get_config_str(key: str, default: str) -> str:
+    result = _get_config(key, default)
+    assert result is not None
+    return result
+
 
-class _Registry:  # pragma: no cover
+class _Registry:
     settings = None
 
-    def __init__(self, settings):
+    def __init__(self, settings: Optional[Dict[str, Any]]) -> None:
         self.settings = settings
 
 
-class _Request:  # pragma: no cover
+class _Request:
     params: Dict[str, str] = {}
     matchdict: Dict[str, str] = {}
     GET: Dict[str, str] = {}
 
-    def __init__(self, settings=None):
+    def __init__(self, settings: Optional[Dict[str, Any]] = None) -> None:
         self.registry: _Registry = _Registry(settings)
 
     @staticmethod
-    def static_url(*args, **kwargs):
+    def static_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def static_path(*args, **kwargs):
+    def static_path(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def route_url(*args, **kwargs):
+    def route_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def current_route_url(*args, **kwargs):
+    def current_route_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
 
-class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
-    """
-    GeoMapFish angular extractor
-    """
+class GeomapfishAngularExtractor(Extractor):  # type: ignore
+    """GeoMapFish angular extractor."""
 
     extensions = [".js", ".html"]
 
@@ -118,27 +136,49 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
             self.config = None
         self.tpl = None
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
+    @staticmethod
+    def get_message_cleaner(filename: str) -> Callable[[str], str]:
+        """Return a function for cleaning messages according to input file format."""
+        ext = os.path.splitext(filename)[1]
+
+        if ext in [".html", ".ejs"]:
+            # Remove \n in HTML multi-line strings
+            pattern = re.compile("\n *")
+            return lambda s: re.sub(pattern, " ", s)
+
+        return lambda s: s
+
+    def __call__(
+        self,
+        filename: str,
+        options: Dict[str, Any],
+        fileobj: Optional[Dict[str, Any]] = None,
+        lineno: int = 0,
+    ) -> List[Message]:
         del fileobj, lineno
 
+        print(f"Running {self.__class__.__name__} on {filename}")
+
+        cleaner = self.get_message_cleaner(filename)
+
         init_region({"backend": "dogpile.cache.memory"}, "std")
         init_region({"backend": "dogpile.cache.memory"}, "obj")
 
         int_filename = filename
-        if re.match("^" + re.escape("./{}/templates".format(self.config["package"])), filename):
+        if re.match("^" + re.escape(f"./{self.config['package']}/templates"), filename):
             try:
                 empty_template = Template("")  # nosec
 
-                class Lookup(TemplateLookup):
+                class Lookup(TemplateLookup):  # type: ignore
                     @staticmethod
-                    def get_template(uri):
+                    def get_template(uri: str) -> Template:
                         del uri  # unused
                         return empty_template
 
-                class MyTemplate(MakoTemplate):
+                class MyTemplate(MakoTemplate):  # type: ignore
                     tpl = None
 
-                    def prepare(self, **kwargs):
+                    def prepare(self, **kwargs: Any) -> None:
                         options.update({"input_encoding": self.encoding})
                         lookup = Lookup(**kwargs)
                         if self.source:
@@ -149,10 +189,12 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
                             )
 
                 try:
+                    request = pyramid.threadlocal.get_current_request()
+                    request = _Request() if request is None else request
                     processed = template(
                         filename,
                         {
-                            "request": _Request(self.config),
+                            "request": request,
                             "lang": "fr",
                             "debug": False,
                             "extra_params": {},
@@ -167,11 +209,9 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
                     with open(int_filename, "wb") as file_open:
                         file_open.write(processed.encode("utf-8"))
                 except Exception:
-                    print(
-                        colorize("ERROR! Occurred during the '{}' template generation".format(filename), RED)
-                    )
-                    print(colorize(traceback.format_exc(), RED))
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                    print(colorize(f"ERROR! Occurred during the '{filename}' template generation", Color.RED))
+                    print(colorize(traceback.format_exc(), Color.RED))
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                         # Continue with the original one
                         int_filename = filename
                     else:
@@ -179,37 +219,85 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
             except Exception:
                 print(traceback.format_exc())
 
-        message_str = subprocess.check_output(
-            ["node", "geoportal/tools/extract-messages.js", int_filename]
-        ).decode("utf-8")
+        # Path in geomapfish-tools
+        script_path = "geoportal/tools/extract-messages.js"
+        if not os.path.isfile(script_path):
+            # Path in geomapfish runner
+            script_path = "/app/tools/extract-messages.js"
+        message_str = subprocess.check_output(["node", script_path, int_filename]).decode("utf-8")
         if int_filename != filename:
             os.unlink(int_filename)
         try:
             messages = []
             for contexts, message in json.loads(message_str):
+                assert message is not None
+                message = cleaner(message)
                 for context in contexts.split(", "):
                     messages.append(Message(None, message, None, [], "", "", context.split(":")))
             return messages
         except Exception:
-            print(colorize("An error occurred", RED))
-            print(colorize(message_str, RED))
+            print(colorize("An error occurred", Color.RED))
+            print(colorize(message_str, Color.RED))
             print("------")
             raise
 
 
-class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
+def init_db(settings: Dict[str, Any]) -> None:
     """
-    GeoMapFish config extractor (raster layers, and print templates)
+    Initialize the SQLAlchemy Session.
+
+    First test the connection, on wen environment it should be OK, with the command line we should get
+    an exception ind initialise the connection.
     """
 
+    try:
+        from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
+        from c2cgeoportal_commons.models.main import Theme  # pylint: disable=import-outside-toplevel
+
+        session = DBSession()
+        session.query(Theme).count()
+    except:  # pylint: disable=bare-except
+
+        # Init db sessions
+
+        class R:
+            settings: Dict[str, Any] = {}
+
+        class C:
+            registry = R()
+
+            def get_settings(self) -> Dict[str, Any]:
+                return self.registry.settings
+
+            def add_tween(self, *args: Any, **kwargs: Any) -> None:
+                pass
+
+        config_ = C()
+        config_.registry.settings = settings
+
+        c2cgeoportal_geoportal.init_db_sessions(settings, config_)
+
+
+class GeomapfishConfigExtractor(Extractor):  # type: ignore
+    """GeoMapFish config extractor (raster layers, and print templates)."""
+
     extensions = [".yaml", ".tmpl"]
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
-        del fileobj, lineno
+    def __call__(
+        self,
+        filename: str,
+        options: Dict[str, Any],
+        fileobj: Optional[Dict[str, Any]] = None,
+        lineno: int = 0,
+    ) -> List[Message]:
+        del options, fileobj, lineno
+
+        print(f"Running {self.__class__.__name__} on {filename}")
+
         init_region({"backend": "dogpile.cache.memory"}, "std")
         init_region({"backend": "dogpile.cache.memory"}, "obj")
 
-        with open(filename) as config_file:
+        with open(filename, encoding="utf8") as config_file:
             gmf_config = yaml.load(config_file, Loader=yaml.BaseLoader)  # nosec
             # For application config (config.yaml)
             if "vars" in gmf_config:
@@ -219,37 +307,26 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
                 return self._collect_print_config(gmf_config, filename)
             raise Exception("Not a known config file")
 
-    def _collect_app_config(self, filename):
+    def _collect_app_config(self, filename: str) -> List[Message]:
         config.init(filename)
         settings = config.get_config()
+        assert not [
+            raster_layer for raster_layer in list(settings.get("raster", {}).keys()) if raster_layer is None
+        ]
         # Collect raster layers names
         raster = [
-            Message(None, raster_layer, None, [], "", "", (filename, "raster/{}".format(raster_layer)))
+            Message(None, raster_layer, None, [], "", "", (filename, f"raster/{raster_layer}"))
             for raster_layer in list(settings.get("raster", {}).keys())
         ]
 
-        # Init db sessions
-
-        class R:
-            settings = None
-
-        class C:
-            registry = R()
-
-            def get_settings(self):
-                return self.registry.settings
-
-            def add_tween(self, *args, **kwargs):
-                pass
-
-        config_ = C()
-        config_.registry.settings = settings
-
-        c2cgeoportal_geoportal.init_dbsessions(settings, config_)
+        init_db(settings)
 
         # Collect layers enum values (for filters)
 
-        from c2cgeoportal_commons.models import DBSessions  # pylint: disable=import-outside-toplevel
+        from c2cgeoportal_commons.models import (  # pylint: disable=import-outside-toplevel
+            DBSession,
+            DBSessions,
+        )
         from c2cgeoportal_commons.models.main import Metadata  # pylint: disable=import-outside-toplevel
 
         enums = []
@@ -258,61 +335,95 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
             layerinfos = enum_layers.get(layername, {})
             attributes = layerinfos.get("attributes", {})
             for fieldname in list(attributes.keys()):
-                values = self._enumerate_attributes_values(DBSessions, Layers, layerinfos, fieldname)
+                values = self._enumerate_attributes_values(DBSessions, layerinfos, fieldname)
                 for (value,) in values:
                     if isinstance(value, str) and value != "":
                         msgid = value
-                        location = "/layers/{}/values/{}/{}".format(
-                            layername, fieldname, value.encode("ascii", errors="replace").decode("ascii"),
+                        location = (
+                            f"/layers/{layername}/values/{fieldname}/"
+                            f"{value.encode('ascii', errors='replace').decode('ascii')}"
                         )
+                        assert msgid is not None
                         enums.append(Message(None, msgid, None, [], "", "", (filename, location)))
 
         metadata_list = []
-        defs = config["admin_interface"]["available_metadata"]
+        defs = config["admin_interface"]["available_metadata"]  # pylint: disable=unsubscriptable-object
         names = [e["name"] for e in defs if e.get("translate", False)]
 
         if names:
-            engine = sqlalchemy.create_engine(config["sqlalchemy.url"])
-            Session = sqlalchemy.orm.session.sessionmaker()  # noqa
-            Session.configure(bind=engine)
-            session = Session()
+            session = DBSession()
 
             query = session.query(Metadata).filter(Metadata.name.in_(names))
             for metadata in query.all():
-                location = "metadata/{}/{}".format(metadata.name, metadata.id)
+                location = f"metadata/{metadata.name}/{metadata.id}"
+                assert metadata.value is not None
                 metadata_list.append(Message(None, metadata.value, None, [], "", "", (filename, location)))
 
-        return raster + enums + metadata_list
+        interfaces_messages = []
+        for interface, interface_config in config["interfaces_config"].items():
+            for ds_index, datasource in enumerate(
+                interface_config.get("constants", {}).get("gmfSearchOptions", {}).get("datasources", [])
+            ):
+                for a_index, action in enumerate(datasource.get("groupActions", [])):
+                    location = (
+                        f"interfaces_config/{interface}/constants/gmfSearchOptions/datasources[{ds_index}]/"
+                        f"groupActions[{a_index}]/title"
+                    )
+                    assert action["title"] is not None
+                    interfaces_messages.append(
+                        Message(None, action["title"], None, [], "", "", (filename, location))
+                    )
+
+            for merge_tab in (
+                interface_config.get("constants", {})
+                .get("gmfDisplayQueryGridOptions", {})
+                .get("mergeTabs", {})
+                .keys()
+            ):
+                location = (
+                    f"interfaces_config/{interface}/constants/gmfDisplayQueryGridOptions/"
+                    f"mergeTabs/{merge_tab}/"
+                )
+                assert merge_tab is not None
+                interfaces_messages.append(Message(None, merge_tab, None, [], "", "", (filename, location)))
+
+        return raster + enums + metadata_list + interfaces_messages
 
     @staticmethod
-    def _enumerate_attributes_values(dbsessions, layers, layerinfos, fieldname):
+    def _enumerate_attributes_values(
+        dbsessions: Dict[str, Session], layerinfos: Dict[str, Any], fieldname: str
+    ) -> Set[Tuple[str, ...]]:
         dbname = layerinfos.get("dbsession", "dbsession")
-        translate = layerinfos.get("attributes").get(fieldname, {}).get("translate", True)
+        translate = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("translate", True)
         if not translate:
-            return []
+            return set()
         try:
             dbsession = dbsessions.get(dbname)
-            return layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname)
+            return Layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname)
         except Exception as e:
-            table = layerinfos.get("attributes").get(fieldname, {}).get("table")
+            table = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("table")
             print(
                 colorize(
                     "ERROR! Unable to collect enumerate attributes for "
-                    "db: {}, table: {}, column: {} ({})".format(dbname, table, fieldname, e),
-                    RED,
+                    f"db: {dbname}, table: {table}, column: {fieldname} ({e!s})",
+                    Color.RED,
                 )
             )
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
-                return []
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                return set()
             raise
 
     @staticmethod
-    def _collect_print_config(print_config, filename):
+    def _collect_print_config(print_config: Dict[str, Any], filename: str) -> List[Message]:
         result = []
-        for template_ in list(print_config.get("templates").keys()):
-            result.append(
-                Message(None, template_, None, [], "", "", (filename, "template/{}".format(template_)))
-            )
+        for template_ in list(cast(Dict[str, Any], print_config.get("templates")).keys()):
+            assert template_ is not None
+            result.append(Message(None, template_, None, [], "", "", (filename, f"template/{template_}")))
+            assert not [
+                attribute
+                for attribute in list(print_config["templates"][template_]["attributes"].keys())
+                if attribute is None
+            ]
             result += [
                 Message(
                     None,
@@ -321,22 +432,20 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
                     [],
                     "",
                     "",
-                    (filename, "template/{}/{}".format(template_, attribute)),
+                    (filename, f"template/{template_}/{attribute}"),
                 )
                 for attribute in list(print_config["templates"][template_]["attributes"].keys())
             ]
         return result
 
 
-class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
-    """
-    GeoMapFish theme extractor
-    """
+class GeomapfishThemeExtractor(Extractor):  # type: ignore
+    """GeoMapFish theme extractor."""
 
     # Run on the development.ini file
     extensions = [".ini"]
-    featuretype_cache: Dict[str, Optional[Dict]] = {}
-    wmscap_cache: Dict[str, WebMapService] = {}
+    featuretype_cache: Dict[str, Optional[Dict[str, Any]]] = {}
+    wms_capabilities_cache: Dict[str, WebMapService] = {}
 
     def __init__(self) -> None:
         super().__init__()
@@ -347,33 +456,53 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             self.config = None
         self.env = None
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
+    def __call__(
+        self, filename: str, options: Dict[str, Any], fileobj: Optional[str] = None, lineno: int = 0
+    ) -> List[Message]:
         del fileobj, lineno
+
+        print(f"Running {self.__class__.__name__} on {filename}")
+
         messages: List[Message] = []
 
         try:
-            engine = sqlalchemy.engine_from_config(self.config, "sqlalchemy_slave.")
-            factory = sqlalchemy.orm.sessionmaker(bind=engine)
-            db_session = sqlalchemy.orm.scoped_session(factory)
-            c2cgeoportal_commons.models.DBSession = db_session
-            c2cgeoportal_commons.models.Base.metadata.bind = engine
+            init_db(self.config)
+            from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
+
+            db_session = DBSession()
 
             try:
                 from c2cgeoportal_commons.models.main import (  # pylint: disable=import-outside-toplevel
-                    Theme,
+                    FullTextSearch,
                     LayerGroup,
                     LayerWMS,
                     LayerWMTS,
-                    FullTextSearch,
+                    Theme,
                 )
 
-                self._import(Theme, messages)
-                self._import(LayerGroup, messages)
-                self._import(LayerWMS, messages, self._import_layer_wms)
-                self._import(LayerWMTS, messages, self._import_layer_wmts)
+                self._import(Theme, messages, name_regex=_get_config_str("THEME_REGEX", ".*"))
+                self._import(
+                    LayerGroup,
+                    messages,
+                    name_regex=_get_config_str("GROUP_REGEX", ".*"),
+                    has_interfaces=False,
+                )
+                self._import(
+                    LayerWMS,
+                    messages,
+                    self._import_layer_wms,
+                    name_regex=_get_config_str("WMSLAYER_REGEX", ".*"),
+                )
+                self._import(
+                    LayerWMTS,
+                    messages,
+                    self._import_layer_wmts,
+                    name_regex=_get_config_str("WMTSLAYER_REGEX", ".*"),
+                )
 
                 for (layer_name,) in db_session.query(FullTextSearch.layer_name).distinct().all():
                     if layer_name is not None and layer_name != "":
+                        assert layer_name is not None
                         messages.append(
                             Message(
                                 None,
@@ -389,6 +518,7 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 for (actions,) in db_session.query(FullTextSearch.actions).distinct().all():
                     if actions is not None and actions != "":
                         for action in actions:
+                            assert action["data"] is not None
                             messages.append(
                                 Message(
                                     None,
@@ -405,59 +535,75 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                     colorize(
                         "ERROR! The database is probably not up to date "
                         "(should be ignored when happen during the upgrade)",
-                        RED,
+                        Color.RED,
                     )
                 )
-                print(colorize(e, RED))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                print(colorize(e, Color.RED))
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
         except NoSuchTableError as e:
             print(
                 colorize(
                     "ERROR! The schema didn't seem to exists "
                     "(should be ignored when happen during the deploy)",
-                    RED,
+                    Color.RED,
                 )
             )
-            print(colorize(e, RED))
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+            print(colorize(e, Color.RED))
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                 raise
         except OperationalError as e:
             print(
                 colorize(
                     "ERROR! The database didn't seem to exists "
                     "(should be ignored when happen during the deploy)",
-                    RED,
+                    Color.RED,
                 )
             )
-            print(colorize(e, RED))
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+            print(colorize(e, Color.RED))
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                 raise
 
         return messages
 
     @staticmethod
-    def _import(object_type, messages, callback=None):
+    def _import(
+        object_type: Type[Any],
+        messages: List[str],
+        callback: Optional[Callable[["main.Layer", List[str]], None]] = None,
+        has_interfaces: bool = True,
+        name_regex: str = ".*",
+    ) -> None:
         from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
-
-        items = DBSession.query(object_type)
-        for item in items:
-            messages.append(
-                Message(
-                    None,
-                    item.name,
-                    None,
-                    [],
-                    "",
-                    "",
-                    (item.item_type, item.name.encode("ascii", errors="replace")),
+        from c2cgeoportal_commons.models.main import Interface  # pylint: disable=import-outside-toplevel
+
+        filter_re = re.compile(name_regex)
+
+        query = DBSession.query(object_type)
+
+        interfaces = _get_config("INTERFACES")
+        if has_interfaces and interfaces is not None:
+            query.join(object_type.interface).filter(Interface.name in interfaces.split("."))
+
+        for item in query.all():
+            assert item.name is not None
+            if filter_re.match(item.name):
+                messages.append(
+                    Message(
+                        None,
+                        item.name,
+                        None,
+                        [],
+                        "",
+                        "",
+                        (item.item_type, item.name.encode("ascii", errors="replace")),
+                    )
                 )
-            )
 
             if callback is not None:
                 callback(item, messages)
 
-    def _import_layer_wms(self, layer, messages):
+    def _import_layer_wms(self, layer: "main.Layer", messages: List[str]) -> None:
         server = layer.ogc_server
         url = server.url_wfs or server.url
         if url is None:
@@ -481,6 +627,7 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                                     name = column.name + "_"
                             else:
                                 name = column_property.key
+                            assert name is not None
                             messages.append(
                                 Message(
                                     None,
@@ -495,14 +642,15 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             except NoSuchTableError:
                 print(
                     colorize(
-                        "ERROR! No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name), RED
+                        f"ERROR! No such table '{layer.geo_table}' for layer '{layer.name}'.",
+                        Color.RED,
                     )
                 )
-                print(colorize(traceback.format_exc(), RED))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                print(colorize(traceback.format_exc(), Color.RED))
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
 
-    def _import_layer_wmts(self, layer, messages):
+    def _import_layer_wmts(self, layer: "main.Layer", messages: List[str]) -> None:
         from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
         from c2cgeoportal_commons.models.main import OGCServer  # pylint: disable=import-outside-toplevel
 
@@ -511,7 +659,7 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"]
         server = [d.value for d in layer.metadatas if d.name == "ogcServer"]
         if server and layers:
-            layers = [l for ls in layers for l in ls.split(",")]
+            layers = [layer for ls in layers for layer in ls.split(",")]
             for wms_layer in layers:
                 try:
                     db_server = DBSession.query(OGCServer).filter(OGCServer.name == server[0]).one()
@@ -526,18 +674,20 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 except NoResultFound:
                     print(
                         colorize(
-                            "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format(
-                                server[0], layer.name
-                            ),
-                            RED,
+                            f"ERROR! the OGC server '{server[0]}' from the "
+                            f"WMTS layer '{layer.name}' does not exist.",
+                            Color.RED,
                         )
                     )
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                         raise
 
-    def _import_layer_attributes(self, url, layer, item_type, name, messages):
+    def _import_layer_attributes(
+        self, url: str, layer: "main.Layer", item_type: str, name: str, messages: List[str]
+    ) -> None:
         attributes, layers = self._layer_attributes(url, layer)
         for sub_layer in layers:
+            assert sub_layer is not None
             messages.append(
                 Message(
                     None,
@@ -550,6 +700,7 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 )
             )
         for attribute in attributes:
+            assert attribute is not None
             messages.append(
                 Message(
                     None,
@@ -562,105 +713,151 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 )
             )
 
-    def _build_url(self, url):
-        url_split = urlsplit(url)
-        hostname = url_split.hostname
+    def _build_url(self, url: Url) -> Tuple[Url, Dict[str, str], Dict[str, Any]]:
+        hostname = url.hostname
         host_map = self.config.get("lingua_extractor", {}).get("host_map", {})
         if hostname in host_map:
             map_ = host_map[hostname]
             if "netloc" in map_:
-                url_split = url_split._replace(netloc=map_["netloc"])
+                url.netloc = map_["netloc"]
             if "scheme" in map_:
-                url_split = url_split._replace(scheme=map_["scheme"])
+                url.scheme = map_["scheme"]
             kwargs = {"verify": map_["verify"]} if "verify" in map_ else {}
-            return url_split.geturl(), map_.get("headers", {}), kwargs
+            return url, map_.get("headers", {}), kwargs
         return url, {}, {}
 
-    def _layer_attributes(self, url, layer):
+    def _layer_attributes(self, url: str, layer: str) -> Tuple[List[str], List[str]]:
         errors: Set[str] = set()
 
-        request = _Request()
-        request.registry.settings = self.config
+        request = pyramid.threadlocal.get_current_request()
+        if request is None:
+            request = _Request()
+            request.registry.settings = self.config
+
         # Static schema will not be supported
-        url = get_url2("Layer", url, request, errors)
+        url_obj_ = get_url2("Layer", url, request, errors)
         if errors:
             print("\n".join(errors))
             return [], []
-        url, headers, kwargs = self._build_url(url)
-
-        if url not in self.wmscap_cache:
-            print("Get WMS GetCapabilities for URL: {}".format(url))
-            self.wmscap_cache[url] = None
-
-            wms_getcap_url = add_url_params(
-                url, {"SERVICE": "WMS", "VERSION": "1.1.1", "REQUEST": "GetCapabilities"}
+        if not url_obj_:
+            print(f"No URL for: {url}")
+            return [], []
+        url_obj: Url = url_obj_
+        url_obj, headers, kwargs = self._build_url(url_obj)
+
+        if url not in self.wms_capabilities_cache:
+            print(f"Get WMS GetCapabilities for URL: {url_obj}")
+            self.wms_capabilities_cache[url] = None
+
+            wms_getcap_url = (
+                url_obj.clone()
+                .add_query(
+                    {
+                        "SERVICE": "WMS",
+                        "VERSION": "1.1.1",
+                        "REQUEST": "GetCapabilities",
+                        "ROLE_IDS": "0",
+                        "USER_ID": "0",
+                    }
+                )
+                .url()
             )
             try:
-                print(
-                    "Get WMS GetCapabilities for URL {},\nwith headers: {}".format(
-                        wms_getcap_url, " ".join(["=".join(h) for h in headers.items()])
-                    )
+                rendered_headers = " ".join(
+                    [
+                        f"{h}={v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in headers.items()
+                    ]
                 )
+                print(f"Get WMS GetCapabilities for URL {wms_getcap_url},\nwith headers: {rendered_headers}")
                 response = requests.get(wms_getcap_url, headers=headers, **kwargs)
 
-                try:
-                    self.wmscap_cache[url] = WebMapService(None, xml=response.content)
-                except Exception as e:
+                if response.ok:
+                    try:
+                        self.wms_capabilities_cache[url] = WebMapService(None, xml=response.content)
+                    except Exception as e:
+                        print(
+                            colorize(
+                                "ERROR! an error occurred while trying to parse "
+                                "the GetCapabilities document.",
+                                Color.RED,
+                            )
+                        )
+                        print(colorize(str(e), Color.RED))
+                        print(f"URL: {wms_getcap_url}\nxml:\n{response.text}")
+                        if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                            raise
+                else:
                     print(
                         colorize(
-                            "ERROR! an error occurred while trying to " "parse the GetCapabilities document.",
-                            RED,
+                            f"ERROR! Unable to GetCapabilities from URL: {wms_getcap_url},\n"
+                            f"with headers: {rendered_headers}",
+                            Color.RED,
                         )
                     )
-                    print(colorize(str(e), RED))
-                    print("URL: {}\nxml:\n{}".format(wms_getcap_url, response.text))
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
-                        raise
-            except Exception as e:  # pragma: no cover
-                print(colorize(str(e), RED))
+                    print(f"Response: {response.status_code} {response.reason}\n{response.text}")
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                        raise Exception(response.reason)
+            except Exception as e:
+                print(colorize(str(e), Color.RED))
+                rendered_headers = " ".join(
+                    [
+                        f"{h}={v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in headers.items()
+                    ]
+                )
                 print(
                     colorize(
-                        "ERROR! Unable to GetCapabilities from URL: {},\nwith headers: {}".format(
-                            wms_getcap_url, " ".join(["=".join(h) for h in headers.items()])
-                        ),
-                        RED,
+                        f"ERROR! Unable to GetCapabilities from URL: {wms_getcap_url},\n"
+                        f"with headers: {rendered_headers}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
 
-        wmscap = self.wmscap_cache[url]
+        wms_capabilities = self.wms_capabilities_cache[url]
 
         if url not in self.featuretype_cache:
-            print("Get WFS DescribeFeatureType for URL: {}".format(url))
+            print(f"Get WFS DescribeFeatureType for URL: {url_obj}")
             self.featuretype_cache[url] = None
 
-            wfs_descrfeat_url = add_url_params(
-                url, {"SERVICE": "WFS", "VERSION": "1.1.0", "REQUEST": "DescribeFeatureType"}
+            wfs_describe_feature_url = (
+                url_obj.clone()
+                .add_query(
+                    {
+                        "SERVICE": "WFS",
+                        "VERSION": "1.1.0",
+                        "REQUEST": "DescribeFeatureType",
+                        "ROLE_IDS": "0",
+                        "USER_ID": "0",
+                    }
+                )
+                .url()
             )
             try:
-                response = requests.get(wfs_descrfeat_url, headers=headers, **kwargs)
-            except Exception as e:  # pragma: no cover
-                print(colorize(str(e), RED))
+                response = requests.get(wfs_describe_feature_url, headers=headers, **kwargs)
+            except Exception as e:
+                print(colorize(str(e), Color.RED))
                 print(
                     colorize(
-                        "ERROR! Unable to DescribeFeatureType from URL: {}".format(wfs_descrfeat_url), RED
+                        f"ERROR! Unable to DescribeFeatureType from URL: {wfs_describe_feature_url}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
 
-            if not response.ok:  # pragma: no cover
+            if not response.ok:
                 print(
                     colorize(
-                        "ERROR! DescribeFeatureType from URL {} return the error: {:d} {}".format(
-                            wfs_descrfeat_url, response.status_code, response.reason
-                        ),
-                        RED,
+                        f"ERROR! DescribeFeatureType from URL {wfs_describe_feature_url} return the error: "
+                        f"{response.status_code:d} {response.reason}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise Exception("Aborted")
 
@@ -675,13 +872,13 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             except ExpatError as e:
                 print(
                     colorize(
-                        "ERROR! an error occurred while trying to " "parse the DescribeFeatureType document.",
-                        RED,
+                        "ERROR! an error occurred while trying to parse the DescribeFeatureType document.",
+                        Color.RED,
                     )
                 )
-                print(colorize(str(e), RED))
-                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, response.text))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                print(colorize(str(e), Color.RED))
+                print(f"URL: {wfs_describe_feature_url}\nxml:\n{response.text}")
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
             except AttributeError:
@@ -689,11 +886,11 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                     colorize(
                         "ERROR! an error occurred while trying to "
                         "read the Mapfile and recover the themes.",
-                        RED,
+                        Color.RED,
                     )
                 )
-                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, response.text))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                print(f"URL: {wfs_describe_feature_url}\nxml:\n{response.text}")
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
         else:
@@ -702,16 +899,16 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
         if featurestype is None:
             return [], []
 
-        layers = [layer]
-        if wmscap is not None and layer in list(wmscap.contents):
-            layer_obj = wmscap[layer]
+        layers: List[str] = [layer]
+        if wms_capabilities is not None and layer in list(wms_capabilities.contents):
+            layer_obj = wms_capabilities[layer]
             if layer_obj.layers:
-                layers = [l.name for l in layer_obj.layers]
+                layers = [layer.name for layer in layer_obj.layers]
 
-        attributes = []
+        attributes: List[str] = []
         for sub_layer in layers:
             # Should probably be adapted for other king of servers
-            type_element = featurestype.get("{}Type".format(sub_layer))
+            type_element = featurestype.get(f"{sub_layer}Type")
             if type_element is not None:
                 for element in type_element.getElementsByTagNameNS(
                     "http://www.w3.org/2001/XMLSchema", "element"