c2cgeoportal-geoportal 2.6.0__py2.py3-none-any.whl → 2.7.1.83__py2.py3-none-any.whl

c2cgeoportal_geoportal/lib/lingua_extractor.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2011-2021, Camptocamp SA
+# Copyright (c) 2011-2023, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -33,15 +31,14 @@ import os
 import re
 import subprocess
 import traceback
-from typing import Dict, List, Optional, Set, cast
-from urllib.parse import urlsplit
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, cast
 from xml.dom import Node
 from xml.parsers.expat import ExpatError
 
+import pyramid.threadlocal
 import requests
-import sqlalchemy
 import yaml
-from bottle import MakoTemplate, template  # pylint: disable=wrong-import-order,useless-suppression
+from bottle import MakoTemplate, template
 from c2c.template.config import config
 from defusedxml.minidom import parseString
 from geoalchemy2.types import Geometry
@@ -52,60 +49,81 @@ from owslib.wms import WebMapService
 from sqlalchemy.exc import NoSuchTableError, OperationalError, ProgrammingError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.properties import ColumnProperty
+from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.util import class_mapper
 
-import c2cgeoportal_commons.models
 import c2cgeoportal_geoportal
-from c2cgeoportal_commons.lib.url import add_url_params, get_url2
-from c2cgeoportal_geoportal.lib.bashcolor import RED, colorize
+from c2cgeoportal_commons.lib.url import Url, get_url2
+from c2cgeoportal_geoportal.lib.bashcolor import Color, colorize
 from c2cgeoportal_geoportal.lib.caching import init_region
 from c2cgeoportal_geoportal.views.layers import Layers, get_layer_class
 
+if TYPE_CHECKING:
+    from c2cgeoportal_commons.models import main  # pylint: disable=ungrouped-imports,useless-suppression
+
+
+def _get_config(key: str, default: Optional[str] = None) -> Optional[str]:
+    """
+    Return the config value for passed key.
+
+    Passed throw environment variable for the command line,
+    or throw the query string on HTTP request.
+    """
+    request = pyramid.threadlocal.get_current_request()
+    if request is not None:
+        return cast(Optional[str], request.params.get(key.lower(), default))
+
+    return os.environ.get(key, default)
+
+
+def _get_config_str(key: str, default: str) -> str:
+    result = _get_config(key, default)
+    assert result is not None
+    return result
 
-class _Registry:  # pragma: no cover
+
+class _Registry:
     settings = None
 
-    def __init__(self, settings):
+    def __init__(self, settings: Optional[Dict[str, Any]]) -> None:
         self.settings = settings
 
 
-class _Request:  # pragma: no cover
+class _Request:
     params: Dict[str, str] = {}
     matchdict: Dict[str, str] = {}
     GET: Dict[str, str] = {}
 
-    def __init__(self, settings=None):
+    def __init__(self, settings: Optional[Dict[str, Any]] = None) -> None:
         self.registry: _Registry = _Registry(settings)
 
     @staticmethod
-    def static_url(*args, **kwargs):
+    def static_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def static_path(*args, **kwargs):
+    def static_path(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def route_url(*args, **kwargs):
+    def route_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
     @staticmethod
-    def current_route_url(*args, **kwargs):
+    def current_route_url(*args: Any, **kwargs: Any) -> str:
         del args
         del kwargs
         return ""
 
 
-class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
-    """
-    GeoMapFish angular extractor
-    """
+class GeomapfishAngularExtractor(Extractor):  # type: ignore
+    """GeoMapFish angular extractor."""
 
     extensions = [".js", ".html"]
 
@@ -118,27 +136,49 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
             self.config = None
         self.tpl = None
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
+    @staticmethod
+    def get_message_cleaner(filename: str) -> Callable[[str], str]:
+        """Return a function for cleaning messages according to input file format."""
+        ext = os.path.splitext(filename)[1]
+
+        if ext in [".html", ".ejs"]:
+            # Remove \n in HTML multi-line strings
+            pattern = re.compile("\n *")
+            return lambda s: re.sub(pattern, " ", s)
+
+        return lambda s: s
+
+    def __call__(
+        self,
+        filename: str,
+        options: Dict[str, Any],
+        fileobj: Optional[Dict[str, Any]] = None,
+        lineno: int = 0,
+    ) -> List[Message]:
         del fileobj, lineno
 
+        print(f"Running {self.__class__.__name__} on {filename}")
+
+        cleaner = self.get_message_cleaner(filename)
+
         init_region({"backend": "dogpile.cache.memory"}, "std")
         init_region({"backend": "dogpile.cache.memory"}, "obj")
 
         int_filename = filename
-        if re.match("^" + re.escape("./{}/templates".format(self.config["package"])), filename):
+        if re.match("^" + re.escape(f"./{self.config['package']}/templates"), filename):
             try:
                 empty_template = Template("")  # nosec
 
-                class Lookup(TemplateLookup):
+                class Lookup(TemplateLookup):  # type: ignore
                     @staticmethod
-                    def get_template(uri):
+                    def get_template(uri: str) -> Template:
                         del uri  # unused
                         return empty_template
 
-                class MyTemplate(MakoTemplate):
+                class MyTemplate(MakoTemplate):  # type: ignore
                     tpl = None
 
-                    def prepare(self, **kwargs):
+                    def prepare(self, **kwargs: Any) -> None:
                         options.update({"input_encoding": self.encoding})
                         lookup = Lookup(**kwargs)
                         if self.source:
@@ -149,10 +189,12 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
                             )
 
                 try:
+                    request = pyramid.threadlocal.get_current_request()
+                    request = _Request() if request is None else request
                     processed = template(
                         filename,
                         {
-                            "request": _Request(self.config),
+                            "request": request,
                             "lang": "fr",
                             "debug": False,
                             "extra_params": {},
@@ -167,11 +209,9 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
                     with open(int_filename, "wb") as file_open:
                         file_open.write(processed.encode("utf-8"))
                 except Exception:
-                    print(
-                        colorize("ERROR! Occurred during the '{}' template generation".format(filename), RED)
-                    )
-                    print(colorize(traceback.format_exc(), RED))
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                    print(colorize(f"ERROR! Occurred during the '{filename}' template generation", Color.RED))
+                    print(colorize(traceback.format_exc(), Color.RED))
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                         # Continue with the original one
                         int_filename = filename
                     else:
@@ -179,38 +219,85 @@ class GeomapfishAngularExtractor(Extractor):  # pragma: no cover
             except Exception:
                 print(traceback.format_exc())
 
-        message_str = subprocess.check_output(
-            ["node", "geoportal/tools/extract-messages.js", int_filename]
-        ).decode("utf-8")
+        # Path in geomapfish-tools
+        script_path = "geoportal/tools/extract-messages.js"
+        if not os.path.isfile(script_path):
+            # Path in geomapfish runner
+            script_path = "/app/tools/extract-messages.js"
+        message_str = subprocess.check_output(["node", script_path, int_filename]).decode("utf-8")
         if int_filename != filename:
             os.unlink(int_filename)
         try:
             messages = []
             for contexts, message in json.loads(message_str):
+                assert message is not None
+                message = cleaner(message)
                 for context in contexts.split(", "):
-                    assert message is not None
                     messages.append(Message(None, message, None, [], "", "", context.split(":")))
             return messages
         except Exception:
-            print(colorize("An error occurred", RED))
-            print(colorize(message_str, RED))
+            print(colorize("An error occurred", Color.RED))
+            print(colorize(message_str, Color.RED))
             print("------")
             raise
 
 
-class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
+def init_db(settings: Dict[str, Any]) -> None:
     """
-    GeoMapFish config extractor (raster layers, and print templates)
+    Initialize the SQLAlchemy Session.
+
+    First test the connection, on wen environment it should be OK, with the command line we should get
+    an exception ind initialise the connection.
     """
 
+    try:
+        from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
+        from c2cgeoportal_commons.models.main import Theme  # pylint: disable=import-outside-toplevel
+
+        session = DBSession()
+        session.query(Theme).count()
+    except:  # pylint: disable=bare-except
+
+        # Init db sessions
+
+        class R:
+            settings: Dict[str, Any] = {}
+
+        class C:
+            registry = R()
+
+            def get_settings(self) -> Dict[str, Any]:
+                return self.registry.settings
+
+            def add_tween(self, *args: Any, **kwargs: Any) -> None:
+                pass
+
+        config_ = C()
+        config_.registry.settings = settings
+
+        c2cgeoportal_geoportal.init_db_sessions(settings, config_)
+
+
+class GeomapfishConfigExtractor(Extractor):  # type: ignore
+    """GeoMapFish config extractor (raster layers, and print templates)."""
+
     extensions = [".yaml", ".tmpl"]
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
-        del fileobj, lineno
+    def __call__(
+        self,
+        filename: str,
+        options: Dict[str, Any],
+        fileobj: Optional[Dict[str, Any]] = None,
+        lineno: int = 0,
+    ) -> List[Message]:
+        del options, fileobj, lineno
+
+        print(f"Running {self.__class__.__name__} on {filename}")
+
         init_region({"backend": "dogpile.cache.memory"}, "std")
         init_region({"backend": "dogpile.cache.memory"}, "obj")
 
-        with open(filename) as config_file:
+        with open(filename, encoding="utf8") as config_file:
             gmf_config = yaml.load(config_file, Loader=yaml.BaseLoader)  # nosec
             # For application config (config.yaml)
             if "vars" in gmf_config:
@@ -220,7 +307,7 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
                 return self._collect_print_config(gmf_config, filename)
             raise Exception("Not a known config file")
 
-    def _collect_app_config(self, filename):
+    def _collect_app_config(self, filename: str) -> List[Message]:
         config.init(filename)
         settings = config.get_config()
         assert not [
@@ -228,32 +315,18 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
         ]
         # Collect raster layers names
         raster = [
-            Message(None, raster_layer, None, [], "", "", (filename, "raster/{}".format(raster_layer)))
+            Message(None, raster_layer, None, [], "", "", (filename, f"raster/{raster_layer}"))
             for raster_layer in list(settings.get("raster", {}).keys())
         ]
 
-        # Init db sessions
-
-        class R:
-            settings = None
-
-        class C:
-            registry = R()
-
-            def get_settings(self):
-                return self.registry.settings
-
-            def add_tween(self, *args, **kwargs):
-                pass
-
-        config_ = C()
-        config_.registry.settings = settings
-
-        c2cgeoportal_geoportal.init_dbsessions(settings, config_)
+        init_db(settings)
 
         # Collect layers enum values (for filters)
 
-        from c2cgeoportal_commons.models import DBSessions  # pylint: disable=import-outside-toplevel
+        from c2cgeoportal_commons.models import (  # pylint: disable=import-outside-toplevel
+            DBSession,
+            DBSessions,
+        )
         from c2cgeoportal_commons.models.main import Metadata  # pylint: disable=import-outside-toplevel
 
         enums = []
@@ -262,14 +335,13 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
             layerinfos = enum_layers.get(layername, {})
             attributes = layerinfos.get("attributes", {})
             for fieldname in list(attributes.keys()):
-                values = self._enumerate_attributes_values(DBSessions, Layers, layerinfos, fieldname)
+                values = self._enumerate_attributes_values(DBSessions, layerinfos, fieldname)
                 for (value,) in values:
                     if isinstance(value, str) and value != "":
                         msgid = value
-                        location = "/layers/{}/values/{}/{}".format(
-                            layername,
-                            fieldname,
-                            value.encode("ascii", errors="replace").decode("ascii"),
+                        location = (
+                            f"/layers/{layername}/values/{fieldname}/"
+                            f"{value.encode('ascii', errors='replace').decode('ascii')}"
                         )
                         assert msgid is not None
                         enums.append(Message(None, msgid, None, [], "", "", (filename, location)))
@@ -279,14 +351,11 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
         names = [e["name"] for e in defs if e.get("translate", False)]
 
         if names:
-            engine = sqlalchemy.create_engine(config["sqlalchemy.url"])
-            Session = sqlalchemy.orm.session.sessionmaker()  # noqa
-            Session.configure(bind=engine)
-            session = Session()
+            session = DBSession()
 
-            query = session.query(Metadata).filter(Metadata.name.in_(names))  # pylint: disable=no-member
+            query = session.query(Metadata).filter(Metadata.name.in_(names))
             for metadata in query.all():
-                location = "metadata/{}/{}".format(metadata.name, metadata.id)
+                location = f"metadata/{metadata.name}/{metadata.id}"
                 assert metadata.value is not None
                 metadata_list.append(Message(None, metadata.value, None, [], "", "", (filename, location)))
 
@@ -297,8 +366,8 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
             ):
                 for a_index, action in enumerate(datasource.get("groupActions", [])):
                     location = (
-                        "interfaces_config/{}/constants/gmfSearchOptions/datasources[{}]/"
-                        "groupActions[{}]/title".format(interface, ds_index, a_index)
+                        f"interfaces_config/{interface}/constants/gmfSearchOptions/datasources[{ds_index}]/"
+                        f"groupActions[{a_index}]/title"
                     )
                     assert action["title"] is not None
                     interfaces_messages.append(
@@ -311,8 +380,9 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
                 .get("mergeTabs", {})
                 .keys()
             ):
-                location = "interfaces_config/{}/constants/gmfDisplayQueryGridOptions/mergeTabs/{}/".format(
-                    interface, merge_tab
+                location = (
+                    f"interfaces_config/{interface}/constants/gmfDisplayQueryGridOptions/"
+                    f"mergeTabs/{merge_tab}/"
                 )
                 assert merge_tab is not None
                 interfaces_messages.append(Message(None, merge_tab, None, [], "", "", (filename, location)))
@@ -320,35 +390,35 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
         return raster + enums + metadata_list + interfaces_messages
 
     @staticmethod
-    def _enumerate_attributes_values(dbsessions, layers, layerinfos, fieldname):
+    def _enumerate_attributes_values(
+        dbsessions: Dict[str, Session], layerinfos: Dict[str, Any], fieldname: str
+    ) -> Set[Tuple[str, ...]]:
         dbname = layerinfos.get("dbsession", "dbsession")
-        translate = layerinfos.get("attributes").get(fieldname, {}).get("translate", True)
+        translate = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("translate", True)
         if not translate:
-            return []
+            return set()
         try:
             dbsession = dbsessions.get(dbname)
-            return layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname)
+            return Layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname)
         except Exception as e:
-            table = layerinfos.get("attributes").get(fieldname, {}).get("table")
+            table = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("table")
             print(
                 colorize(
                     "ERROR! Unable to collect enumerate attributes for "
-                    "db: {}, table: {}, column: {} ({})".format(dbname, table, fieldname, e),
-                    RED,
+                    f"db: {dbname}, table: {table}, column: {fieldname} ({e!s})",
+                    Color.RED,
                 )
             )
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
-                return []
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                return set()
             raise
 
     @staticmethod
-    def _collect_print_config(print_config, filename):
+    def _collect_print_config(print_config: Dict[str, Any], filename: str) -> List[Message]:
         result = []
-        for template_ in list(print_config.get("templates").keys()):
+        for template_ in list(cast(Dict[str, Any], print_config.get("templates")).keys()):
             assert template_ is not None
-            result.append(
-                Message(None, template_, None, [], "", "", (filename, "template/{}".format(template_)))
-            )
+            result.append(Message(None, template_, None, [], "", "", (filename, f"template/{template_}")))
             assert not [
                 attribute
                 for attribute in list(print_config["templates"][template_]["attributes"].keys())
@@ -362,22 +432,20 @@ class GeomapfishConfigExtractor(Extractor):  # pragma: no cover
                     [],
                     "",
                     "",
-                    (filename, "template/{}/{}".format(template_, attribute)),
+                    (filename, f"template/{template_}/{attribute}"),
                 )
                 for attribute in list(print_config["templates"][template_]["attributes"].keys())
             ]
         return result
 
 
-class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
-    """
-    GeoMapFish theme extractor
-    """
+class GeomapfishThemeExtractor(Extractor):  # type: ignore
+    """GeoMapFish theme extractor."""
 
     # Run on the development.ini file
     extensions = [".ini"]
-    featuretype_cache: Dict[str, Optional[Dict]] = {}
-    wmscap_cache: Dict[str, WebMapService] = {}
+    featuretype_cache: Dict[str, Optional[Dict[str, Any]]] = {}
+    wms_capabilities_cache: Dict[str, WebMapService] = {}
 
     def __init__(self) -> None:
         super().__init__()
@@ -388,16 +456,20 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             self.config = None
         self.env = None
 
-    def __call__(self, filename, options, fileobj=None, lineno=0):
+    def __call__(
+        self, filename: str, options: Dict[str, Any], fileobj: Optional[str] = None, lineno: int = 0
+    ) -> List[Message]:
         del fileobj, lineno
+
+        print(f"Running {self.__class__.__name__} on {filename}")
+
         messages: List[Message] = []
 
         try:
-            engine = sqlalchemy.engine_from_config(self.config, "sqlalchemy_slave.")
-            factory = sqlalchemy.orm.sessionmaker(bind=engine)
-            db_session = sqlalchemy.orm.scoped_session(factory)
-            c2cgeoportal_commons.models.DBSession = db_session
-            c2cgeoportal_commons.models.Base.metadata.bind = engine
+            init_db(self.config)
+            from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
+
+            db_session = DBSession()
 
             try:
                 from c2cgeoportal_commons.models.main import (  # pylint: disable=import-outside-toplevel
@@ -408,10 +480,25 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                     Theme,
                 )
 
-                self._import(Theme, messages)
-                self._import(LayerGroup, messages)
-                self._import(LayerWMS, messages, self._import_layer_wms)
-                self._import(LayerWMTS, messages, self._import_layer_wmts)
+                self._import(Theme, messages, name_regex=_get_config_str("THEME_REGEX", ".*"))
+                self._import(
+                    LayerGroup,
+                    messages,
+                    name_regex=_get_config_str("GROUP_REGEX", ".*"),
+                    has_interfaces=False,
+                )
+                self._import(
+                    LayerWMS,
+                    messages,
+                    self._import_layer_wms,
+                    name_regex=_get_config_str("WMSLAYER_REGEX", ".*"),
+                )
+                self._import(
+                    LayerWMTS,
+                    messages,
+                    self._import_layer_wmts,
+                    name_regex=_get_config_str("WMTSLAYER_REGEX", ".*"),
+                )
 
                 for (layer_name,) in db_session.query(FullTextSearch.layer_name).distinct().all():
                     if layer_name is not None and layer_name != "":
@@ -448,60 +535,75 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                     colorize(
                         "ERROR! The database is probably not up to date "
                         "(should be ignored when happen during the upgrade)",
-                        RED,
+                        Color.RED,
                     )
                 )
-                print(colorize(e, RED))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                print(colorize(e, Color.RED))
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
         except NoSuchTableError as e:
             print(
                 colorize(
                     "ERROR! The schema didn't seem to exists "
                     "(should be ignored when happen during the deploy)",
-                    RED,
+                    Color.RED,
                 )
             )
-            print(colorize(e, RED))
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+            print(colorize(e, Color.RED))
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                 raise
         except OperationalError as e:
             print(
                 colorize(
                     "ERROR! The database didn't seem to exists "
                     "(should be ignored when happen during the deploy)",
-                    RED,
+                    Color.RED,
                 )
             )
-            print(colorize(e, RED))
-            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+            print(colorize(e, Color.RED))
+            if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                 raise
 
         return messages
 
     @staticmethod
-    def _import(object_type, messages, callback=None):
+    def _import(
+        object_type: Type[Any],
+        messages: List[str],
+        callback: Optional[Callable[["main.Layer", List[str]], None]] = None,
+        has_interfaces: bool = True,
+        name_regex: str = ".*",
+    ) -> None:
         from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
+        from c2cgeoportal_commons.models.main import Interface  # pylint: disable=import-outside-toplevel
+
+        filter_re = re.compile(name_regex)
 
-        items = DBSession.query(object_type)
-        for item in items:
+        query = DBSession.query(object_type)
+
+        interfaces = _get_config("INTERFACES")
+        if has_interfaces and interfaces is not None:
+            query.join(object_type.interface).filter(Interface.name in interfaces.split("."))
+
+        for item in query.all():
             assert item.name is not None
-            messages.append(
-                Message(
-                    None,
-                    item.name,
-                    None,
-                    [],
-                    "",
-                    "",
-                    (item.item_type, item.name.encode("ascii", errors="replace")),
+            if filter_re.match(item.name):
+                messages.append(
+                    Message(
+                        None,
+                        item.name,
+                        None,
+                        [],
+                        "",
+                        "",
+                        (item.item_type, item.name.encode("ascii", errors="replace")),
+                    )
                 )
-            )
 
             if callback is not None:
                 callback(item, messages)
 
-    def _import_layer_wms(self, layer, messages):
+    def _import_layer_wms(self, layer: "main.Layer", messages: List[str]) -> None:
         server = layer.ogc_server
         url = server.url_wfs or server.url
         if url is None:
@@ -540,14 +642,15 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             except NoSuchTableError:
                 print(
                     colorize(
-                        "ERROR! No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name), RED
+                        f"ERROR! No such table '{layer.geo_table}' for layer '{layer.name}'.",
+                        Color.RED,
                     )
                 )
-                print(colorize(traceback.format_exc(), RED))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                print(colorize(traceback.format_exc(), Color.RED))
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
 
-    def _import_layer_wmts(self, layer, messages):
+    def _import_layer_wmts(self, layer: "main.Layer", messages: List[str]) -> None:
         from c2cgeoportal_commons.models import DBSession  # pylint: disable=import-outside-toplevel
         from c2cgeoportal_commons.models.main import OGCServer  # pylint: disable=import-outside-toplevel
 
@@ -571,16 +674,17 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 except NoResultFound:
                     print(
                         colorize(
-                            "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format(
-                                server[0], layer.name
-                            ),
-                            RED,
+                            f"ERROR! the OGC server '{server[0]}' from the "
+                            f"WMTS layer '{layer.name}' does not exist.",
+                            Color.RED,
                         )
                     )
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                         raise
 
-    def _import_layer_attributes(self, url, layer, item_type, name, messages):
+    def _import_layer_attributes(
+        self, url: str, layer: "main.Layer", item_type: str, name: str, messages: List[str]
+    ) -> None:
         attributes, layers = self._layer_attributes(url, layer)
         for sub_layer in layers:
             assert sub_layer is not None
@@ -609,131 +713,151 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                 )
             )
 
-    def _build_url(self, url):
-        url_split = urlsplit(url)
-        hostname = url_split.hostname
+    def _build_url(self, url: Url) -> Tuple[Url, Dict[str, str], Dict[str, Any]]:
+        hostname = url.hostname
         host_map = self.config.get("lingua_extractor", {}).get("host_map", {})
         if hostname in host_map:
             map_ = host_map[hostname]
             if "netloc" in map_:
-                url_split = url_split._replace(netloc=map_["netloc"])
+                url.netloc = map_["netloc"]
             if "scheme" in map_:
-                url_split = url_split._replace(scheme=map_["scheme"])
+                url.scheme = map_["scheme"]
             kwargs = {"verify": map_["verify"]} if "verify" in map_ else {}
-            return url_split.geturl(), map_.get("headers", {}), kwargs
+            return url, map_.get("headers", {}), kwargs
         return url, {}, {}
 
-    def _layer_attributes(self, url, layer):
+    def _layer_attributes(self, url: str, layer: str) -> Tuple[List[str], List[str]]:
         errors: Set[str] = set()
 
-        request = _Request()
-        request.registry.settings = self.config
+        request = pyramid.threadlocal.get_current_request()
+        if request is None:
+            request = _Request()
+            request.registry.settings = self.config
+
         # Static schema will not be supported
-        url = get_url2("Layer", url, request, errors)
+        url_obj_ = get_url2("Layer", url, request, errors)
         if errors:
             print("\n".join(errors))
             return [], []
-        url, headers, kwargs = self._build_url(url)
-
-        if url not in self.wmscap_cache:
-            print("Get WMS GetCapabilities for URL: {}".format(url))
-            self.wmscap_cache[url] = None
-
-            wms_getcap_url = add_url_params(
-                url,
-                {
-                    "SERVICE": "WMS",
-                    "VERSION": "1.1.1",
-                    "REQUEST": "GetCapabilities",
-                    "ROLE_IDS": "0",
-                    "USER_ID": "0",
-                },
+        if not url_obj_:
+            print(f"No URL for: {url}")
+            return [], []
+        url_obj: Url = url_obj_
+        url_obj, headers, kwargs = self._build_url(url_obj)
+
+        if url not in self.wms_capabilities_cache:
+            print(f"Get WMS GetCapabilities for URL: {url_obj}")
+            self.wms_capabilities_cache[url] = None
+
+            wms_getcap_url = (
+                url_obj.clone()
+                .add_query(
+                    {
+                        "SERVICE": "WMS",
+                        "VERSION": "1.1.1",
+                        "REQUEST": "GetCapabilities",
+                        "ROLE_IDS": "0",
+                        "USER_ID": "0",
+                    }
+                )
+                .url()
             )
             try:
-                print(
-                    "Get WMS GetCapabilities for URL {},\nwith headers: {}".format(
-                        wms_getcap_url,
-                        " ".join(
-                            [
-                                "{}={}".format(h, v if h not in ("Authorization", "Cookies") else "***")
-                                for h, v in headers.items()
-                            ]
-                        ),
-                    )
+                rendered_headers = " ".join(
+                    [
+                        f"{h}={v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in headers.items()
+                    ]
                 )
+                print(f"Get WMS GetCapabilities for URL {wms_getcap_url},\nwith headers: {rendered_headers}")
                 response = requests.get(wms_getcap_url, headers=headers, **kwargs)
 
-                try:
-                    self.wmscap_cache[url] = WebMapService(None, xml=response.content)
-                except Exception as e:
+                if response.ok:
+                    try:
+                        self.wms_capabilities_cache[url] = WebMapService(None, xml=response.content)
+                    except Exception as e:
+                        print(
+                            colorize(
+                                "ERROR! an error occurred while trying to parse "
+                                "the GetCapabilities document.",
+                                Color.RED,
+                            )
+                        )
+                        print(colorize(str(e), Color.RED))
+                        print(f"URL: {wms_getcap_url}\nxml:\n{response.text}")
+                        if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                            raise
+                else:
                     print(
                         colorize(
-                            "ERROR! an error occurred while trying to " "parse the GetCapabilities document.",
-                            RED,
+                            f"ERROR! Unable to GetCapabilities from URL: {wms_getcap_url},\n"
+                            f"with headers: {rendered_headers}",
+                            Color.RED,
                         )
                     )
-                    print(colorize(str(e), RED))
-                    print("URL: {}\nxml:\n{}".format(wms_getcap_url, response.text))
-                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
-                        raise
-            except Exception as e:  # pragma: no cover
-                print(colorize(str(e), RED))
+                    print(f"Response: {response.status_code} {response.reason}\n{response.text}")
+                    if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                        raise Exception(response.reason)
+            except Exception as e:
+                print(colorize(str(e), Color.RED))
+                rendered_headers = " ".join(
+                    [
+                        f"{h}={v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in headers.items()
+                    ]
+                )
                 print(
                     colorize(
-                        "ERROR! Unable to GetCapabilities from URL: {},\nwith headers: {}".format(
-                            wms_getcap_url,
-                            " ".join(
-                                [
-                                    "{}={}".format(h, v if h not in ("Authorization", "Cookies") else "***")
-                                    for h, v in headers.items()
-                                ]
-                            ),
-                        ),
-                        RED,
+                        f"ERROR! Unable to GetCapabilities from URL: {wms_getcap_url},\n"
+                        f"with headers: {rendered_headers}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                     raise
 
-        wmscap = self.wmscap_cache[url]
+        wms_capabilities = self.wms_capabilities_cache[url]
 
         if url not in self.featuretype_cache:
-            print("Get WFS DescribeFeatureType for URL: {}".format(url))
+            print(f"Get WFS DescribeFeatureType for URL: {url_obj}")
             self.featuretype_cache[url] = None
 
-            wfs_descrfeat_url = add_url_params(
-                url,
-                {
-                    "SERVICE": "WFS",
-                    "VERSION": "1.1.0",
-                    "REQUEST": "DescribeFeatureType",
-                    "ROLE_IDS": "0",
-                    "USER_ID": "0",
-                },
+            wfs_describe_feature_url = (
+                url_obj.clone()
+                .add_query(
+                    {
+                        "SERVICE": "WFS",
+                        "VERSION": "1.1.0",
+                        "REQUEST": "DescribeFeatureType",
+                        "ROLE_IDS": "0",
+                        "USER_ID": "0",
+                    }
+                )
+                .url()
             )
             try:
-                response = requests.get(wfs_descrfeat_url, headers=headers, **kwargs)
-            except Exception as e:  # pragma: no cover
-                print(colorize(str(e), RED))
+                response = requests.get(wfs_describe_feature_url, headers=headers, **kwargs)
+            except Exception as e:
+                print(colorize(str(e), Color.RED))
                 print(
                     colorize(
-                        "ERROR! Unable to DescribeFeatureType from URL: {}".format(wfs_descrfeat_url), RED
+                        f"ERROR! Unable to DescribeFeatureType from URL: {wfs_describe_feature_url}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
 
-            if not response.ok:  # pragma: no cover
+            if not response.ok:
                 print(
                     colorize(
-                        "ERROR! DescribeFeatureType from URL {} return the error: {:d} {}".format(
-                            wfs_descrfeat_url, response.status_code, response.reason
-                        ),
-                        RED,
+                        f"ERROR! DescribeFeatureType from URL {wfs_describe_feature_url} return the error: "
+                        f"{response.status_code:d} {response.reason}",
+                        Color.RED,
                     )
                 )
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise Exception("Aborted")
 
@@ -748,13 +872,13 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
             except ExpatError as e:
                 print(
                     colorize(
-                        "ERROR! an error occurred while trying to " "parse the DescribeFeatureType document.",
-                        RED,
+                        "ERROR! an error occurred while trying to parse the DescribeFeatureType document.",
+                        Color.RED,
                     )
                 )
-                print(colorize(str(e), RED))
-                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, response.text))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                print(colorize(str(e), Color.RED))
+                print(f"URL: {wfs_describe_feature_url}\nxml:\n{response.text}")
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
             except AttributeError:
@@ -762,11 +886,11 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
                     colorize(
                         "ERROR! an error occurred while trying to "
                         "read the Mapfile and recover the themes.",
-                        RED,
+                        Color.RED,
                     )
                 )
-                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, response.text))
-                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
+                print(f"URL: {wfs_describe_feature_url}\nxml:\n{response.text}")
+                if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                     return [], []
                 raise
         else:
@@ -775,16 +899,16 @@ class GeomapfishThemeExtractor(Extractor):  # pragma: no cover
         if featurestype is None:
             return [], []
 
-        layers = [layer]
-        if wmscap is not None and layer in list(wmscap.contents):
-            layer_obj = wmscap[layer]
+        layers: List[str] = [layer]
+        if wms_capabilities is not None and layer in list(wms_capabilities.contents):
+            layer_obj = wms_capabilities[layer]
             if layer_obj.layers:
                 layers = [layer.name for layer in layer_obj.layers]
 
-        attributes = []
+        attributes: List[str] = []
         for sub_layer in layers:
             # Should probably be adapted for other king of servers
-            type_element = featurestype.get("{}Type".format(sub_layer))
+            type_element = featurestype.get(f"{sub_layer}Type")
             if type_element is not None:
                 for element in type_element.getElementsByTagNameNS(
                     "http://www.w3.org/2001/XMLSchema", "element"