c2cgeoportal-geoportal 2.7.1.157__py2.py3-none-any.whl → 2.8.1.87__py2.py3-none-any.whl

c2cgeoportal_geoportal/views/theme.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2011-2022, Camptocamp SA
+# Copyright (c) 2011-2023, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -38,6 +38,7 @@ from math import sqrt
 from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
 
 import dogpile.cache.api
+import pyramid.httpexceptions
 import pyramid.request
 import requests
 import sqlalchemy
@@ -52,7 +53,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from c2cgeoportal_commons import models
 from c2cgeoportal_commons.lib.url import Url, get_url2
-from c2cgeoportal_commons.models import main
+from c2cgeoportal_commons.models import cache_invalidate_cb, main
 from c2cgeoportal_geoportal.lib import get_roles_id, get_typed, get_types_map, is_intranet
 from c2cgeoportal_geoportal.lib.caching import get_region
 from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
@@ -63,27 +64,31 @@ from c2cgeoportal_geoportal.lib.layers import (
     get_protected_layers_query,
 )
 from c2cgeoportal_geoportal.lib.wmstparsing import TimeInformation, parse_extent
-from c2cgeoportal_geoportal.views import restrict_headers
 from c2cgeoportal_geoportal.views.layers import get_layer_metadata
 
 LOG = logging.getLogger(__name__)
 CACHE_REGION = get_region("std")
+CACHE_OGC_SERVER_REGION = get_region("ogc-server")
 TIMEOUT = int(os.environ.get("C2CGEOPORTAL_THEME_TIMEOUT", "300"))
 
 Metadata = Union[str, int, float, bool, List[Any], Dict[str, Any]]
 
 
-def get_http_cached(http_options: Dict[str, Any], url: str, headers: Dict[str, str]) -> Tuple[bytes, str]:
+def get_http_cached(
+    http_options: Dict[str, Any], url: str, headers: Dict[str, str], cache: bool = True
+) -> Tuple[bytes, str]:
     """Get the content of the URL with a cash (dogpile)."""
 
-    @CACHE_REGION.cache_on_arguments()  # type: ignore
+    @CACHE_OGC_SERVER_REGION.cache_on_arguments()  # type: ignore
     def do_get_http_cached(url: str) -> Tuple[bytes, str]:
         response = requests.get(url, headers=headers, timeout=TIMEOUT, **http_options)
         response.raise_for_status()
-        LOG.info("Get url '%s' in %.1fs.", url, response.elapsed.total_seconds())
+        LOG.info("Get URL '%s' in %.1fs.", url, response.elapsed.total_seconds())
         return response.content, response.headers.get("Content-Type", "")
 
-    return do_get_http_cached(url)  # type: ignore
+    if cache:
+        return do_get_http_cached(url)  # type: ignore
+    return do_get_http_cached.refresh(url)  # type: ignore
 
 
 class DimensionInformation:
@@ -143,8 +148,6 @@ class Theme:
         self.request = request
         self.settings = request.registry.settings
         self.http_options = self.settings.get("http_options", {})
-        self.headers_whitelist = self.settings.get("headers_whitelist", [])
-        self.headers_blacklist = self.settings.get("headers_blacklist", [])
         self.metadata_type = get_types_map(
             self.settings.get("admin_interface", {}).get("available_metadata", [])
         )
@@ -179,14 +182,16 @@ class Theme:
         return metadatas
 
     async def _wms_getcap(
-        self, ogc_server: main.OGCServer, preload: bool = False
+        self, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
     ) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
-        @CACHE_REGION.cache_on_arguments()  # type: ignore
+        LOG.debug("Get the WMS Capabilities of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache)
+
+        @CACHE_OGC_SERVER_REGION.cache_on_arguments()  # type: ignore
         def build_web_map_service(ogc_server_id: int) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
             del ogc_server_id  # Just for cache
 
             if url is None:
-                raise RuntimeError("Url is None")
+                raise RuntimeError("URL is None")
 
             version = url.query.get("VERSION", "1.1.1")
             layers = {}
@@ -220,16 +225,16 @@ class Theme:
                 }
 
             del wms
-            LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
 
             return {"layers": layers}, set()
 
-        result = build_web_map_service.get(ogc_server.id)
-        if result != dogpile.cache.api.NO_VALUE:
-            return result  # type: ignore
+        if cache:
+            result = build_web_map_service.get(ogc_server.id)
+            if result != dogpile.cache.api.NO_VALUE:
+                return result  # type: ignore
 
         try:
-            url, content, errors = await self._wms_getcap_cached(ogc_server)
+            url, content, errors = await self._wms_getcap_cached(ogc_server, cache=cache)
         except requests.exceptions.RequestException as exception:
             error = (
                 f"Unable to get the WMS Capabilities for OGC server '{ogc_server.name}', "
@@ -240,10 +245,10 @@ class Theme:
         if errors or preload:
             return None, errors
 
-        return build_web_map_service(ogc_server.id)  # type: ignore
+        return build_web_map_service.refresh(ogc_server.id)  # type: ignore
 
     async def _wms_getcap_cached(
-        self, ogc_server: main.OGCServer
+        self, ogc_server: main.OGCServer, cache: bool = True
     ) -> Tuple[Optional[Url], Optional[bytes], Set[str]]:
         errors: Set[str] = set()
         url = get_url2(f"The OGC server '{ogc_server.name}'", ogc_server.url, self.request, errors)
@@ -269,23 +274,15 @@ class Theme:
 
         LOG.debug("Get WMS GetCapabilities for URL: %s", url)
 
-        # Forward request to target (without Host Header)
-        headers = dict(self.request.headers)
+        headers = {}
 
         # Add headers for Geoserver
         if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
             headers["sec-username"] = "root"
             headers["sec-roles"] = "root"
 
-        if url.hostname != "localhost" and "Host" in headers:
-            headers.pop("Host")
-
-        headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
-
         try:
-            content, content_type = await asyncio.get_event_loop().run_in_executor(
-                None, get_http_cached, self.http_options, url, headers
-            )
+            content, content_type = get_http_cached(self.http_options, url.url(), headers, cache=cache)
         except Exception:
             error = f"Unable to GetCapabilities from URL {url}"
             errors.add(error)
@@ -858,7 +855,7 @@ class Theme:
         return None if self.request.user is None else {role.id for role in self.request.user.roles}
 
     async def _wfs_get_features_type(
-        self, wfs_url: Url, ogc_server_name: str, preload: bool = False
+        self, wfs_url: Url, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
     ) -> Tuple[Optional[etree.Element], Set[str]]:
         errors = set()
 
@@ -872,23 +869,23 @@ class Theme:
             }
         )
 
-        LOG.debug("WFS DescribeFeatureType for the URL: %s", wfs_url)
+        LOG.debug(
+            "Get the WFS DescribeFeatureType of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache
+        )
 
-        # forward request to target (without Host Header)
-        headers = dict(self.request.headers)
-        if wfs_url.hostname != "localhost" and "Host" in headers:
-            headers.pop("Host")
+        headers = {}
 
-        headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
+        # Add headers for Geoserver
+        if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
+            headers["sec-username"] = "root"
+            headers["sec-roles"] = "root"
 
         try:
-            content, _ = await asyncio.get_event_loop().run_in_executor(
-                None, get_http_cached, self.http_options, wfs_url, headers
-            )
+            content, _ = get_http_cached(self.http_options, wfs_url.url(), headers, cache)
         except requests.exceptions.RequestException as exception:
             error = (
                 f"Unable to get WFS DescribeFeatureType from the URL '{wfs_url.url()}' for "
-                f"OGC server {ogc_server_name}, "
+                f"OGC server {ogc_server.name}, "
                 + (
                     f"return the error: {exception.response.status_code} {exception.response.reason}"
                     if exception.response is not None
@@ -901,7 +898,7 @@ class Theme:
         except Exception:
             error = (
                 f"Unable to get WFS DescribeFeatureType from the URL {wfs_url} for "
-                f"OGC server {ogc_server_name}"
+                f"OGC server {ogc_server.name}"
             )
             errors.add(error)
             LOG.exception(error)
@@ -948,10 +945,10 @@ class Theme:
             url_internal_wfs = url_wfs
         return url_internal_wfs, url, url_wfs
 
-    async def preload(self, errors: Set[str]) -> None:
+    async def _preload(self, errors: Set[str]) -> None:
         tasks = set()
         for ogc_server in models.DBSession.query(main.OGCServer).all():
-            # Don't load unused OGC servers, required for Landigpage, because the related OGC server
+            # Don't load unused OGC servers, required for landing page, because the related OGC server
             # will be on error in those functions.
             nb_layers = (
                 models.DBSession.query(sqlalchemy.func.count(main.LayerWMS.id))
@@ -963,26 +960,30 @@ class Theme:
                 LOG.debug("Preload OGC server '%s'", ogc_server.name)
                 url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
                 if url_internal_wfs is not None:
-                    if ogc_server.wfs_support:
-                        tasks.add(self._wfs_get_features_type(url_internal_wfs, ogc_server.name, True))
-                    tasks.add(self._wms_getcap(ogc_server, True))
+                    tasks.add(self.preload_ogc_server(ogc_server, url_internal_wfs))
 
         await asyncio.gather(*tasks)
 
+    async def preload_ogc_server(
+        self, ogc_server: main.OGCServer, url_internal_wfs: Url, cache: bool = True
+    ) -> None:
+        if ogc_server.wfs_support:
+            await self._get_features_attributes(url_internal_wfs, ogc_server, cache=cache)
+        await self._wms_getcap(ogc_server, False, cache=cache)
+
     async def _get_features_attributes(
-        self, url_internal_wfs: Url, ogc_server_name: str
+        self, url_internal_wfs: Url, ogc_server: main.OGCServer, cache: bool = True
     ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
-        @CACHE_REGION.cache_on_arguments()  # type: ignore
+        @CACHE_OGC_SERVER_REGION.cache_on_arguments()  # type: ignore
         def _get_features_attributes_cache(
             url_internal_wfs: Url, ogc_server_name: str
         ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
             del url_internal_wfs  # Just for cache
             all_errors: Set[str] = set()
-            LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
             if errors:
                 all_errors |= errors
                 return None, None, all_errors
-            assert feature_type
+            assert feature_type is not None
             namespace: str = feature_type.attrib.get("targetNamespace")
             types: Dict[Any, Dict[str, Any]] = {}
             elements = {}
@@ -1041,9 +1042,8 @@ class Theme:
                     attributes[name] = types[type_]
                 elif (type_ == "Character") and (name + "Type") in types:
                     LOG.debug(
-                        "Due MapServer strange result the type 'ms:Character' is fallbacked to type '%sType'"
-                        " for feature '%s', This is a strange comportement of MapServer when we use the "
-                        'METADATA "gml_types" "auto"',
+                        'Due to MapServer weird behavior when using METADATA "gml_types" "auto"'
+                        "the type 'ms:Character' is returned as type '%sType' for feature '%s'.",
                         name,
                         name,
                     )
@@ -1057,12 +1057,14 @@ class Theme:
 
             return attributes, namespace, all_errors
 
-        result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server_name)
-        if result != dogpile.cache.api.NO_VALUE:
-            return result  # type: ignore
+        if cache:
+            result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server.name)
+            if result != dogpile.cache.api.NO_VALUE:
+                return result  # type: ignore
+
+        feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server, False, cache)
 
-        feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server_name)
-        return _get_features_attributes_cache(url_internal_wfs, ogc_server_name)  # type: ignore
+        return _get_features_attributes_cache.refresh(url_internal_wfs, ogc_server.name)  # type: ignore
 
     @view_config(route_name="themes", renderer="json")  # type: ignore
     def themes(self) -> Dict[str, Union[Dict[str, Dict[str, Any]], List[str]]]:
@@ -1083,11 +1085,12 @@ class Theme:
             all_errors: Set[str] = set()
             LOG.debug("Start preload")
             start_time = time.time()
-            await self.preload(all_errors)
+            await self._preload(all_errors)
             LOG.debug("End preload")
             # Don't log if it looks to be already preloaded.
             if (time.time() - start_time) > 1:
                 LOG.info("Do preload in %.3fs.", time.time() - start_time)
+            LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
             result["ogcServers"] = {}
             for ogc_server in models.DBSession.query(main.OGCServer).all():
                 nb_layers = (
@@ -1096,9 +1099,11 @@ class Theme:
                     .one()
                 )
                 if nb_layers[0] == 0:
-                    # QGIS Server langing page requires an OGC server that can't be used here.
+                    # QGIS Server landing page requires an OGC server that can't be used here.
                     continue
 
+                LOG.debug("Process OGC server '%s'", ogc_server.name)
+
                 url_internal_wfs, url, url_wfs = self.get_url_internal_wfs(ogc_server, all_errors)
 
                 attributes = None
@@ -1110,7 +1115,7 @@ class Theme:
                     )
                 if ogc_server.wfs_support and url_internal_wfs:
                     attributes, namespace, errors = await self._get_features_attributes(
-                        url_internal_wfs, ogc_server.name
+                        url_internal_wfs, ogc_server
                     )
                     # Create a local copy (don't modify the cache)
                     if attributes is not None:
@@ -1209,3 +1214,34 @@ class Theme:
                     f"{', '.join([i[0] for i in models.DBSession.query(main.LayerGroup.name).all()])}"
                 },
             )
+
+    @view_config(route_name="ogc_server_clear_cache", renderer="json")  # type: ignore
+    def ogc_server_clear_cache_view(self) -> Dict[str, Any]:
+        self._ogc_server_clear_cache(
+            models.DBSession.query(main.OGCServer).filter_by(id=self.request.matchdict.get("id")).one()
+        )
+        came_from = self.request.params.get("came_from")
+        if came_from:
+            raise pyramid.httpexceptions.HTTPFound(location=came_from)
+        return {"success": True}
+
+    def _ogc_server_clear_cache(self, ogc_server: main.OGCServer) -> None:
+        errors: Set[str] = set()
+        url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
+        if errors:
+            LOG.error(
+                "Error while getting the URL of the OGC Server %s:\n%s", ogc_server.id, "\n".join(errors)
+            )
+            return
+        if url_internal_wfs is None:
+            return
+
+        asyncio.run(self._async_cache_invalidate_ogc_server_cb(ogc_server, url_internal_wfs))
+
+    async def _async_cache_invalidate_ogc_server_cb(
+        self, ogc_server: main.OGCServer, url_internal_wfs: Url
+    ) -> None:
+        # Fill the cache
+        await self.preload_ogc_server(ogc_server, url_internal_wfs, False)
+
+        cache_invalidate_cb()

c2cgeoportal_geoportal/views/tinyowsproxy.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015-2021, Camptocamp SA
+# Copyright (c) 2015-2023, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -27,7 +27,7 @@
 
 
 import logging
-from typing import Any, Dict, Optional, Set, Tuple
+from typing import Any, Dict, Set, Tuple
 
 import pyramid.request
 from defusedxml import ElementTree
@@ -68,9 +68,6 @@ class TinyOWSProxy(OGCProxy):
         # params hold the parameters we are going to send to TinyOWS
         self.lower_params = self._get_lower_params(dict(self.request.params))
 
-    def _get_wfs_url(self, errors: Set[str]) -> Optional[Url]:
-        return Url(self.settings.get("tinyows_url"))
-
     @view_config(route_name="tinyowsproxy")  # type: ignore
     def proxy(self) -> pyramid.response.Response:
         if self.user is None:
@@ -102,9 +99,7 @@ class TinyOWSProxy(OGCProxy):
             # for DescribeFeatureType we require that exactly one type-name
             # is given, otherwise we would have to filter the result
             if len(typenames) != 1:
-                raise HTTPBadRequest(
-                    "Exactly one type-name must be given for " "DescribeFeatureType requests"
-                )
+                raise HTTPBadRequest("Exactly one type-name must be given for DescribeFeatureType requests")
 
         if not self._is_allowed(typenames):
             raise HTTPForbidden("No access rights for at least one of the given type-names")
@@ -113,11 +108,7 @@ class TinyOWSProxy(OGCProxy):
         use_cache = method == "GET" and operation in ("getcapabilities", "describefeaturetype")
         cache_control = Cache.PRIVATE if use_cache else Cache.PRIVATE_NO
 
-        errors: Set[str] = set()
         url = Url(self.settings.get("tinyows_url"))
-        if url is None:
-            LOG.error("Error getting the URL:\n%s", "\n".join(errors))
-            raise HTTPInternalServerError()
 
         response = self._proxy_callback(
             operation,

c2cgeoportal_geoportal/views/vector_tiles.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, Camptocamp SA
+# Copyright (c) 2021-2023, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without

{c2cgeoportal_geoportal-2.7.1.157.dist-info → c2cgeoportal_geoportal-2.8.1.87.dist-info}/METADATA

@@ -1,11 +1,13 @@
 Metadata-Version: 2.1
 Name: c2cgeoportal-geoportal
-Version: 2.7.1.157
+Version: 2.8.1.87
 Summary: c2cgeoportal geoportal
 Home-page: https://github.com/camptocamp/c2cgeoportal/
 Author: Camptocamp
 Author-email: info@camptocamp.com
+License: UNKNOWN
 Keywords: web gis geoportail c2cgeoportal geocommune pyramid
+Platform: UNKNOWN
 Classifier: Development Status :: 6 - Mature
 Classifier: Environment :: Web Environment
 Classifier: Framework :: Pyramid
@@ -17,23 +19,30 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Topic :: Scientific/Engineering :: GIS
 Classifier: Typing :: Typed
+Requires-Dist: Fiona
+Requires-Dist: GeoAlchemy2
+Requires-Dist: Mako
+Requires-Dist: OWSLib (>=0.6.0)
+Requires-Dist: PyYAML
+Requires-Dist: SQLAlchemy
+Requires-Dist: Shapely
 Requires-Dist: alembic
 Requires-Dist: bottle
+Requires-Dist: c2c.template (>=2.0.7)
 Requires-Dist: c2cgeoportal-commons[upgrade]
 Requires-Dist: c2cwsgiutils
-Requires-Dist: c2c.template (>=2.0.7)
+Requires-Dist: certifi (>=2022.12.7)
 Requires-Dist: defusedxml
 Requires-Dist: dogpile.cache (>=0.6)
-Requires-Dist: Fiona
-Requires-Dist: GeoAlchemy2
 Requires-Dist: geojson
+Requires-Dist: idna (>=3.7)
 Requires-Dist: isodate
+Requires-Dist: jinja2 (>=3.1.3)
 Requires-Dist: lingua
-Requires-Dist: Mako
-Requires-Dist: OWSLib (>=0.6.0)
 Requires-Dist: papyrus
 Requires-Dist: psycopg2
 Requires-Dist: pycryptodome
+Requires-Dist: pygments (>=2.15.0)
 Requires-Dist: pyotp
 Requires-Dist: pyramid
 Requires-Dist: pyramid-debugtoolbar
@@ -41,18 +50,13 @@ Requires-Dist: pyramid-mako
 Requires-Dist: pyramid-multiauth
 Requires-Dist: pyramid-tm
 Requires-Dist: python-dateutil
-Requires-Dist: PyYAML
 Requires-Dist: rasterio
-Requires-Dist: requests
 Requires-Dist: redis
-Requires-Dist: Shapely
-Requires-Dist: SQLAlchemy
-Requires-Dist: transaction
-Requires-Dist: jinja2 (>=2.11.3)
-Requires-Dist: pygments (>=2.7.4)
-Requires-Dist: setuptools (>=65.5.1)
+Requires-Dist: requests
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: fiona (>=1.10b2)
+Requires-Dist: setuptools (>=65.5.1)
+Requires-Dist: transaction
+Requires-Dist: urllib3 (>=1.26.17)
 
 c2cgeoportal is the server part of `GeoMapFish <http://geomapfish.org/>`_,
 the client part is `ngeo <https://github.com/camptocamp/ngeo/>`_.
@@ -60,3 +64,4 @@ the client part is `ngeo <https://github.com/camptocamp/ngeo/>`_.
 Read the `Documentation <https://camptocamp.github.io/c2cgeoportal/master/>`_.
 
 `Sources <https://github.com/camptocamp/c2cgeoportal/>`_
+