c2cgeoportal-geoportal 2.5.0.100__py2.py3-none-any.whl → 2.7.1.83__py2.py3-none-any.whl

c2cgeoportal_geoportal/views/proxy.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2011-2020, Camptocamp SA
+# Copyright (c) 2011-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -30,64 +30,61 @@
 
 import logging
 import sys
-from typing import Dict, List, Union
-import urllib.parse
+from typing import Any, Dict, List, Optional, Union
 
-from pyramid.httpexceptions import HTTPBadGateway, exception_response
-from pyramid.response import Response
+import pyramid.request
+import pyramid.response
 import requests
+from pyramid.httpexceptions import HTTPBadGateway, exception_response
 
-from c2cgeoportal_geoportal.lib.caching import (
-    NO_CACHE,
-    PRIVATE_CACHE,
-    PUBLIC_CACHE,
-    get_region,
-    set_common_headers,
-)
+from c2cgeoportal_commons.lib.url import Url
+from c2cgeoportal_geoportal.lib.caching import get_region
+from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
+from c2cgeoportal_geoportal.views import restrict_headers
 
 LOG = logging.getLogger(__name__)
 CACHE_REGION = get_region("std")
 
 
 class Proxy:
-    def __init__(self, request):
+    """Some methodes used by all the proxy."""
+
+    def __init__(self, request: pyramid.request.Request):
         self.request = request
-        self.host_forward_host = request.registry.settings["host_forward_host"]
+        self.host_forward_host = request.registry.settings.get("host_forward_host", [])
+        self.headers_whitelist = request.registry.settings.get("headers_whitelist", [])
+        self.headers_blacklist = request.registry.settings.get("headers_blacklist", [])
         self.http_options = self.request.registry.settings.get("http_options", {})
 
-    def _proxy(self, url, params=None, method=None, cache=False, body=None, headers=None):
-        # get query string
+    def _proxy(
+        self,
+        url: Url,
+        params: Optional[Dict[str, str]] = None,
+        method: Optional[str] = None,
+        cache: bool = False,
+        body: Optional[bytes] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> requests.models.Response:
+        # Get query string
         params = dict(self.request.params) if params is None else params
-        parsed_url = urllib.parse.urlparse(url)
-        url_params = urllib.parse.parse_qs(parsed_url.query)
-        all_params: Dict[str, Union[str]] = {}
-        all_params.update(params)
-        all_params.update({k: ",".join(v) for k, v in url_params.items()})
-        query_string = urllib.parse.urlencode(all_params)
-
-        if parsed_url.port is None:
-            url = "{}://{}{}?{}".format(parsed_url.scheme, parsed_url.hostname, parsed_url.path, query_string)
-        else:  # pragma: no cover
-            url = "{}://{}:{:d}{}?{}".format(
-                parsed_url.scheme, parsed_url.hostname, parsed_url.port, parsed_url.path, query_string
-            )
+        url = url.clone().add_query(params, True)
 
         LOG.debug("Send query to URL:\n%s.", url)
 
         if method is None:
             method = self.request.method
 
-        if headers is None:  # pragma: no cover
-            headers = dict(self.request.headers)
+        headers = dict(self.request.headers if headers is None else headers)
 
-        # Forward request to target (without Host Header)
-        if parsed_url.hostname not in self.host_forward_host and "Host" in headers:  # pragma: no cover
+        # Forward request to target (without Host Header).
+        # The original Host will be added back by pyramid.
+        if url.hostname not in self.host_forward_host and "Host" in headers:
             headers.pop("Host")
 
         # Forward the request tracking ID to the other service. This will allow to follow the logs belonging
         # to a single request coming from the user
         headers.setdefault("X-Request-ID", self.request.c2c_request_id)
-        # If we releay want to respect the specification, we should chain with the content of the previus
+        # If we really want to respect the specification, we should chain with the content of the previous
         # proxy, see also:
         # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
         forwarded = {"for": self.request.client_addr, "proto": self.request.scheme}
@@ -98,34 +95,52 @@ class Proxy:
             headers["Forwarded"] = ",".join([headers["Forwarded"], forwarded_str])
         else:
             headers["Forwarded"] = forwarded_str
+        # Set alternative "X-Forwarded" headers
+        for forwarded_elements in reversed(headers["Forwarded"].split(",")):
+            for element in forwarded_elements.split(";"):
+                key, value = element.split("=")
+                header_key = f"X-Forwarded-{key.capitalize()}"
+                header_value = headers.get(header_key)
+                headers[header_key] = value if header_value is None else ", ".join([header_value, value])
 
         if not cache:
             headers["Cache-Control"] = "no-cache"
 
-        if method in ("POST", "PUT") and body is None:  # pragma: no cover
+        if method in ("POST", "PUT") and body is None:
             body = self.request.body
 
+        headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
+
         try:
             if method in ("POST", "PUT"):
-                response = requests.request(method, url, data=body, headers=headers, **self.http_options)
+                response = requests.request(
+                    method, url.url(), data=body, headers=headers, **self.http_options
+                )
             else:
-                response = requests.request(method, url, headers=headers, **self.http_options)
-        except Exception:  # pragma: no cover
+                response = requests.request(method, url.url(), headers=headers, **self.http_options)
+        except Exception:
             errors = ["Error '%s' while getting the URL:", "%s", "Method: %s", "--- With headers ---", "%s"]
             args1 = [
                 sys.exc_info()[0],
                 url,
                 method,
-                "\n".join(["{}: {}".format(*h) for h in list(headers.items())]),
+                "\n".join(
+                    [
+                        f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in list(headers.items())
+                    ]
+                ),
             ]
-            if method in ("POST", "PUT"):
+            if method in ("POST", "PUT") and body is not None:
                 errors += ["--- Query with body ---", "%s"]
                 args1.append(body.decode("utf-8"))
-            LOG.error("\n".join(errors), *args1, exc_info=True)
+            LOG.exception("\n".join(errors), *args1)
 
-            raise HTTPBadGateway("Error on backend, See logs for detail")
+            raise HTTPBadGateway(  # pylint: disable=raise-missing-from
+                "Error on backend, See logs for detail"
+            )
 
-        if not response.ok:  # pragma: no cover
+        if not response.ok:
             errors = [
                 "Error '%s' in response of URL:",
                 "%s",
@@ -136,12 +151,17 @@ class Proxy:
             ]
             args2: List[Union[str, int]] = [
                 response.reason,
-                url,
+                url.url(),
                 response.status_code,
                 method,
-                "\n".join(["{}: {}".format(*h) for h in list(headers.items())]),
+                "\n".join(
+                    [
+                        f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}"
+                        for h, v in list(headers.items())
+                    ]
+                ),
             ]
-            if method in ("POST", "PUT"):
+            if method in ("POST", "PUT") and body is not None:
                 errors += ["--- Query with body ---", "%s"]
                 args2.append(body.decode("utf-8"))
             errors += ["--- Return content ---", "%s"]
@@ -149,19 +169,26 @@ class Proxy:
             LOG.error("\n".join(errors), *args2)
 
             raise exception_response(response.status_code)
+        if not response.headers.get("Content-Type", "").startswith("image/"):
+            LOG.debug("Get result for URL: %s:\n%s.", url, body)
 
         return response
 
-    @CACHE_REGION.cache_on_arguments()
-    def _proxy_cache(self, method, *args, **kwargs):  # pragma: no cover
+    @CACHE_REGION.cache_on_arguments()  # type: ignore
+    def _proxy_cache(self, method: str, *args: Any, **kwargs: Any) -> pyramid.response.Response:
         # Method is only for the cache
         del method
         kwargs["cache"] = True
         return self._proxy(*args, **kwargs)
 
     def _proxy_response(
-        self, service_name, url, headers_update=None, public=False, **kwargs
-    ):  # pragma: no cover
+        self,
+        service_name: str,
+        url: Url,
+        headers_update: Optional[Dict[str, str]] = None,
+        public: bool = False,
+        **kwargs: Any,
+    ) -> pyramid.response.Response:
         if headers_update is None:
             headers_update = {}
         cache = kwargs.get("cache", False)
@@ -170,23 +197,25 @@ class Proxy:
         else:
             response = self._proxy(url, **kwargs)
 
-        cache_control = (PUBLIC_CACHE if public else PRIVATE_CACHE) if cache else NO_CACHE
+        cache_control = (
+            (Cache.PUBLIC if public else Cache.PRIVATE)
+            if cache
+            else (Cache.PUBLIC_NO if public else Cache.PRIVATE_NO)
+        )
         return self._build_response(
             response, response.content, cache_control, service_name, headers_update=headers_update
         )
 
     def _build_response(
         self,
-        response,
-        content,
-        cache_control,
-        service_name,
-        headers=None,
-        headers_update=None,
-        content_type=None,
-    ):
-        if isinstance(content, str):
-            content = content.encode("utf-8")
+        response: pyramid.response.Response,
+        content: bytes,
+        cache_control: Cache,
+        service_name: str,
+        headers: Optional[Dict[str, str]] = None,
+        headers_update: Optional[Dict[str, str]] = None,
+        content_type: Optional[str] = None,
+    ) -> pyramid.response.Response:
         if headers_update is None:
             headers_update = {}
         headers = response.headers if headers is None else headers
@@ -204,28 +233,28 @@ class Proxy:
             "Trailers",
             "Transfer-Encoding",
             "Upgrade",
-        ]:  # pragma: no cover
+        ]:
             if header in headers:
                 del headers[header]
         # Other problematic headers
-        for header in ["Content-Length", "Content-Location", "Content-Encoding"]:  # pragma: no cover
+        for header in ["Content-Length", "Content-Location", "Content-Encoding"]:
             if header in headers:
                 del headers[header]
 
         headers.update(headers_update)
 
-        response = Response(content, status=response.status_code, headers=headers)
+        response = pyramid.response.Response(content, status=response.status_code, headers=headers)
 
         return set_common_headers(
             self.request, service_name, cache_control, response=response, content_type=content_type
         )
 
     @staticmethod
-    def _get_lower_params(params):
+    def _get_lower_params(params: Dict[str, str]) -> Dict[str, str]:
         return dict((k.lower(), str(v).lower()) for k, v in params.items())
 
-    def _get_headers(self):
-        headers = self.request.headers
-        if "Cookie" in headers:  # pragma: no cover
+    def get_headers(self) -> Dict[str, str]:
+        headers: Dict[str, str] = self.request.headers
+        if "Cookie" in headers:
             headers.pop("Cookie")
         return headers

c2cgeoportal_geoportal/views/raster.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2012-2019, Camptocamp SA
+# Copyright (c) 2012-2021, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -28,31 +26,40 @@
 # either expressed or implied, of the FreeBSD Project.
 
 
-from decimal import Decimal
+import decimal
 import logging
 import math
 import os
-from typing import Any, Dict  # noqa, pylint: disable=unused-import
+import traceback
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
 
+import numpy
+import pyramid.request
+import zope.event.classhandler
 from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound
 from pyramid.view import view_config
-import zope.event.classhandler
+from rasterio.io import DatasetReader
 
 from c2cgeoportal_commons.models import InvalidateCacheEvent
-from c2cgeoportal_geoportal.lib.caching import NO_CACHE, set_common_headers
+from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
+
+if TYPE_CHECKING:
+    import fiona.collection
 
 LOG = logging.getLogger(__name__)
 
 
 class Raster:
-    data: Dict[str, Any] = {}
+    """All the view concerned the raster (point, not the profile profile)."""
 
-    def __init__(self, request):
+    data: Dict[str, "fiona.collection.Collection"] = {}
+
+    def __init__(self, request: pyramid.request.Request):
         self.request = request
         self.rasters = self.request.registry.settings["raster"]
 
-        @zope.event.classhandler.handler(InvalidateCacheEvent)
-        def handle(event: InvalidateCacheEvent):  # pylint: disable=unused-variable
+        @zope.event.classhandler.handler(InvalidateCacheEvent)  # type: ignore
+        def handle(event: InvalidateCacheEvent) -> None:
             del event
             for _, v in Raster.data.items():
                 v.close()
@@ -60,21 +67,21 @@ class Raster:
 
     def _get_required_finite_float_param(self, name: str) -> float:
         if name not in self.request.params:
-            raise HTTPBadRequest("'{}' should be in the query string parameters".format(name))
+            raise HTTPBadRequest(f"'{name}' should be in the query string parameters")
         try:
             result = float(self.request.params[name])
         except ValueError:
-            raise HTTPBadRequest(
-                "'{}' ({}) parameters should be a number".format(name, self.request.params[name])
+            raise HTTPBadRequest(  # pylint: disable=raise-missing-from
+                f"'{name}' ({self.request.params[name]}) parameters should be a number"
             )
         if not math.isfinite(result):
             raise HTTPBadRequest(
-                "'{}' ({}) parameters should be a finite number".format(name, self.request.params[name])
+                f"'{name}' ({self.request.params[name]}) parameters should be a finite number"
             )
         return result
 
-    @view_config(route_name="raster", renderer="fast_json")
-    def raster(self):
+    @view_config(route_name="raster", renderer="fast_json")  # type: ignore
+    def raster(self) -> Dict[str, Any]:
         lon = self._get_required_finite_float_param("lon")
         lat = self._get_required_finite_float_param("lat")
 
@@ -85,7 +92,7 @@ class Raster:
                 if layer in self.rasters:
                     rasters[layer] = self.rasters[layer]
                 else:
-                    raise HTTPNotFound("Layer {} not found".format(layer))
+                    raise HTTPNotFound(f"Layer {layer} not found")
         else:
             rasters = self.rasters
 
@@ -93,10 +100,10 @@ class Raster:
         for ref in list(rasters.keys()):
             result[ref] = self._get_raster_value(rasters[ref], ref, lon, lat)
 
-        set_common_headers(self.request, "raster", NO_CACHE)
+        set_common_headers(self.request, "raster", Cache.PUBLIC_NO)
         return result
 
-    def _get_data(self, layer, name):
+    def _get_data(self, layer: Dict[str, Any], name: str) -> "fiona.collection.Collection":
         if name not in self.data:
             path = layer["file"]
             if layer.get("type", "shp_index") == "shp_index":
@@ -112,7 +119,9 @@ class Raster:
 
         return self.data[name]
 
-    def _get_raster_value(self, layer, name, lon, lat):
+    def _get_raster_value(
+        self, layer: Dict[str, Any], name: str, lon: float, lat: float
+    ) -> Optional[decimal.Decimal]:
         data = self._get_data(layer, name)
         type_ = layer.get("type", "shp_index")
         if type_ == "shp_index":
@@ -133,21 +142,25 @@ class Raster:
         else:
             raise ValueError("Unsupported type " + type_)
 
-        if "round" in layer:
-            result = self._round(result, layer["round"])
+        result_d = None
+        if "round" in layer and result is not None:
+            result_d = self._round(result, layer["round"])
         elif result is not None:
-            result = Decimal(str(result))
+            result_d = decimal.Decimal(str(result))
 
-        return result
+        return result_d
 
     @staticmethod
-    def _get_value(layer, name, dataset, lon, lat):
+    def _get_value(
+        layer: Dict[str, Any], name: str, dataset: DatasetReader, lon: float, lat: float
+    ) -> Optional[numpy.float32]:
         index = dataset.index(lon, lat)
 
         shape = dataset.shape
+        result: Optional[numpy.float32]
         if 0 <= index[0] < shape[0] and 0 <= index[1] < shape[1]:
 
-            def get_index(index_):
+            def get_index(index_: int) -> Tuple[int, int]:
                 return index_, index_ + 1
 
             result = dataset.read(1, window=(get_index(index[0]), get_index(index[1])))[0][0]
@@ -169,7 +182,12 @@ class Raster:
         return result
 
     @staticmethod
-    def _round(value, round_to):
+    def _round(value: numpy.float32, round_to: float) -> Optional[decimal.Decimal]:
         if value is not None:
-            return Decimal(str(value)).quantize(Decimal(str(round_to)))
+            decimal_value = decimal.Decimal(str(value))
+            try:
+                return decimal_value.quantize(decimal.Decimal(str(round_to)))
+            except decimal.InvalidOperation:
+                LOG.info("Error on rounding %s: %s", decimal_value, traceback.format_exc())
+                return decimal_value
         return None

c2cgeoportal_geoportal/views/resourceproxy.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2011-2019, Camptocamp SA
+# Copyright (c) 2011-2021, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -31,41 +29,45 @@
 import ast
 import logging
 
+import pyramid.request
+import pyramid.response
 from pyramid.httpexceptions import HTTPBadRequest
 from pyramid.view import view_config
 
-from c2cgeoportal_geoportal.lib.caching import NO_CACHE
+from c2cgeoportal_geoportal.lib.common_headers import Cache
 from c2cgeoportal_geoportal.views.proxy import Proxy
 
 LOG = logging.getLogger(__name__)
 
 
 class ResourceProxy(Proxy):
-    def __init__(self, request):  # pragma: no cover
+    """All the views concerned the resources (it's a kind of proxy)."""
+
+    def __init__(self, request: pyramid.request.Request):
         Proxy.__init__(self, request)
         self.request = request
         self.settings = request.registry.settings.get("resourceproxy", {})
 
-    @view_config(route_name="resourceproxy")
-    def proxy(self):  # pragma: no cover
+    @view_config(route_name="resourceproxy")  # type: ignore
+    def proxy(self) -> pyramid.response.Response:
         target = self.request.params.get("target", "")
         targets = self.settings.get("targets", [])
-        if target in targets:  # pragma: no cover
+        if target in targets:
             url = targets[target]
             values = ast.literal_eval(self.request.params.get("values"))
             url = url % values
 
             response = self._proxy(url=url)
 
-            cache_control = NO_CACHE
+            cache_control = Cache.PRIVATE_NO
             content_type = response.headers["Content-Type"]
 
             response = self._build_response(
-                response, response.text, cache_control, "externalresource", content_type=content_type
+                response, response.content, cache_control, "externalresource", content_type=content_type
             )
             for header in response.headers.keys():
                 if header not in self.settings["allowed_headers"]:
                     response.headers.pop(header)
             return response
-        LOG.warning("Target url not found: %s", target)
+        LOG.warning("Target URL not found: %s", target)
         return HTTPBadRequest("URL not allowed")

c2cgeoportal_geoportal/views/shortener.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2013-2019, Camptocamp SA
+# Copyright (c) 2013-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -28,47 +26,51 @@
 # either expressed or implied, of the FreeBSD Project.
 
 
-from datetime import datetime
 import logging
 import random
 import string
+from datetime import datetime
+from typing import Dict
 from urllib.parse import urlparse
 
+import pyramid.request
 from pyramid.httpexceptions import HTTPBadRequest, HTTPFound, HTTPInternalServerError, HTTPNotFound
 from pyramid.view import view_config
 
 from c2cgeoportal_commons.lib.email_ import send_email_config
 from c2cgeoportal_commons.models import DBSession
 from c2cgeoportal_commons.models.static import Shorturl
-from c2cgeoportal_geoportal.lib.caching import NO_CACHE, set_common_headers
+from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
 
 logger = logging.getLogger(__name__)
 
 
 class Shortener:
-    def __init__(self, request):
+    """All the views conserne the shortener."""
+
+    def __init__(self, request: pyramid.request.Request):
         self.request = request
         self.settings = request.registry.settings.get("shortener", {})
         self.short_bases = [self.request.route_url("shortener_get", ref="")]
         if "base_url" in self.settings:
             self.short_bases.append(self.settings["base_url"])
 
-    @view_config(route_name="shortener_get")
-    def get(self):
+    @view_config(route_name="shortener_get")  # type: ignore
+    def get(self) -> HTTPFound:
         ref = self.request.matchdict["ref"]
         short_urls = DBSession.query(Shorturl).filter(Shorturl.ref == ref).all()
 
         if len(short_urls) != 1:
-            raise HTTPNotFound("Ref '{0!s}' not found".format(ref))
+            raise HTTPNotFound(f"Ref '{ref!s}' not found")
 
         short_urls[0].nb_hits += 1
         short_urls[0].last_hit = datetime.now()
 
-        set_common_headers(self.request, "shortener", NO_CACHE)
+        set_common_headers(self.request, "shortener", Cache.PUBLIC_NO)
         return HTTPFound(location=short_urls[0].url)
 
-    @view_config(route_name="shortener_create", renderer="json")
-    def create(self):
+    @view_config(route_name="shortener_create", renderer="json")  # type: ignore
+    def create(self) -> Dict[str, str]:
 
         if "url" not in self.request.params:
             raise HTTPBadRequest("The parameter url is required")
@@ -76,19 +78,22 @@ class Shortener:
         url = self.request.params["url"]
 
         # see: https://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline
-        if len(url) > 8190:  # pragma: no cover
-            raise HTTPBadRequest("The parameter url is too long ({} > {})".format(len(url), 8190))
+        if len(url) > 8190:
+            raise HTTPBadRequest(f"The parameter url is too long ({len(url)} > {8190})")
 
         # Check that it is an internal URL...
         uri_parts = urlparse(url)
-        hostname = uri_parts.hostname
         if "allowed_hosts" in self.settings:
-            if hostname not in self.settings["allowed_hosts"]:  # pragma: no cover
-                raise HTTPBadRequest("The requested host is not allowed.")
+            if uri_parts.netloc not in self.settings["allowed_hosts"]:
+                raise HTTPBadRequest(
+                    f"The requested host '{uri_parts.netloc}' is not part of allowed hosts: "
+                    f"{', '.join(self.settings['allowed_hosts'])}"
+                )
         else:
+            hostname = uri_parts.hostname
             if hostname != self.request.server_name:
                 raise HTTPBadRequest(
-                    "The requested host '{0!s}' should be '{1!s}'".format(hostname, self.request.server_name)
+                    f"The requested host '{hostname!s}' should be '{self.request.server_name!s}'"
                 )
 
         shortened = False
@@ -102,16 +107,16 @@ class Shortener:
         tries = 0
         while not shortened:
             ref = "".join(
-                random.choice(string.ascii_letters + string.digits)
+                random.choice(string.ascii_letters + string.digits)  # nosec
                 for i in range(self.settings.get("length", 4))
             )
             test_url = DBSession.query(Shorturl).filter(Shorturl.ref == ref).all()
             if not test_url:
                 break
-            tries += 1  # pragma: no cover
-            if tries > 20:  # pragma: no cover
+            tries += 1
+            if tries > 20:
                 message = "No free ref found, considered to increase the length"
-                logging.error(message)
+                logger.error(message)
                 raise HTTPInternalServerError(message)
 
         user_email = self.request.user.email if self.request.user is not None else None
@@ -131,7 +136,7 @@ class Shortener:
         else:
             s_url = self.request.route_url("shortener_get", ref=ref)
 
-        if email is not None:  # pragma: no cover
+        if email is not None:
             send_email_config(
                 self.request.registry.settings,
                 "shortener",
@@ -139,7 +144,9 @@ class Shortener:
                 full_url=url,
                 short_url=s_url,
                 message=self.request.params.get("message", ""),
+                application_url=self.request.route_url("base"),
+                current_url=self.request.current_route_url(),
             )
 
-        set_common_headers(self.request, "shortener", NO_CACHE)
+        set_common_headers(self.request, "shortener", Cache.PRIVATE_NO)
         return {"short_url": s_url}