c2cgeoportal-geoportal 2.5.0.100__py2.py3-none-any.whl → 2.7.1.156__py2.py3-none-any.whl

c2cgeoportal_geoportal/lib/caching.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2012-2020, Camptocamp SA
+# Copyright (c) 2012-2021, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -30,89 +28,92 @@
 
 import inspect
 import logging
-from typing import Any, Dict, List
-
-from dogpile.cache.api import NO_VALUE
-from dogpile.cache.backends.redis import RedisBackend
-from dogpile.cache.region import make_region
-from dogpile.cache.util import compat, sha1_mangle_key
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+
+import sqlalchemy.ext.declarative
+from dogpile.cache.api import NO_VALUE, CacheBackend
+from dogpile.cache.backends.memory import MemoryBackend
+from dogpile.cache.backends.redis import RedisBackend, RedisSentinelBackend
+from dogpile.cache.region import CacheRegion, make_region
+from dogpile.cache.util import sha1_mangle_key
 from pyramid.request import Request
 from sqlalchemy.orm.util import identity_key
 
 from c2cgeoportal_commons.models import Base
 
+if TYPE_CHECKING:
+    from dogpile.cache.api import SerializedReturnType
+else:
+    SerializedReturnType = Any
+
 LOG = logging.getLogger(__name__)
 _REGION: Dict[str, Any] = {}
 MEMORY_CACHE_DICT: Dict[str, Any] = {}
 
 
-def map_dbobject(item):
-
+def map_dbobject(
+    item: sqlalchemy.ext.declarative.ConcreteBase,
+) -> sqlalchemy.ext.declarative.ConcreteBase:
+    """Get an cache identity key for the cache."""
     return identity_key(item) if isinstance(item, Base) else item
 
 
-def keygen_function(namespace, function):
+def keygen_function(namespace: Any, function: Callable[..., Any]) -> Callable[..., str]:
     """
-    Return a function that generates a string
-    key, based on a given function as well as
-    arguments to the returned function itself.
+    Return a function that generates a string key.
+
+    Based on a given function as well as arguments to the returned function itself.
 
-    This is used by :meth:`.CacheRegion.cache_on_arguments`
-    to generate a cache key from a decorated function.
+    This is used by :meth:`.CacheRegion.cache_on_arguments` to generate a cache key from a decorated function.
     """
 
     if namespace is None:
         namespace = (function.__module__, function.__name__)
-    else:  # pragma: no cover
+    else:
         namespace = (function.__module__, function.__name__, namespace)
 
     args = inspect.getfullargspec(function)
     ignore_first_argument = args[0] and args[0][0] in ("self", "cls")
 
-    def generate_key(*args, **kw):
-        if kw:  # pragma: no cover
+    def generate_key(*args: Any, **kw: Any) -> str:
+        if kw:
             raise ValueError("key creation function does not accept keyword arguments.")
         parts: List[str] = []
         parts.extend(namespace)
         if ignore_first_argument:
             args = args[1:]
         new_args: List[str] = [arg for arg in args if not isinstance(arg, Request)]
-        parts.extend(map(compat.text_type, map(map_dbobject, new_args)))
+        parts.extend(map(str, map(map_dbobject, new_args)))
         return "|".join(parts)
 
     return generate_key
 
 
-def init_region(conf, region):
-    """
-    Initialize the caching module.
-    """
+def init_region(conf: Dict[str, Any], region: str) -> CacheRegion:
+    """Initialize the caching module."""
     cache_region = get_region(region)
     _configure_region(conf, cache_region)
     return cache_region
 
 
-def _configure_region(conf, cache_region):
-    global MEMORY_CACHE_DICT
-    kwargs = {"replace_existing_backend": True}
+def _configure_region(conf: Dict[str, Any], cache_region: CacheRegion) -> None:
+    kwargs: Dict[str, Any] = {"replace_existing_backend": True}
     backend = conf["backend"]
     kwargs.update({k: conf[k] for k in conf if k != "backend"})
-    kwargs.setdefault("arguments", {})  # type: ignore
-    kwargs["arguments"]["cache_dict"] = MEMORY_CACHE_DICT  # type: ignore
+    kwargs.setdefault("arguments", {})
+    kwargs["arguments"]["cache_dict"] = MEMORY_CACHE_DICT
     cache_region.configure(backend, **kwargs)
 
 
-def get_region(region):
-    """
-    Return a cache region.
-    """
-    global _REGION
+def get_region(region: str) -> CacheRegion:
+    """Return a cache region."""
     if region not in _REGION:
         _REGION[region] = make_region(function_key_generator=keygen_function)
     return _REGION[region]
 
 
-def invalidate_region(region=None):
+def invalidate_region(region: Optional[str] = None) -> None:
+    """Invalidate a cache region."""
     if region is None:
         for cache_region in _REGION.values():
             cache_region.invalidate()
@@ -120,144 +121,49 @@ def invalidate_region(region=None):
         get_region(region).invalidate()
 
 
-class HybridBackend(RedisBackend):
-    """
-    A memory and redis backend
-    """
+class HybridRedisBackend(CacheBackend):  # type: ignore
+    """A Dogpile cache backend with a memory cache backend in front of a Redis backend for performance."""
 
-    def __init__(self, arguments):
-        self._cache = arguments.pop("cache_dict", {})
+    def __init__(self, arguments: Dict[str, Any]):  # pylint: disable=super-init-not-called
         self._use_memory_cache = not arguments.pop("disable_memory_cache", False)
+        self._memory = MemoryBackend({"cache_dict": arguments.pop("cache_dict", {})})
+        self._redis = RedisBackend(arguments)
 
-        super().__init__(arguments)
-
-    def get(self, key):
-        value = self._cache.get(key, NO_VALUE)
+    def get(self, key: str) -> SerializedReturnType:
+        value = self._memory.get(key)
         if value == NO_VALUE:
-            value = super().get(sha1_mangle_key(key.encode()))
-        if value != NO_VALUE and self._use_memory_cache:
-            self._cache[key] = value
+            val = self._redis.get_serialized(sha1_mangle_key(key.encode()))
+            if val in (None, NO_VALUE):
+                return NO_VALUE
+            value = self._redis.deserializer(val)
+            if value != NO_VALUE and self._use_memory_cache:
+                self._memory.set(key, value)
         return value
 
-    def get_multi(self, keys):
+    def get_multi(self, keys: List[str]) -> List[SerializedReturnType]:
         return [self.get(key) for key in keys]
 
-    def set(self, key, value):
+    def set(self, key: str, value: SerializedReturnType) -> None:
         if self._use_memory_cache:
-            self._cache[key] = value
-        super().set(sha1_mangle_key(key.encode()), value)
+            self._memory.set(key, value)
+        self._redis.set_serialized(sha1_mangle_key(key.encode()), self._redis.serializer(value))
 
-    def set_multi(self, mapping):
+    def set_multi(self, mapping: Dict[str, SerializedReturnType]) -> None:
         for key, value in mapping.items():
             self.set(key, value)
 
-    def delete(self, key):
-        self._cache.pop(key, None)
-        super().delete(key)
-
-
-NO_CACHE = 0
-PUBLIC_CACHE = 1
-PRIVATE_CACHE = 2
-
-CORS_METHODS = "HEAD, GET, POST, PUT, DELETE"
-
-
-def _set_cors_headers(request, response, service_name, service_headers_settings, credentials):
-    """
-    Handle CORS requests, as specified in https://www.w3.org/TR/cors/
-    """
-    response.vary = (response.vary or ()) + ("Origin",)
-
-    if "Origin" not in request.headers:
-        return  # Not a CORS request if this header is missing
-    origin = request.headers["Origin"]
-
-    if request.method == "OPTIONS" and "Access-Control-Request-Method" not in request.headers:
-        LOG.warning("CORS preflight query missing the Access-Control-Request-Method header")
-        return
+    def delete(self, key: str) -> None:
+        self._memory.delete(key)
+        self._redis.delete(key)
 
-    allowed_origins = service_headers_settings.get("access_control_allow_origin", [])
-    if origin not in allowed_origins:
-        if "*" in allowed_origins:
-            origin = "*"
-            credentials = False  # Force no credentials
-        else:
-            LOG.warning("CORS query not allowed for origin=%s, service=%s", origin, service_name)
-            return
+    def delete_multi(self, keys: List[str]) -> None:
+        self._memory.delete_multi(keys)
+        self._redis.delete_multi(keys)
 
-    response.headers.update(
-        {"Access-Control-Allow-Origin": origin, "Access-Control-Allow-Methods": CORS_METHODS}
-    )
 
-    max_age = service_headers_settings.get("access_control_max_age", 3600)
-    response.headers["Access-Control-Max-Age"] = str(max_age)
+class HybridRedisSentinelBackend(HybridRedisBackend):
+    """Same as HybridRedisBackend but using the Redis Sentinel."""
 
-    if credentials:
-        response.headers["Access-Control-Allow-Credentials"] = "true"
-
-    if request.method != "OPTIONS":
-        return
-
-    response.cache_control.max_age = max_age
-
-    if not service_headers_settings or "access_control_allow_origin" not in service_headers_settings:
-        LOG.warning("CORS query not configured for service=%s", service_name)
-        return
-
-    requested_headers = request.headers.get("Access-Control-Request-Headers", False)
-    if requested_headers:
-        # For the moment, we allow all requested headers
-        response.headers["Access-Control-Allow-Headers"] = requested_headers
-
-    # If we start using headers in responses, we'll have to add
-    # Access-Control-Expose-Headers
-
-
-def _set_common_headers(request, response, service_headers_settings, cache, content_type):
-    """
-    Set the common headers
-    """
-
-    response.headers.update(service_headers_settings.get("headers", {}))
-
-    if cache == NO_CACHE:
-        response.cache_control.no_cache = True
-        response.cache_control.max_age = 0
-    elif cache == PUBLIC_CACHE:
-        response.cache_control.public = True
-    elif cache == PRIVATE_CACHE:
-        if request.user is not None:
-            response.cache_control.private = True
-        else:
-            response.cache_control.public = True
-    else:  # pragma: no cover
-        raise Exception("Invalid cache type")
-
-    if cache != NO_CACHE:
-        max_age = service_headers_settings.get("cache_control_max_age", 3600)
-
-        response.cache_control.max_age = max_age
-        if max_age == 0:
-            response.cache_control.no_cache = True
-
-    if content_type is not None:
-        response.content_type = content_type
-
-    return response
-
-
-def set_common_headers(request, service_name, cache, response=None, credentials=True, content_type=None):
-    """
-    Set the common headers
-    """
-    if response is None:
-        response = request.response
-
-    headers_settings = request.registry.settings.get("headers", {})
-    service_headers_settings = headers_settings.get(service_name, {})
-
-    _set_cors_headers(request, response, service_name, service_headers_settings, credentials)
-    if request.method == "OPTIONS":
-        return response
-    return _set_common_headers(request, response, service_headers_settings, cache, content_type)
+    def __init__(self, arguments: Dict[str, Any]):
+        super().__init__(arguments)
+        self._redis = RedisSentinelBackend(arguments)

c2cgeoportal_geoportal/lib/check_collector.py

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2011-2020, Camptocamp SA
+# Copyright (c) 2011-2021, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -27,8 +25,11 @@
 # of the authors and should not be interpreted as representing official policies,
 # either expressed or implied, of the FreeBSD Project.
 import logging
-from typing import Any
+from typing import Any, Dict, Optional, cast
 
+import c2cwsgiutils.health_check
+import pyramid.config
+import pyramid.request
 import requests
 
 from c2cgeoportal_geoportal.lib.checker import build_url
@@ -36,7 +37,12 @@ from c2cgeoportal_geoportal.lib.checker import build_url
 LOG = logging.getLogger(__name__)
 
 
-def init(config, health_check):
+def init(config: pyramid.config.Configurator, health_check: c2cwsgiutils.health_check.HealthCheck) -> None:
+    """
+    Initialize the check collector.
+
+    Add him in the c2cwsgichecks.
+    """
     global_settings = config.get_settings()
     if "check_collector" not in global_settings:
         return
@@ -48,25 +54,26 @@ def init(config, health_check):
     for host in settings["hosts"]:
 
         class Check:
-            def __init__(self, host):
+            def __init__(self, host: Dict[str, Any]):
                 self.host = host
 
-            def __call__(self, request) -> Any:  # pylint: disable=inconsistent-return-statements
+            def __call__(self, request: pyramid.request.Request) -> Optional[Dict[str, Any]]:
                 params = request.params
                 display = self.host["display"]
                 if "host" not in params or display == params["host"]:
                     url_headers = build_url(
                         "check_collector",
-                        "%s/%s/health_check" % (self.host["url"].rstrip("/"), c2c_base.strip("/")),
+                        f"{self.host['url'].rstrip('/')}/{c2c_base.strip('/')}/health_check",
                         request,
                     )
                     r = requests.get(
                         params={"max_level": str(self.host.get("max_level", max_level))},
                         timeout=120,
-                        **url_headers,
+                        **url_headers,  # type: ignore
                     )
                     r.raise_for_status()
-                    return r.json()
+                    return cast(Dict[str, Any], r.json())
+                return None
 
         health_check.add_custom_check(
             name="check_collector_" + host["display"], check_cb=Check(host), level=settings["level"]