c2cwsgiutils 6.0.10.dev9__py3-none-any.whl → 6.1.0__py3-none-any.whl

c2cwsgiutils/db.py

@@ -1,10 +1,11 @@
 """SQLalchemy models."""
+
 import logging
 import re
 import warnings
 from collections.abc import Iterable
 from re import Pattern
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
+from typing import Any, Callable, Optional, Union, cast
 
 import pyramid.config
 import pyramid.config.settings
@@ -16,18 +17,14 @@ import zope.sqlalchemy
 from sqlalchemy import engine_from_config
 from zope.sqlalchemy import register
 
-if TYPE_CHECKING:
-    scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
-    sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session]
-else:
-    scoped_session = sqlalchemy.orm.scoped_session
-    sessionmaker = sqlalchemy.orm.sessionmaker
+_scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
+_sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session]  # pylint: disable=unsubscriptable-object
 
 
-LOG = logging.getLogger(__name__)
-RE_COMPILE: Callable[[str], Pattern[str]] = re.compile
+_LOG = logging.getLogger(__name__)
+_RE_COMPILE: Callable[[str], Pattern[str]] = re.compile
 
-force_readonly = False
+FORCE_READONLY = False
 
 
 class Tweens:
@@ -44,7 +41,7 @@ def setup_session(
     force_master: Optional[Iterable[str]] = None,
     force_slave: Optional[Iterable[str]] = None,
 ) -> tuple[
-    Union[sqlalchemy.orm.Session, scoped_session],
+    Union[sqlalchemy.orm.Session, _scoped_session],
     sqlalchemy.engine.Engine,
     sqlalchemy.engine.Engine,
 ]:
@@ -59,8 +56,7 @@ def setup_session(
     Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
     path includes the route_prefix.
 
-    Keyword Arguments:
-
+    Arguments:
         config: The pyramid Configuration object
         master_prefix: The prefix for the master connection configuration entries in the application \
                           settings
@@ -83,7 +79,7 @@ def setup_session(
 
     # Setup a slave DB connection and add a tween to use it.
     if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
-        LOG.info("Using a slave DB for reading %s", master_prefix)
+        _LOG.info("Using a slave DB for reading %s", master_prefix)
         ro_engine = sqlalchemy.engine_from_config(config.get_settings(), slave_prefix + ".")
         ro_engine.c2c_name = slave_prefix  # type: ignore
         tween_name = master_prefix.replace(".", "_")
@@ -104,7 +100,7 @@ def create_session(
     force_master: Optional[Iterable[str]] = None,
     force_slave: Optional[Iterable[str]] = None,
     **engine_config: Any,
-) -> Union[sqlalchemy.orm.Session, scoped_session]:
+) -> Union[sqlalchemy.orm.Session, _scoped_session]:
     """
     Create a SQLAlchemy session.
 
@@ -116,8 +112,7 @@ def create_session(
     Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
     path includes the route_prefix.
 
-    Keyword Arguments:
-
+    Arguments:
         config: The pyramid Configuration object. If None, only master is used
         name: The name of the check
         url: The URL for the master DB
@@ -139,7 +134,7 @@ def create_session(
 
     # Setup a slave DB connection and add a tween to use it.
     if url != slave_url and config is not None:
-        LOG.info("Using a slave DB for reading %s", name)
+        _LOG.info("Using a slave DB for reading %s", name)
         ro_engine = sqlalchemy.create_engine(slave_url, **engine_config)
         _add_tween(config, name, db_session, force_master, force_slave)
         rw_engine.c2c_name = name + "_master"  # type: ignore
@@ -156,15 +151,15 @@ def create_session(
 def _add_tween(
     config: pyramid.config.Configurator,
     name: str,
-    db_session: scoped_session,
+    db_session: _scoped_session,
     force_master: Optional[Iterable[str]],
     force_slave: Optional[Iterable[str]],
 ) -> None:
     master_paths: Iterable[Pattern[str]] = (
-        list(map(RE_COMPILE, force_master)) if force_master is not None else []
+        list(map(_RE_COMPILE, force_master)) if force_master is not None else []
     )
     slave_paths: Iterable[Pattern[str]] = (
-        list(map(RE_COMPILE, force_slave)) if force_slave is not None else []
+        list(map(_RE_COMPILE, force_slave)) if force_slave is not None else []
     )
 
     def db_chooser_tween_factory(
@@ -181,18 +176,18 @@ def _add_tween(
             old = session.bind
             method_path: Any = f"{request.method} {request.path}"
             has_force_master = any(r.match(method_path) for r in master_paths)
-            if force_readonly or (
+            if FORCE_READONLY or (
                 not has_force_master
                 and (request.method in ("GET", "OPTIONS") or any(r.match(method_path) for r in slave_paths))
             ):
-                LOG.debug(
+                _LOG.debug(
                     "Using %s database for: %s",
                     db_session.c2c_ro_bind.c2c_name,  # type: ignore
                     method_path,
                 )
                 session.bind = db_session.c2c_ro_bind  # type: ignore
             else:
-                LOG.debug(
+                _LOG.debug(
                     "Using %s database for: %s",
                     db_session.c2c_rw_bind.c2c_name,  # type: ignore
                     method_path,
@@ -210,7 +205,7 @@ def _add_tween(
     config.add_tween("c2cwsgiutils.db.tweens." + name, over="pyramid_tm.tm_tween_factory")
 
 
-class SessionFactory(sessionmaker):
+class SessionFactory(_sessionmaker):
     """The custom session factory that manage the read only and read write sessions."""
 
     def __init__(
@@ -222,42 +217,43 @@ class SessionFactory(sessionmaker):
     ):
         super().__init__()
         self.master_paths: Iterable[Pattern[str]] = (
-            list(map(RE_COMPILE, force_master)) if force_master else []
+            list(map(_RE_COMPILE, force_master)) if force_master else []
         )
-        self.slave_paths: Iterable[Pattern[str]] = list(map(RE_COMPILE, force_slave)) if force_slave else []
+        self.slave_paths: Iterable[Pattern[str]] = list(map(_RE_COMPILE, force_slave)) if force_slave else []
         self.ro_engine = ro_engine
         self.rw_engine = rw_engine
 
     def engine_name(self, readwrite: bool) -> str:
+        """Get the engine name."""
         if readwrite:
             return cast(str, self.rw_engine.c2c_name)  # type: ignore
         return cast(str, self.ro_engine.c2c_name)  # type: ignore
 
     def __call__(  # type: ignore
         self, request: Optional[pyramid.request.Request], readwrite: Optional[bool] = None, **local_kw: Any
-    ) -> scoped_session:
+    ) -> _scoped_session:
         if readwrite is not None:
-            if readwrite and not force_readonly:
-                LOG.debug("Using %s database", self.rw_engine.c2c_name)  # type: ignore
+            if readwrite and not FORCE_READONLY:
+                _LOG.debug("Using %s database", self.rw_engine.c2c_name)  # type: ignore
                 self.configure(bind=self.rw_engine)
             else:
-                LOG.debug("Using %s database", self.ro_engine.c2c_name)  # type: ignore
+                _LOG.debug("Using %s database", self.ro_engine.c2c_name)  # type: ignore
                 self.configure(bind=self.ro_engine)
         else:
             assert request is not None
             method_path: str = f"{request.method} {request.path}" if request is not None else ""
             has_force_master = any(r.match(method_path) for r in self.master_paths)
-            if force_readonly or (
+            if FORCE_READONLY or (
                 not has_force_master
                 and (
                     request.method in ("GET", "OPTIONS")
                     or any(r.match(method_path) for r in self.slave_paths)
                 )
             ):
-                LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path)  # type: ignore
+                _LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path)  # type: ignore
                 self.configure(bind=self.ro_engine)
             else:
-                LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path)  # type: ignore
+                _LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path)  # type: ignore
                 self.configure(bind=self.rw_engine)
         return super().__call__(**local_kw)  # type: ignore
 
@@ -271,15 +267,15 @@ def get_engine(
 
 def get_session_factory(
     engine: sqlalchemy.engine.Engine,
-) -> sessionmaker:
+) -> _sessionmaker:
     """Get the session factory from the engine."""
-    factory = sessionmaker()
+    factory = _sessionmaker()
     factory.configure(bind=engine)
     return factory
 
 
 def get_tm_session(
-    session_factory: sessionmaker,
+    session_factory: _sessionmaker,
     transaction_manager: transaction.TransactionManager,
 ) -> sqlalchemy.orm.Session:
     """
@@ -347,7 +343,7 @@ def get_tm_session_pyramid(
     session_factory: SessionFactory,
     transaction_manager: transaction.TransactionManager,
     request: pyramid.request.Request,
-) -> scoped_session:
+) -> _scoped_session:
     """
     Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.
 
@@ -368,8 +364,7 @@ def init(
     """
     Initialize the database for a Pyramid app.
 
-    Keyword Arguments:
-
+    Arguments:
         config: The pyramid Configuration object
         master_prefix: The prefix for the master connection configuration entries in the application \
                           settings
@@ -391,7 +386,7 @@ def init(
 
         # Setup a slave DB connection and add a tween to use it.
         if slave_prefix and settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
-            LOG.info("Using a slave DB for reading %s", master_prefix)
+            _LOG.info("Using a slave DB for reading %s", master_prefix)
             ro_engine = get_engine(config.get_settings(), slave_prefix + ".")
             ro_engine.c2c_name = slave_prefix  # type: ignore
         else:

c2cwsgiutils/db_maintenance_view.py

@@ -7,10 +7,10 @@ import pyramid.request
 
 from c2cwsgiutils import auth, broadcast, config_utils, db, redis_utils
 
-LOG = logging.getLogger(__name__)
-CONFIG_KEY = "c2c.db_maintenance_view_enabled"
-ENV_KEY = "C2C_DB_MAINTENANCE_VIEW_ENABLED"
-REDIS_PREFIX = "c2c_db_maintenance_"
+_LOG = logging.getLogger(__name__)
+_CONFIG_KEY = "c2c.db_maintenance_view_enabled"
+_ENV_KEY = "C2C_DB_MAINTENANCE_VIEW_ENABLED"
+_REDIS_PREFIX = "c2c_db_maintenance_"
 
 
 def install_subscriber(config: pyramid.config.Configurator) -> None:
@@ -21,7 +21,7 @@ def install_subscriber(config: pyramid.config.Configurator) -> None:
 
 def includeme(config: pyramid.config.Configurator) -> None:
     """Install the view to configure the loggers, if configured to do so."""
-    if auth.is_enabled(config, ENV_KEY, CONFIG_KEY):
+    if auth.is_enabled(config, _ENV_KEY, _CONFIG_KEY):
         config.add_route(
             "c2c_db_maintenance",
             config_utils.get_base_path(config) + r"/db/maintenance",
@@ -29,7 +29,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
         )
         config.add_view(_db_maintenance, route_name="c2c_db_maintenance", renderer="fast_json", http_cache=0)
         _restore(config)
-        LOG.info("Enabled the /db/maintenance API")
+        _LOG.info("Enabled the /db/maintenance API")
 
 
 def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
@@ -38,7 +38,7 @@ def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
     if readonly_param is not None:
         readonly = readonly_param.lower() == "true"
 
-        LOG.critical("Readonly DB status changed from %s to %s", db.force_readonly, readonly)
+        _LOG.critical("Readonly DB status changed from %s to %s", db.FORCE_READONLY, readonly)
         _set_readonly(value=readonly)
         _store(request.registry.settings, readonly)
         return {"status": 200, "readonly": readonly}
@@ -51,7 +51,7 @@ def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
 
 @broadcast.decorator(expect_answers=True)
 def _set_readonly(value: bool) -> bool:
-    db.force_readonly = value
+    db.FORCE_READONLY = value
     return True
 
 
@@ -59,24 +59,24 @@ def _restore(config: pyramid.config.Configurator) -> None:
     try:
         readonly = _get_redis_value(config.get_settings())
         if readonly is not None:
-            LOG.debug("Restoring readonly DB status to %s", readonly)
-            db.force_readonly = readonly == "true"
+            _LOG.debug("Restoring readonly DB status to %s", readonly)
+            db.FORCE_READONLY = readonly == "true"
     except ImportError:
         pass  # don't have redis
     except Exception:  # pylint: disable=broad-except
         # survive an error since crashing now can have bad consequences for the service. :/
-        LOG.error("Cannot restore readonly DB status.", exc_info=True)
+        _LOG.error("Cannot restore readonly DB status.", exc_info=True)
 
 
 def _store(settings: Mapping[str, Any], readonly: bool) -> None:
     master, _, _ = redis_utils.get(settings)
     if master is not None:
-        master.set(REDIS_PREFIX + "force_readonly", "true" if readonly else "false")
+        master.set(_REDIS_PREFIX + "force_readonly", "true" if readonly else "false")
 
 
 def _get_redis_value(settings: Mapping[str, Any]) -> Optional[str]:
     _, slave, _ = redis_utils.get(settings)
     if slave is not None:
-        value = slave.get(REDIS_PREFIX + "force_readonly")
+        value = slave.get(_REDIS_PREFIX + "force_readonly")
         return str(value) if value else None
     return None

c2cwsgiutils/debug/__init__.py

@@ -23,7 +23,7 @@ def init(config: pyramid.config.Configurator) -> None:
 def includeme(config: pyramid.config.Configurator) -> None:
     """Initialize the debug tools."""
     if auth.is_enabled(config, ENV_KEY, CONFIG_KEY):
-        from c2cwsgiutils.debug import _views
+        from c2cwsgiutils.debug import _views  # pylint: disable=import-outside-toplevel
 
         init_daemon(config)
         _views.init(config)
@@ -37,6 +37,6 @@ def init_daemon(config: Optional[pyramid.config.Configurator] = None) -> None:
     those requests.
     """
     if config_utils.env_or_config(config, ENV_KEY, CONFIG_KEY, type_=config_utils.config_bool):
-        from c2cwsgiutils.debug import _listeners
+        from c2cwsgiutils.debug import _listeners  # pylint: disable=import-outside-toplevel
 
         _listeners.init()

c2cwsgiutils/debug/_listeners.py

@@ -90,7 +90,7 @@ def _dump_memory_impl(
                 for i in sorted(mod_counts.items(), key=lambda x: -x[1])[:limit]
             ]
         elif analyze_type == "linecache":
-            import linecache
+            import linecache  # pylint: disable=import-outside-toplevel
 
             cache = linecache.cache
             result[analyze_type]["biggest_objects"] = sorted(

c2cwsgiutils/debug/_views.py

@@ -16,8 +16,8 @@ from pyramid.httpexceptions import HTTPException, exception_response
 from c2cwsgiutils import auth, broadcast, config_utils
 from c2cwsgiutils.debug.utils import dump_memory_maps, get_size
 
-LOG = logging.getLogger(__name__)
-SPACE_RE = re.compile(r" +")
+_LOG = logging.getLogger(__name__)
+_SPACE_RE = re.compile(r" +")
 
 
 def _beautify_stacks(source: list[Mapping[str, Any]]) -> list[Mapping[str, Any]]:
@@ -80,7 +80,7 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
     except Exception:  # nosec  # pylint: disable=broad-except
         pass
 
-    LOG.debug("checking memory growth for %s", path)
+    _LOG.debug("checking memory growth for %s", path)
 
     peak_stats: dict[Any, Any] = {}
     for i in range(3):
@@ -91,10 +91,10 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
     response = None
     try:
         response = request.invoke_subrequest(sub_request)
-        LOG.debug("response was %d", response.status_code)
+        _LOG.debug("response was %d", response.status_code)
 
     except HTTPException as ex:
-        LOG.debug("response was %s", str(ex))
+        _LOG.debug("response was %s", str(ex))
 
     del response
 
@@ -142,6 +142,7 @@ def _error(request: pyramid.request.Request) -> Any:
 
 
 def _time(request: pyramid.request.Request) -> Any:
+    del request  # unused
     return {
         "local_time": str(datetime.now()),
         "gmt_time": str(datetime.utcnow()),
@@ -211,4 +212,4 @@ def init(config: pyramid.config.Configurator) -> None:
     _add_view(config, "error", "error", _error)
     _add_view(config, "time", "time", _time)
     _add_view(config, "show_refs", "show_refs.dot", _show_refs)
-    LOG.info("Enabled the /debug/... API")
+    _LOG.info("Enabled the /debug/... API")

c2cwsgiutils/debug/utils.py

@@ -8,18 +8,18 @@ from types import FunctionType, ModuleType
 from typing import Any
 
 # 7ff7d33bd000-7ff7d33be000 r--p 00000000 00:65 49                         /usr/lib/toto.so
-SMAPS_LOCATION_RE = re.compile(r"^[0-9a-f]+-[0-9a-f]+ +.... +[0-9a-f]+ +[^ ]+ +\d+ +(.*)$")
+_SMAPS_LOCATION_RE = re.compile(r"^[0-9a-f]+-[0-9a-f]+ +.... +[0-9a-f]+ +[^ ]+ +\d+ +(.*)$")
 
 # Size:                  4 kB
-SMAPS_ENTRY_RE = re.compile(r"^([\w]+): +(\d+) kB$")
+_SMAPS_ENTRY_RE = re.compile(r"^([\w]+): +(\d+) kB$")
 
-BLACKLIST = type, ModuleType, FunctionType
-LOG = logging.getLogger(__name__)
+_BLACKLIST = type, ModuleType, FunctionType
+_LOG = logging.getLogger(__name__)
 
 
 def get_size(obj: Any) -> int:
     """Get the sum size of object & members."""
-    if isinstance(obj, BLACKLIST):
+    if isinstance(obj, _BLACKLIST):
         return 0
     seen_ids: set[int] = set()
     size = 0
@@ -27,7 +27,7 @@ def get_size(obj: Any) -> int:
     while objects:
         need_referents = []
         for obj_ in objects:
-            if not isinstance(obj_, BLACKLIST) and id(obj_) not in seen_ids:
+            if not isinstance(obj_, _BLACKLIST) and id(obj_) not in seen_ids:
                 seen_ids.add(id(obj_))
                 size += sys.getsizeof(obj_)
                 need_referents.append(obj_)
@@ -45,11 +45,11 @@ def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
         sizes: dict[str, Any] = {}
         for line in input_:
             line = line.rstrip("\n")
-            matcher = SMAPS_LOCATION_RE.match(line)
+            matcher = _SMAPS_LOCATION_RE.match(line)
             if matcher:
                 cur_dict = sizes.setdefault(matcher.group(1), defaultdict(int))
             else:
-                matcher = SMAPS_ENTRY_RE.match(line)
+                matcher = _SMAPS_ENTRY_RE.match(line)
                 if matcher:
                     name = matcher.group(1)
                     if name in ("Size", "Rss", "Pss"):
@@ -59,5 +59,5 @@ def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
                     and not line.startswith("ProtectionKey:")
                     and not line.startswith("THPeligible:")
                 ):
-                    LOG.debug("Don't know how to parse /proc/%s/smaps line: %s", pid, line)
+                    _LOG.debug("Don't know how to parse /proc/%s/smaps line: %s", pid, line)
         return [{"name": name, **value} for name, value in sizes.items() if value.get("pss_kb", 0) > 0]

c2cwsgiutils/errors.py

@@ -1,4 +1,5 @@
 """Install exception views to have nice JSON error pages."""
+
 import logging
 import os
 import traceback
@@ -13,14 +14,12 @@ from webob.request import DisconnectionError
 
 from c2cwsgiutils import auth, config_utils
 
-DEVELOPMENT = os.environ.get("DEVELOPMENT", "0") != "0"
-DEPRECATED_CONFIG_KEY = "c2c.error_details_secret"
-DEPRECATED_ENV_KEY = "ERROR_DETAILS_SECRET"
+_DEVELOPMENT = os.environ.get("DEVELOPMENT", "0") != "0"
 
-LOG = logging.getLogger(__name__)
-STATUS_LOGGER = {
-    401: LOG.debug,
-    500: LOG.error
+_LOG = logging.getLogger(__name__)
+_STATUS_LOGGER = {
+    401: _LOG.debug,
+    500: _LOG.error,
     # The rest are warnings
 }
 
@@ -52,7 +51,7 @@ def _do_error(
     request: pyramid.request.Request,
     status: int,
     exception: Exception,
-    logger: Callable[..., None] = LOG.error,
+    logger: Callable[..., None] = _LOG.error,
     reduce_info_sent: Callable[[Exception], None] = lambda e: None,
 ) -> pyramid.response.Response:
     logger(
@@ -81,7 +80,7 @@ def _do_error(
 
 def _http_error(exception: HTTPException, request: pyramid.request.Request) -> Any:
     if request.method != "OPTIONS":
-        log = STATUS_LOGGER.get(exception.status_code, LOG.warning)
+        log = _STATUS_LOGGER.get(exception.status_code, _LOG.warning)
         log(
             "%s %s returned status code %s: %s",
             request.method,
@@ -101,7 +100,7 @@ def _http_error(exception: HTTPException, request: pyramid.request.Request) -> A
 
 
 def _include_dev_details(request: pyramid.request.Request) -> bool:
-    return DEVELOPMENT or auth.is_auth(request)
+    return _DEVELOPMENT or auth.is_auth(request)
 
 
 def _integrity_error(
@@ -120,7 +119,7 @@ def _client_interrupted_error(
     exception: Exception, request: pyramid.request.Request
 ) -> pyramid.response.Response:
     # No need to cry wolf if it's just the client that interrupted the connection
-    return _do_error(request, 500, exception, logger=LOG.info)
+    return _do_error(request, 500, exception, logger=_LOG.info)
 
 
 def _boto_client_error(exception: Any, request: pyramid.request.Request) -> pyramid.response.Response:
@@ -131,7 +130,7 @@ def _boto_client_error(exception: Any, request: pyramid.request.Request) -> pyra
         status_code = exception.response["ResponseMetadata"]["HTTPStatusCode"]
     else:
         status_code = int(exception.response["Error"]["Code"])
-    log = STATUS_LOGGER.get(status_code, LOG.warning)
+    log = _STATUS_LOGGER.get(status_code, _LOG.warning)
     return _do_error(request, status_code, exception, logger=log)
 
 
@@ -142,7 +141,7 @@ def _other_error(exception: Exception, request: pyramid.request.Request) -> pyra
     status = 500
     if exception_class == "beaker.exceptions.BeakerException" and str(exception) == "Invalid signature":
         status = 401
-    LOG.debug("Actual exception: %s.%s", exception.__class__.__module__, exception.__class__.__name__)
+    _LOG.debug("Actual exception: %s.%s", exception.__class__.__module__, exception.__class__.__name__)
     return _do_error(request, status, exception)
 
 
@@ -178,4 +177,4 @@ def includeme(config: pyramid.config.Configurator) -> None:
             config.add_view(view=_client_interrupted_error, context=exception, **common_options)
 
         config.add_view(view=_other_error, context=Exception, **common_options)
-        LOG.info("Installed the error catching views")
+        _LOG.info("Installed the error catching views")