c2cwsgiutils 5.1.7.dev20230901073305__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl

c2cwsgiutils/db.py

@@ -2,8 +2,9 @@
 import logging
 import re
 import warnings
-from typing import Pattern  # noqa  # pylint: disable=unused-import
-from typing import Any, Callable, Iterable, Optional, Tuple, Union, cast
+from collections.abc import Iterable
+from re import Pattern
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
 
 import pyramid.config
 import pyramid.config.settings
@@ -13,9 +14,16 @@ import sqlalchemy.orm
 import transaction
 import zope.sqlalchemy
 from sqlalchemy import engine_from_config
-from sqlalchemy.orm import sessionmaker
 from zope.sqlalchemy import register
 
+if TYPE_CHECKING:
+    scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
+    sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session]
+else:
+    scoped_session = sqlalchemy.orm.scoped_session
+    sessionmaker = sqlalchemy.orm.sessionmaker
+
+
 LOG = logging.getLogger(__name__)
 RE_COMPILE: Callable[[str], Pattern[str]] = re.compile
 
@@ -35,8 +43,8 @@ def setup_session(
     slave_prefix: Optional[str] = None,
     force_master: Optional[Iterable[str]] = None,
     force_slave: Optional[Iterable[str]] = None,
-) -> Tuple[
-    Union[sqlalchemy.orm.Session, sqlalchemy.orm.scoped_session],
+) -> tuple[
+    Union[sqlalchemy.orm.Session, scoped_session],
     sqlalchemy.engine.Engine,
     sqlalchemy.engine.Engine,
 ]:
@@ -51,7 +59,7 @@ def setup_session(
     Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
     path includes the route_prefix.
 
-    Arguments:
+    Keyword Arguments:
 
         config: The pyramid Configuration object
         master_prefix: The prefix for the master connection configuration entries in the application \
@@ -68,7 +76,7 @@ def setup_session(
         slave_prefix = master_prefix
     settings = config.registry.settings
     rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".")
-    rw_engine.c2c_name = master_prefix
+    rw_engine.c2c_name = master_prefix  # type: ignore
     factory = sqlalchemy.orm.sessionmaker(bind=rw_engine)
     register(factory)
     db_session = sqlalchemy.orm.scoped_session(factory)
@@ -77,14 +85,14 @@ def setup_session(
     if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
         LOG.info("Using a slave DB for reading %s", master_prefix)
         ro_engine = sqlalchemy.engine_from_config(config.get_settings(), slave_prefix + ".")
-        ro_engine.c2c_name = slave_prefix
+        ro_engine.c2c_name = slave_prefix  # type: ignore
         tween_name = master_prefix.replace(".", "_")
         _add_tween(config, tween_name, db_session, force_master, force_slave)
     else:
         ro_engine = rw_engine
 
-    db_session.c2c_rw_bind = rw_engine
-    db_session.c2c_ro_bind = ro_engine
+    db_session.c2c_rw_bind = rw_engine  # type: ignore
+    db_session.c2c_ro_bind = ro_engine  # type: ignore
     return db_session, rw_engine, ro_engine
 
 
@@ -96,7 +104,7 @@ def create_session(
     force_master: Optional[Iterable[str]] = None,
     force_slave: Optional[Iterable[str]] = None,
     **engine_config: Any,
-) -> Union[sqlalchemy.orm.Session, sqlalchemy.orm.scoped_session]:
+) -> Union[sqlalchemy.orm.Session, scoped_session]:
     """
     Create a SQLAlchemy session.
 
@@ -108,7 +116,7 @@ def create_session(
     Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
     path includes the route_prefix.
 
-    Arguments:
+    Keyword Arguments:
 
         config: The pyramid Configuration object. If None, only master is used
         name: The name of the check
@@ -134,21 +142,21 @@ def create_session(
         LOG.info("Using a slave DB for reading %s", name)
         ro_engine = sqlalchemy.create_engine(slave_url, **engine_config)
         _add_tween(config, name, db_session, force_master, force_slave)
-        rw_engine.c2c_name = name + "_master"
-        ro_engine.c2c_name = name + "_slave"
+        rw_engine.c2c_name = name + "_master"  # type: ignore
+        ro_engine.c2c_name = name + "_slave"  # type: ignore
     else:
-        rw_engine.c2c_name = name
+        rw_engine.c2c_name = name  # type: ignore
         ro_engine = rw_engine
 
-    db_session.c2c_rw_bind = rw_engine
-    db_session.c2c_ro_bind = ro_engine
+    db_session.c2c_rw_bind = rw_engine  # type: ignore
+    db_session.c2c_ro_bind = ro_engine  # type: ignore
     return db_session
 
 
 def _add_tween(
     config: pyramid.config.Configurator,
     name: str,
-    db_session: Union[sqlalchemy.orm.Session, sqlalchemy.orm.scoped_session],
+    db_session: scoped_session,
     force_master: Optional[Iterable[str]],
     force_slave: Optional[Iterable[str]],
 ) -> None:
@@ -177,11 +185,19 @@ def _add_tween(
                 not has_force_master
                 and (request.method in ("GET", "OPTIONS") or any(r.match(method_path) for r in slave_paths))
             ):
-                LOG.debug("Using %s database for: %s", db_session.c2c_ro_bind.c2c_name, method_path)
-                session.bind = db_session.c2c_ro_bind
+                LOG.debug(
+                    "Using %s database for: %s",
+                    db_session.c2c_ro_bind.c2c_name,  # type: ignore
+                    method_path,
+                )
+                session.bind = db_session.c2c_ro_bind  # type: ignore
             else:
-                LOG.debug("Using %s database for: %s", db_session.c2c_rw_bind.c2c_name, method_path)
-                session.bind = db_session.c2c_rw_bind
+                LOG.debug(
+                    "Using %s database for: %s",
+                    db_session.c2c_rw_bind.c2c_name,  # type: ignore
+                    method_path,
+                )
+                session.bind = db_session.c2c_rw_bind  # type: ignore
 
             try:
                 return handler(request)
@@ -194,7 +210,7 @@ def _add_tween(
     config.add_tween("c2cwsgiutils.db.tweens." + name, over="pyramid_tm.tm_tween_factory")
 
 
-class SessionFactory(sessionmaker):  # type: ignore
+class SessionFactory(sessionmaker):
     """The custom session factory that manage the read only and read write sessions."""
 
     def __init__(
@@ -214,18 +230,18 @@ class SessionFactory(sessionmaker):  # type: ignore
 
     def engine_name(self, readwrite: bool) -> str:
         if readwrite:
-            return cast(str, self.rw_engine.c2c_name)
-        return cast(str, self.ro_engine.c2c_name)
+            return cast(str, self.rw_engine.c2c_name)  # type: ignore
+        return cast(str, self.ro_engine.c2c_name)  # type: ignore
 
-    def __call__(
+    def __call__(  # type: ignore
         self, request: Optional[pyramid.request.Request], readwrite: Optional[bool] = None, **local_kw: Any
-    ) -> sqlalchemy.orm.Session:
+    ) -> scoped_session:
         if readwrite is not None:
             if readwrite and not force_readonly:
-                LOG.debug("Using %s database", self.rw_engine.c2c_name)
+                LOG.debug("Using %s database", self.rw_engine.c2c_name)  # type: ignore
                 self.configure(bind=self.rw_engine)
             else:
-                LOG.debug("Using %s database", self.ro_engine.c2c_name)
+                LOG.debug("Using %s database", self.ro_engine.c2c_name)  # type: ignore
                 self.configure(bind=self.ro_engine)
         else:
             assert request is not None
@@ -238,12 +254,12 @@ class SessionFactory(sessionmaker):  # type: ignore
                     or any(r.match(method_path) for r in self.slave_paths)
                 )
             ):
-                LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path)
+                LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path)  # type: ignore
                 self.configure(bind=self.ro_engine)
             else:
-                LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path)
+                LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path)  # type: ignore
                 self.configure(bind=self.rw_engine)
-        return super().__call__(**local_kw)
+        return super().__call__(**local_kw)  # type: ignore
 
 
 def get_engine(
@@ -253,7 +269,9 @@ def get_engine(
     return engine_from_config(settings, prefix)
 
 
-def get_session_factory(engine: sqlalchemy.engine.Engine) -> sessionmaker:
+def get_session_factory(
+    engine: sqlalchemy.engine.Engine,
+) -> sessionmaker:
     """Get the session factory from the engine."""
     factory = sessionmaker()
     factory.configure(bind=engine)
@@ -320,6 +338,7 @@ def get_tm_session(
           request = dbsession.info["request"]
     """
     dbsession = session_factory()
+    assert isinstance(dbsession, sqlalchemy.orm.Session), type(dbsession)
     zope.sqlalchemy.register(dbsession, transaction_manager=transaction_manager)
     return dbsession
 
@@ -328,7 +347,7 @@ def get_tm_session_pyramid(
     session_factory: SessionFactory,
     transaction_manager: transaction.TransactionManager,
     request: pyramid.request.Request,
-) -> sqlalchemy.orm.Session:
+) -> scoped_session:
     """
     Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.
 
@@ -349,7 +368,7 @@ def init(
     """
     Initialize the database for a Pyramid app.
 
-    Arguments:
+    Keyword Arguments:
 
         config: The pyramid Configuration object
         master_prefix: The prefix for the master connection configuration entries in the application \
@@ -368,13 +387,13 @@ def init(
     dbengine = settings.get("dbengine")
     if not dbengine:
         rw_engine = get_engine(settings, master_prefix + ".")
-        rw_engine.c2c_name = master_prefix
+        rw_engine.c2c_name = master_prefix  # type: ignore
 
         # Setup a slave DB connection and add a tween to use it.
         if slave_prefix and settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
             LOG.info("Using a slave DB for reading %s", master_prefix)
             ro_engine = get_engine(config.get_settings(), slave_prefix + ".")
-            ro_engine.c2c_name = slave_prefix
+            ro_engine.c2c_name = slave_prefix  # type: ignore
         else:
             ro_engine = rw_engine
     else:
@@ -390,6 +409,8 @@ def init(
         if dbsession is None:
             # request.tm is the transaction manager used by pyramid_tm
             dbsession = get_tm_session_pyramid(session_factory, request.tm, request=request)
+            assert dbsession is not None
+        assert isinstance(dbsession, sqlalchemy.orm.Session), type(dbsession)
         return dbsession
 
     config.add_request_method(dbsession, reify=True)

c2cwsgiutils/db_maintenance_view.py

@@ -1,6 +1,7 @@
 import logging
 import warnings
-from typing import Any, Mapping, Optional, cast
+from collections.abc import Mapping
+from typing import Any, Optional, cast
 
 import pyramid.request
 

c2cwsgiutils/debug/_listeners.py

@@ -3,7 +3,8 @@ import sys
 import threading
 import time
 import traceback
-from typing import Any, Dict, List, Mapping, Optional, Tuple, cast
+from collections.abc import Mapping
+from typing import Any, Optional, cast
 
 import objgraph
 
@@ -13,7 +14,7 @@ from c2cwsgiutils.debug.utils import get_size
 FILES_FIELDS = {"__name__", "__doc__", "__package__", "__loader__", "__spec__", "__file__"}
 
 
-def _dump_stacks_impl() -> Dict[str, Any]:
+def _dump_stacks_impl() -> dict[str, Any]:
     id2name = {th.ident: th.name for th in threading.enumerate()}
     threads = {}
     for thread_id, stack in sys._current_frames().items():  # pylint: disable=W0212
@@ -47,10 +48,10 @@ def _dump_memory_impl(
 
     if analyze_type:
         # timeout after one minute, must be set to a bit less that the timeout of the broadcast in _views.py
-        timeout = time.monotonic() + 60
+        timeout = time.perf_counter() + 60
 
-        mod_counts: Dict[str, int] = {}
-        biggest_objects: List[Tuple[float, Any]] = []
+        mod_counts: dict[str, int] = {}
+        biggest_objects: list[tuple[float, Any]] = []
         result[analyze_type] = {}
         for obj in objgraph.by_type(analyze_type):
             if analyze_type == "builtins.function":
@@ -80,12 +81,12 @@ def _dump_memory_impl(
                     biggest_objects.sort(key=lambda x: x[0])
                     if len(biggest_objects) > limit:
                         biggest_objects = biggest_objects[-limit:]
-            if time.monotonic() > timeout:
+            if time.perf_counter() > timeout:
                 result[analyze_type]["timeout"] = True
                 break
         if analyze_type == "builtins.function":
             result[analyze_type]["modules"] = [
-                dict(module=i[0], nb_func=i[1])
+                {"module": i[0], "nb_func": i[1]}
                 for i in sorted(mod_counts.items(), key=lambda x: -x[1])[:limit]
             ]
         elif analyze_type == "linecache":
@@ -93,13 +94,13 @@ def _dump_memory_impl(
 
             cache = linecache.cache
             result[analyze_type]["biggest_objects"] = sorted(
-                (dict(filename=k, size_kb=get_size(v)) for k, v in cache.items()),
+                ({"filename": k, "size_kb": get_size(v)} for k, v in cache.items()),
                 key=lambda i: -(cast(int, i["size_kb"])),
             )
         else:
             biggest_objects.reverse()
             result[analyze_type]["biggest_objects"] = [
-                dict(size_kb=i[0], repr=repr(i[1])) for i in biggest_objects
+                {"size_kb": i[0], "repr": repr(i[1])} for i in biggest_objects
             ]
     return result
 

c2cwsgiutils/debug/_views.py

@@ -2,9 +2,10 @@ import gc
 import logging
 import re
 import time
+from collections.abc import Mapping
 from datetime import datetime
 from io import StringIO
-from typing import Any, Callable, Dict, List, Mapping, cast
+from typing import Any, Callable, cast
 
 import objgraph
 import pyramid.config
@@ -19,9 +20,9 @@ LOG = logging.getLogger(__name__)
 SPACE_RE = re.compile(r" +")
 
 
-def _beautify_stacks(source: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]:
+def _beautify_stacks(source: list[Mapping[str, Any]]) -> list[Mapping[str, Any]]:
     """Group the identical stacks together along with a list of threads sporting them."""
-    results: List[Mapping[str, Any]] = []
+    results: list[Mapping[str, Any]] = []
     for host_stacks in source:
         host_id = f"{host_stacks['hostname']}/{host_stacks['pid']:d}"
         for thread, frames in host_stacks["threads"].items():
@@ -35,14 +36,14 @@ def _beautify_stacks(source: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]
     return results
 
 
-def _dump_stacks(request: pyramid.request.Request) -> List[Mapping[str, Any]]:
+def _dump_stacks(request: pyramid.request.Request) -> list[Mapping[str, Any]]:
     auth.auth_view(request)
     result = broadcast.broadcast("c2c_dump_stacks", expect_answers=True)
     assert result is not None
     return _beautify_stacks(result)
 
 
-def _dump_memory(request: pyramid.request.Request) -> List[Mapping[str, Any]]:
+def _dump_memory(request: pyramid.request.Request) -> list[Mapping[str, Any]]:
     auth.auth_view(request)
     limit = int(request.params.get("limit", "30"))
     analyze_type = request.params.get("analyze_type")
@@ -57,7 +58,7 @@ def _dump_memory(request: pyramid.request.Request) -> List[Mapping[str, Any]]:
     return result
 
 
-def _dump_memory_diff(request: pyramid.request.Request) -> List[Any]:
+def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
     auth.auth_view(request)
     limit = int(request.params.get("limit", "30"))
     if "path" in request.matchdict:
@@ -72,7 +73,7 @@ def _dump_memory_diff(request: pyramid.request.Request) -> List[Any]:
     if len(split_path) > 1:
         sub_request.query_string = split_path[1]
 
-    # Warm-up run
+    # warm-up run
     try:
         if "no_warmup" not in request.params:
             request.invoke_subrequest(sub_request)
@@ -81,7 +82,7 @@ def _dump_memory_diff(request: pyramid.request.Request) -> List[Any]:
 
     LOG.debug("checking memory growth for %s", path)
 
-    peak_stats: Dict[Any, Any] = {}
+    peak_stats: dict[Any, Any] = {}
     for i in range(3):
         gc.collect(i)
 
@@ -158,7 +159,7 @@ def _add_view(
     config.add_view(view, route_name="c2c_debug_" + name, renderer="fast_json", http_cache=0)
 
 
-def _dump_memory_maps(request: pyramid.request.Request) -> List[Dict[str, Any]]:
+def _dump_memory_maps(request: pyramid.request.Request) -> list[dict[str, Any]]:
     auth.auth_view(request)
     return sorted(dump_memory_maps(), key=lambda i: cast(int, -i.get("pss_kb", 0)))
 
@@ -168,13 +169,13 @@ def _show_refs(request: pyramid.request.Request) -> pyramid.response.Response:
     for generation in range(3):
         gc.collect(generation)
 
-    objs: List[Any] = []
+    objs: list[Any] = []
     if "analyze_type" in request.params:
         objs = objgraph.by_type(request.params["analyze_type"])
     elif "analyze_id" in request.params:
         objs = [objgraph.by(int(request.params["analyze_id"]))]
 
-    args: Dict[str, Any] = {
+    args: dict[str, Any] = {
         "refcounts": True,
     }
     if request.params.get("max_depth", "") != "":

c2cwsgiutils/debug/utils.py

@@ -5,7 +5,7 @@ import re
 import sys
 from collections import defaultdict
 from types import FunctionType, ModuleType
-from typing import Any, Dict, List, Set
+from typing import Any
 
 # 7ff7d33bd000-7ff7d33be000 r--p 00000000 00:65 49                         /usr/lib/toto.so
 SMAPS_LOCATION_RE = re.compile(r"^[0-9a-f]+-[0-9a-f]+ +.... +[0-9a-f]+ +[^ ]+ +\d+ +(.*)$")
@@ -21,7 +21,7 @@ def get_size(obj: Any) -> int:
     """Get the sum size of object & members."""
     if isinstance(obj, BLACKLIST):
         return 0
-    seen_ids: Set[int] = set()
+    seen_ids: set[int] = set()
     size = 0
     objects = [obj]
     while objects:
@@ -35,14 +35,14 @@ def get_size(obj: Any) -> int:
     return size
 
 
-def dump_memory_maps(pid: str = "self") -> List[Dict[str, Any]]:
+def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
     """Get the Linux memory maps."""
     filename = os.path.join("/proc", pid, "smaps")
     if not os.path.exists(filename):
         return []
     with open(filename, encoding="utf-8") as input_:
-        cur_dict: Dict[str, int] = defaultdict(int)
-        sizes: Dict[str, Any] = {}
+        cur_dict: dict[str, int] = defaultdict(int)
+        sizes: dict[str, Any] = {}
         for line in input_:
             line = line.rstrip("\n")
             matcher = SMAPS_LOCATION_RE.match(line)

c2cwsgiutils/errors.py

@@ -60,7 +60,7 @@ def _do_error(
         request.method,
         request.url,
         status,
-        extra={"referer": request.referer},
+        extra={"referrer": request.referrer},
         exc_info=exception,
     )
 
@@ -88,7 +88,7 @@ def _http_error(exception: HTTPException, request: pyramid.request.Request) -> A
             request.url,
             exception.status_code,
             str(exception),
-            extra={"referer": request.referer},
+            extra={"referrer": request.referrer},
         )
         request.response.headers.update(exception.headers)  # forward headers
         _add_cors(request)
@@ -107,10 +107,11 @@ def _include_dev_details(request: pyramid.request.Request) -> bool:
 def _integrity_error(
     exception: sqlalchemy.exc.StatementError, request: pyramid.request.Request
 ) -> pyramid.response.Response:
-    def reduce_info_sent(e: sqlalchemy.exc.StatementError) -> None:
-        # remove details (SQL statement and links to SQLAlchemy) from the error
-        e.statement = None
-        e.code = None
+    def reduce_info_sent(e: Exception) -> None:
+        if isinstance(e, sqlalchemy.exc.StatementError):
+            # remove details (SQL statement and links to SQLAlchemy) from the error
+            e.statement = None
+            e.code = None
 
     return _do_error(request, 400, exception, reduce_info_sent=reduce_info_sent)