c2cwsgiutils 6.1.7.dev4__py3-none-any.whl → 6.2.0.dev54__py3-none-any.whl

c2cwsgiutils/auth.py

@@ -12,32 +12,32 @@ from requests_oauthlib import OAuth2Session
 from c2cwsgiutils.config_utils import config_bool, env_or_config, env_or_settings
 
 _COOKIE_AGE = 7 * 24 * 3600
-SECRET_PROP = "c2c.secret"  # nosec  # noqa
-SECRET_ENV = "C2C_SECRET"  # nosec  # noqa
+SECRET_PROP = "c2c.secret"  # noqa: S105
+SECRET_ENV = "C2C_SECRET"  # noqa: S105
 _GITHUB_REPOSITORY_PROP = "c2c.auth.github.repository"
 _GITHUB_REPOSITORY_ENV = "C2C_AUTH_GITHUB_REPOSITORY"
 _GITHUB_ACCESS_TYPE_PROP = "c2c.auth.github.access_type"
 _GITHUB_ACCESS_TYPE_ENV = "C2C_AUTH_GITHUB_ACCESS_TYPE"
 GITHUB_AUTH_URL_PROP = "c2c.auth.github.auth_url"
 GITHUB_AUTH_URL_ENV = "C2C_AUTH_GITHUB_AUTH_URL"
-GITHUB_TOKEN_URL_PROP = "c2c.auth.github.token_url"  # nosec
-GITHUB_TOKEN_URL_ENV = "C2C_AUTH_GITHUB_TOKEN_URL"  # nosec
+GITHUB_TOKEN_URL_PROP = "c2c.auth.github.token_url"  # noqa: S105
+GITHUB_TOKEN_URL_ENV = "C2C_AUTH_GITHUB_TOKEN_URL"  # noqa: S105
 GITHUB_USER_URL_PROP = "c2c.auth.github.user_url"
 GITHUB_USER_URL_ENV = "C2C_AUTH_GITHUB_USER_URL"
 _GITHUB_REPO_URL_PROP = "c2c.auth.github.repo_url"
 _GITHUB_REPO_URL_ENV = "C2C_AUTH_GITHUB_REPO_URL"
 GITHUB_CLIENT_ID_PROP = "c2c.auth.github.client_id"
 GITHUB_CLIENT_ID_ENV = "C2C_AUTH_GITHUB_CLIENT_ID"
-GITHUB_CLIENT_SECRET_PROP = "c2c.auth.github.client_secret"  # nosec # noqa
-GITHUB_CLIENT_SECRET_ENV = "C2C_AUTH_GITHUB_CLIENT_SECRET"  # nosec # noqa
+GITHUB_CLIENT_SECRET_PROP = "c2c.auth.github.client_secret"  # noqa: S105
+GITHUB_CLIENT_SECRET_ENV = "C2C_AUTH_GITHUB_CLIENT_SECRET"  # noqa: S105
 GITHUB_SCOPE_PROP = "c2c.auth.github.scope"
 GITHUB_SCOPE_ENV = "C2C_AUTH_GITHUB_SCOPE"
 # To be able to use private repository
 GITHUB_SCOPE_DEFAULT = "repo"
 GITHUB_AUTH_COOKIE_PROP = "c2c.auth.github.auth.cookie"
 GITHUB_AUTH_COOKIE_ENV = "C2C_AUTH_GITHUB_COOKIE"
-GITHUB_AUTH_SECRET_PROP = "c2c.auth.github.auth.secret"  # nosec # noqa
-GITHUB_AUTH_SECRET_ENV = "C2C_AUTH_GITHUB_SECRET"  # nosec # noqa
+GITHUB_AUTH_SECRET_PROP = "c2c.auth.github.auth.secret"  # noqa: S105
+GITHUB_AUTH_SECRET_ENV = "C2C_AUTH_GITHUB_SECRET"  # noqa: S105
 GITHUB_AUTH_PROXY_URL_PROP = "c2c.auth.github.auth.proxy_url"
 GITHUB_AUTH_PROXY_URL_ENV = "C2C_AUTH_GITHUB_PROXY_URL"
 USE_SESSION_PROP = "c2c.use_session"
@@ -209,6 +209,7 @@ def check_access(
         request: is the request object.
         repo: is the repository to check access to (<organization>/<repository>).
         access_type: is the type of access to check (admin|push|pull).
+
     """
     if not is_auth(request):
         return False

c2cwsgiutils/broadcast/__init__.py

@@ -19,7 +19,7 @@ _broadcaster: Optional[interface.BaseBroadcaster] = None
 
 def init(config: Optional[pyramid.config.Configurator] = None) -> None:
     """Initialize the broadcaster with Redis, if configured, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/broadcast/local.py

@@ -9,17 +9,21 @@ class LocalBroadcaster(interface.BaseBroadcaster):
     """Fake implementation of broadcasting messages (will just answer locally)."""
 
     def __init__(self) -> None:
+        """Initialize the broadcaster."""
         self._subscribers: MutableMapping[str, Callable[..., Any]] = {}
 
     def subscribe(self, channel: str, callback: Callable[..., Any]) -> None:
+        """Subscribe to a channel."""
         self._subscribers[channel] = callback
 
     def unsubscribe(self, channel: str) -> None:
+        """Unsubscribe from a channel."""
         del self._subscribers[channel]
 
     def broadcast(
         self, channel: str, params: Mapping[str, Any], expect_answers: bool, timeout: float
     ) -> Optional[list[Any]]:
+        """Broadcast a message to all the listeners."""
         subscriber = self._subscribers.get(channel, None)
         answers = [utils.add_host_info(subscriber(**params))] if subscriber is not None else []
         return answers if expect_answers else None

c2cwsgiutils/broadcast/redis.py

@@ -23,6 +23,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
         master: "redis.client.Redis[str]",
         slave: "redis.client.Redis[str]",
     ) -> None:
+        """Initialize the broadcaster."""
         from c2cwsgiutils import redis_utils  # pylint: disable=import-outside-toplevel
 
         self._master = master
@@ -40,6 +41,8 @@ class RedisBroadcaster(interface.BaseBroadcaster):
         return self._broadcast_prefix + channel
 
     def subscribe(self, channel: str, callback: Callable[..., Any]) -> None:
+        """Subscribe to a channel."""
+
         def wrapper(message: Mapping[str, Any]) -> None:
             _LOG.debug("Received a broadcast on %s: %s", message["channel"], repr(message["data"]))
             data = json.loads(message["data"])
@@ -58,6 +61,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
         self._pub_sub.subscribe(**{actual_channel: wrapper})
 
     def unsubscribe(self, channel: str) -> None:
+        """Unsubscribe from a channel."""
         _LOG.debug("Unsubscribing from %s")
         actual_channel = self._get_channel(channel)
         self._pub_sub.unsubscribe(actual_channel)
@@ -65,6 +69,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
     def broadcast(
         self, channel: str, params: Mapping[str, Any], expect_answers: bool, timeout: float
     ) -> Optional[list[Any]]:
+        """Broadcast a message to all the listeners."""
         if expect_answers:
             return self._broadcast_with_answer(channel, params, timeout)
         else:
@@ -85,7 +90,8 @@ class RedisBroadcaster(interface.BaseBroadcaster):
                 cond.notify()
 
         answer_channel = self._get_channel(channel) + "".join(
-            random.choice(string.ascii_uppercase + string.digits) for _ in range(10)  # nosec
+            random.choice(string.ascii_uppercase + string.digits)  # noqa: S311
+            for _ in range(10)
         )
         _LOG.debug("Subscribing for broadcast answers on %s", answer_channel)
         self._pub_sub.subscribe(**{answer_channel: callback})
@@ -98,7 +104,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
             with cond:
                 while len(answers) < nb_received:
                     to_wait = timeout_time - time.perf_counter()
-                    if to_wait <= 0.0:  # pragma: no cover
+                    if to_wait <= 0.0:
                         _LOG.warning(
                             "timeout waiting for %d/%d answers on %s",
                             len(answers),

c2cwsgiutils/client_info.py

@@ -16,9 +16,11 @@ class Filter:
     """
 
     def __init__(self, application: Callable[[dict[str, str], Any], Any]):
+        """Initialize the filter."""
         self._application = application
 
     def __call__(self, environ: dict[str, str], start_response: Any) -> Any:
+        """Update the environ with the headers."""
         # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
         if "HTTP_FORWARDED" in environ:
             _handle_forwarded(environ)

c2cwsgiutils/coverage_setup.py

@@ -10,7 +10,7 @@ _LOG = logging.getLogger(__name__)
 
 def init() -> None:
     """Initialize the code coverage, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme()
 
 
@@ -22,7 +22,7 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
     import coverage  # pylint: disable=import-outside-toplevel
 
     _LOG.warning("Setting up code coverage")
-    report_dir = "/tmp/coverage/api"  # nosec
+    report_dir = "/tmp/coverage/api"  # noqa: S108
     os.makedirs(report_dir, exist_ok=True)
     cov = coverage.Coverage(
         data_file=os.path.join(report_dir, "coverage"),

c2cwsgiutils/db.py

@@ -66,8 +66,11 @@ def setup_session(
         force_slave: The method/paths that needs to use the slave
 
     Returns: The SQLAlchemy session, the R/W engine and the R/O engine
+
     """
-    warnings.warn("setup_session function is deprecated; use init and request.dbsession instead")
+    warnings.warn(
+        "setup_session function is deprecated; use init and request.dbsession instead", stacklevel=2
+    )
     if slave_prefix is None:
         slave_prefix = master_prefix
     settings = config.registry.settings
@@ -122,8 +125,11 @@ def create_session(
         engine_config: The rest of the parameters are passed as is to the sqlalchemy.create_engine function
 
     Returns: The SQLAlchemy session
+
     """
-    warnings.warn("create_session function is deprecated; use init and request.dbsession instead")
+    warnings.warn(
+        "create_session function is deprecated; use init and request.dbsession instead", stacklevel=2
+    )
     if slave_url is None:
         slave_url = url
 
@@ -215,6 +221,7 @@ class SessionFactory(_sessionmaker):
         ro_engine: sqlalchemy.engine.Engine,
         rw_engine: sqlalchemy.engine.Engine,
     ):
+        """Initialize the session factory."""
         super().__init__()
         self.master_paths: Iterable[Pattern[str]] = (
             list(map(_RE_COMPILE, force_master)) if force_master else []
@@ -232,6 +239,7 @@ class SessionFactory(_sessionmaker):
     def __call__(  # type: ignore
         self, request: Optional[pyramid.request.Request], readwrite: Optional[bool] = None, **local_kw: Any
     ) -> _scoped_session:
+        """Set the engine based on the request."""
         if readwrite is not None:
             if readwrite and not FORCE_READONLY:
                 _LOG.debug("Using %s database", self.rw_engine.c2c_name)  # type: ignore
@@ -374,6 +382,7 @@ def init(
         force_slave: The method/paths that needs to use the slave
 
     Returns: The SQLAlchemy session
+
     """
     settings = config.get_settings()
     settings["tm.manager_hook"] = "pyramid_tm.explicit_manager"

c2cwsgiutils/db_maintenance_view.py

@@ -15,7 +15,7 @@ _REDIS_PREFIX = "c2c_db_maintenance_"
 
 def install_subscriber(config: pyramid.config.Configurator) -> None:
     """Install the view to configure the loggers, if configured to do so, for backward compatibility."""
-    warnings.warn("install_subscriber function is deprecated; use includeme instead")
+    warnings.warn("install_subscriber function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/debug/__init__.py

@@ -16,7 +16,7 @@ dump_memory_maps = utils.dump_memory_maps
 
 def init(config: pyramid.config.Configurator) -> None:
     """Initialize the debug tools, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 
@@ -37,6 +37,8 @@ def init_daemon(config: Optional[pyramid.config.Configurator] = None) -> None:
     those requests.
     """
     if config_utils.env_or_config(config, ENV_KEY, CONFIG_KEY, type_=config_utils.config_bool):
-        from c2cwsgiutils.debug import _listeners  # pylint: disable=import-outside-toplevel
+        from c2cwsgiutils.debug import (  # pylint: disable=import-outside-toplevel
+            _listeners,
+        )
 
         _listeners.init()

c2cwsgiutils/debug/_views.py

@@ -1,5 +1,6 @@
 import gc
 import logging
+import os
 import re
 import time
 from collections.abc import Mapping
@@ -8,6 +9,7 @@ from io import StringIO
 from typing import Any, Callable, cast
 
 import objgraph
+import psutil
 import pyramid.config
 import pyramid.request
 import pyramid.response
@@ -77,7 +79,7 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
     try:
         if request.params.get("no_warmup", "0").lower() in ("1", "true", "on"):
             request.invoke_subrequest(sub_request)
-    except Exception:  # nosec  # pylint: disable=broad-except
+    except Exception:  # pylint: disable=broad-except
         pass
 
     _LOG.debug("checking memory growth for %s", path)
@@ -87,6 +89,9 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
         gc.collect(i)
 
     objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False)
+    process = psutil.Process(os.getpid())
+    mem_before = process.memory_info()
+    start_time = time.time()
 
     response = None
     try:
@@ -96,12 +101,26 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
     except HTTPException as ex:
         _LOG.debug("response was %s", str(ex))
 
+    elapsed_time = time.time() - start_time
     del response
 
     for i in range(3):
         gc.collect(i)
 
-    return objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False)  # type: ignore
+    mem_after = process.memory_info()
+    return {
+        "memory.growth": {
+            "rss_kb": (mem_after.rss - mem_before.rss) / 1024,
+            "vms_kb": (mem_after.vms - mem_before.vms) / 1024,
+            "shared_kb": (mem_after.shared - mem_before.shared) / 1024,
+            "text_kb": (mem_after.text - mem_before.text) / 1024,
+            "lib_kb": (mem_after.lib - mem_before.lib) / 1024,
+            "data_kb": (mem_after.data - mem_before.data) / 1024,
+            "dirty_kb": (mem_after.dirty - mem_before.dirty) / 1024,
+        },
+        "elapsed_time": elapsed_time,
+        "objgraph.growth": objgraph.growth(limit=limit, peak_stats=peak_stats, shortnames=False),  # type: ignore
+    }
 
 
 def _sleep(request: pyramid.request.Request) -> pyramid.response.Response:
@@ -132,8 +151,7 @@ def _headers(request: pyramid.request.Request) -> Mapping[str, Any]:
     }
     if "status" in request.params:
         raise exception_response(int(request.params["status"]), detail=result)
-    else:
-        return result
+    return result
 
 
 def _error(request: pyramid.request.Request) -> Any:

c2cwsgiutils/errors.py

@@ -9,7 +9,12 @@ from typing import Any, Callable
 import pyramid.request
 import sqlalchemy.exc
 from cornice import cors
-from pyramid.httpexceptions import HTTPError, HTTPException, HTTPRedirection, HTTPSuccessful
+from pyramid.httpexceptions import (
+    HTTPError,
+    HTTPException,
+    HTTPRedirection,
+    HTTPSuccessful,
+)
 from webob.request import DisconnectionError
 
 from c2cwsgiutils import auth, config_utils
@@ -152,7 +157,7 @@ def _passthrough(exception: HTTPException, request: pyramid.request.Request) ->
 
 def init(config: pyramid.config.Configurator) -> None:
     """Initialize the error views, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/health_check.py

@@ -67,6 +67,7 @@ class JsonCheckException(Exception):
     """Checker exception used to add some structured content to a failure."""
 
     def __init__(self, message: str, json: Any):
+        """Initialize the exception."""
         super().__init__()
         self.message = message
         self.json = json
@@ -199,6 +200,7 @@ class HealthCheck:
     """
 
     def __init__(self, config: pyramid.config.Configurator) -> None:
+        """Initialize the health check view."""
         config.add_route(
             "c2c_health_check", config_utils.get_base_path(config) + r"/health_check", request_method="GET"
         )
@@ -235,6 +237,7 @@ class HealthCheck:
             engine_type: whether to check only the RW, RO or both engines
             rw_engin: the RW engine to use (if None, use the session one)
             ro_engin: the RO engine to use (if None, use the session one)
+
         """
         if query_cb is None:
             query_cb = self._at_least_one(at_least_one_model)
@@ -265,6 +268,7 @@ class HealthCheck:
             version_table: override the table name for the version
             rw_engin: the RW engine to use (if None, use the session one)
             ro_engin: the RO engine to use (if None, use the session one)
+
         """
         version_ = _get_alembic_version(alembic_ini_path, name)
 
@@ -287,26 +291,28 @@ class HealthCheck:
                 assert version_schema
                 assert version_table
                 for binding in _get_bindings(self.session, EngineType.READ_AND_WRITE):
-                    with binding as binded_session:
-                        with _PROMETHEUS_DB_SUMMARY.labels(
+                    with (
+                        binding as binded_session,
+                        _PROMETHEUS_DB_SUMMARY.labels(
                             configuration=alembic_ini_path, connection=binding.name(), check="alembic"
-                        ).time():
-                            result = binded_session.execute(
-                                sqlalchemy.text(
-                                    "SELECT version_num FROM "  # nosec
-                                    f"{sqlalchemy.sql.quoted_name(version_schema, True)}."
-                                    f"{sqlalchemy.sql.quoted_name(version_table, True)}"
-                                )
-                            ).fetchone()
-                            assert result is not None
-                            (actual_version,) = result
-                            _PROMETHEUS_ALEMBIC_VERSION.labels(
-                                version=actual_version, name=name, configuration=alembic_ini_path
-                            ).set(1)
-                            if actual_version != version_:
-                                raise Exception(  # pylint: disable=broad-exception-raised
-                                    f"Invalid alembic version (db: {actual_version}, code: {version_})"
-                                )
+                        ).time(),
+                    ):
+                        result = binded_session.execute(
+                            sqlalchemy.text(
+                                "SELECT version_num FROM "  # noqa: S608
+                                f"{sqlalchemy.sql.quoted_name(version_schema, True)}."
+                                f"{sqlalchemy.sql.quoted_name(version_table, True)}"
+                            )
+                        ).fetchone()
+                        assert result is not None
+                        (actual_version,) = result
+                        _PROMETHEUS_ALEMBIC_VERSION.labels(
+                            version=actual_version, name=name, configuration=alembic_ini_path
+                        ).set(1)
+                        if actual_version != version_:
+                            raise Exception(  # pylint: disable=broad-exception-raised
+                                f"Invalid alembic version (db: {actual_version}, code: {version_})"
+                            )
                 return version_
 
         self._checks.append(
@@ -321,9 +327,8 @@ class HealthCheck:
             Mapping[str, str], Callable[[pyramid.request.Request], Mapping[str, str]], None
         ] = None,
         name: Optional[str] = None,
-        check_cb: Callable[
-            [pyramid.request.Request, requests.Response], Any
-        ] = lambda request, response: None,
+        check_cb: Callable[[pyramid.request.Request, requests.Response], Any] = lambda request,
+        response: None,
         timeout: float = 3,
         level: int = 1,
     ) -> None:
@@ -339,6 +344,7 @@ class HealthCheck:
                          response as parameters)
             timeout: the timeout
             level: the level of the health check
+
         """
 
         def check(request: pyramid.request.Request) -> Any:
@@ -365,6 +371,7 @@ class HealthCheck:
         Arguments:
             name: the name of the check (defaults to url)
             level: the level of the health check
+
         """
 
         def check(request: pyramid.request.Request) -> Any:
@@ -413,6 +420,7 @@ class HealthCheck:
         Arguments:
             name: the name of the check (defaults to "version")
             level: the level of the health check
+
         """
 
         def check(request: pyramid.request.Request) -> dict[str, Any]:
@@ -441,6 +449,7 @@ class HealthCheck:
             name: the name of the check
             check_cb: the callback to call (takes the request as parameter)
             level: the level of the health check
+
         """
         assert name
         self._checks.append((name, check_cb, level))
@@ -453,9 +462,8 @@ class HealthCheck:
             "successes": {},
         }
         checks = None
-        if "checks" in request.params:
-            if request.params["checks"] != "":
-                checks = request.params["checks"].split(",")
+        if "checks" in request.params and request.params["checks"] != "":
+            checks = request.params["checks"].split(",")
         for name, check, level in self._checks:
             if level <= max_level and (checks is None or name in checks):
                 self._run_one(check, is_auth, level, name, request, results)
@@ -498,11 +506,13 @@ class HealthCheck:
     ) -> tuple[str, Callable[[pyramid.request.Request], None]]:
         def check(request: pyramid.request.Request) -> None:
             del request  # unused
-            with binding as session:
-                with _PROMETHEUS_DB_SUMMARY.labels(
+            with (
+                binding as session,
+                _PROMETHEUS_DB_SUMMARY.labels(
                     connection=binding.name(), check="database", configuration="<default>"
-                ).time():
-                    return query_cb(session)
+                ).time(),
+            ):
+                return query_cb(session)
 
         return "db_engine_" + binding.name(), check
 

c2cwsgiutils/index.py

@@ -501,7 +501,7 @@ def _github_logout(request: pyramid.request.Request) -> dict[str, Any]:
 
 def init(config: pyramid.config.Configurator) -> None:
     """Initialize the index page, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/loader.py

@@ -1,4 +1,4 @@
-import logging
+import logging.config
 from typing import Optional, cast
 
 from plaster_pastedeploy import Loader as BaseLoader

c2cwsgiutils/logging_view.py

@@ -15,7 +15,7 @@ _REDIS_PREFIX = "c2c_logging_level_"
 
 def install_subscriber(config: pyramid.config.Configurator) -> None:
     """Install the view to configure the loggers, if configured to do so, for backward compatibility."""
-    warnings.warn("install_subscriber function is deprecated; use includeme instead")
+    warnings.warn("install_subscriber function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/models_graph.py

@@ -44,7 +44,7 @@ def _generate_model_graph(module: Any, base: Any) -> None:
 def _print_node(symbol: Any, interesting: set[Any]) -> None:
     print(f'{symbol.__name__} [label="{_get_table_desc(symbol)}", shape=box];')
     for parent in symbol.__bases__:
-        if parent != object:
+        if parent is not object:
             if parent not in interesting:
                 _print_node(parent, interesting)
                 interesting.add(parent)

c2cwsgiutils/pretty_json.py

@@ -28,7 +28,7 @@ class _FastDumps:
 
 def init(config: pyramid.config.Configurator) -> None:
     """Initialize json and fast_json renderer, for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
+    warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
     includeme(config)
 
 

c2cwsgiutils/profiler.py

@@ -9,6 +9,7 @@ class Profile(contextlib.ContextDecorator):
     """Used to profile a function with a decorator or with a with statement."""
 
     def __init__(self, path: str, print_number: int = 0) -> None:
+        """Initialize the profiler."""
         self.path = path
         self.print_number = print_number
         self.pr = cProfile.Profile()

c2cwsgiutils/prometheus.py

@@ -2,6 +2,7 @@
 
 import os
 import re
+import resource
 from collections.abc import Generator, Iterable
 from typing import Any, Optional, TypedDict, cast
 
@@ -15,6 +16,12 @@ import pyramid.config
 from c2cwsgiutils import broadcast, redis_utils
 from c2cwsgiutils.debug.utils import dump_memory_maps
 
+PSUTILS = True
+try:
+    import psutil
+except ImportError:
+    PSUTILS = False
+
 _NUMBER_RE = re.compile(r"^[0-9]+$")
 MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
     "c2cwsgiutils_prometheus_collector_gc",
@@ -22,12 +29,21 @@ MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
 ]
 
 
+def start_single_process() -> None:
+    """Start separate HTTP server to provide the Prometheus metrics."""
+    if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
+        prometheus_client.REGISTRY.register(ResourceCollector())
+        prometheus_client.REGISTRY.register(MemoryInfoCollector())
+        prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"]))
+
+
 def start(registry: Optional[prometheus_client.CollectorRegistry] = None) -> None:
     """Start separate HTTP server to provide the Prometheus metrics."""
     if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
         broadcast.includeme()
 
         registry = prometheus_client.CollectorRegistry() if registry is None else registry
+        registry.register(ResourceCollector())
         registry.register(MemoryMapCollector())
         registry.register(prometheus_client.PLATFORM_COLLECTOR)
         registry.register(MultiProcessCustomCollector())
@@ -115,6 +131,7 @@ class MultiProcessCustomCollector(prometheus_client.registry.Collector):
     """Get the metrics from the custom collectors."""
 
     def collect(self) -> Generator[prometheus_client.core.Metric, None, None]:
+        """Get the metrics from the custom collectors."""
         results: list[list[SerializedMetric]] = []
         for channel in MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS:
             result = broadcast.broadcast(channel, expect_answers=True)
@@ -159,6 +176,7 @@ class MemoryMapCollector(prometheus_client.registry.Collector):
         Arguments:
             memory_type: can be rss, pss or size
             pids: the list of pids or none
+
         """
         super().__init__()
         self.memory_type = memory_type
@@ -179,3 +197,43 @@ class MemoryMapCollector(prometheus_client.registry.Collector):
             for e in dump_memory_maps(pid):
                 gauge.add_metric([pid, e["name"]], e[self.memory_type + "_kb"] * 1024)
         yield gauge
+
+
+class ResourceCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_resource"),
+            "Python resources",
+            labels=["name"],
+        )
+        r = resource.getrusage(resource.RUSAGE_SELF)
+        for field in dir(r):
+            if field.startswith("ru_"):
+                gauge.add_metric([field[3:]], getattr(r, field))
+        yield gauge
+
+
+class MemoryInfoCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    process = psutil.Process(os.getpid())
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_memory_info"),
+            "Python memory info",
+            labels=["name"],
+        )
+        memory_info = self.process.memory_info()
+        gauge.add_metric(["rss"], memory_info.rss)
+        gauge.add_metric(["vms"], memory_info.vms)
+        gauge.add_metric(["shared"], memory_info.shared)
+        gauge.add_metric(["text"], memory_info.text)
+        gauge.add_metric(["lib"], memory_info.lib)
+        gauge.add_metric(["data"], memory_info.data)
+        gauge.add_metric(["dirty"], memory_info.dirty)
+        yield gauge

c2cwsgiutils/pyramid.py

@@ -31,6 +31,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
 
     Arguments:
         config: The pyramid Configuration
+
     """
     logging.captureWarnings(True)
     config.include(coverage_setup.includeme)