c2cwsgiutils 6.2.0.dev36__tar.gz → 6.2.0.dev41__tar.gz

{c2cwsgiutils-6.2.0.dev36 → c2cwsgiutils-6.2.0.dev41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: c2cwsgiutils
-Version: 6.2.0.dev36
+Version: 6.2.0.dev41
 Summary: Common utilities for Camptocamp WSGI applications
 Home-page: https://github.com/camptocamp/c2cwsgiutils
 License: BSD-2-Clause
@@ -33,23 +33,26 @@ Provides-Extra: sentry
 Provides-Extra: standard
 Provides-Extra: test-images
 Provides-Extra: tests
+Provides-Extra: waitress
 Provides-Extra: webserver
-Requires-Dist: SQLAlchemy ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: SQLAlchemy-Utils ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: Paste ; extra == "standard" or extra == "waitress" or extra == "all"
+Requires-Dist: SQLAlchemy ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: SQLAlchemy-Utils ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: alembic ; extra == "standard" or extra == "alembic" or extra == "all"
 Requires-Dist: boltons ; extra == "tests" or extra == "all"
 Requires-Dist: cee_syslog_handler
-Requires-Dist: cornice ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: cornice ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: coverage ; extra == "debug" or extra == "all"
 Requires-Dist: gunicorn ; extra == "standard" or extra == "webserver" or extra == "all"
 Requires-Dist: lxml ; extra == "tests" or extra == "all"
 Requires-Dist: objgraph ; extra == "debug" or extra == "all"
-Requires-Dist: prometheus-client ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: prometheus-client ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: psutil ; extra == "debug" or extra == "all"
-Requires-Dist: psycopg2 ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: psycopg2 ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: pyjwt ; extra == "standard" or extra == "oauth2" or extra == "all"
-Requires-Dist: pyramid ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: pyramid-tm ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: pyramid_mako ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: pyramid ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: pyramid-tm ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: pyramid_mako ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: pyyaml
 Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all"
 Requires-Dist: requests
@@ -57,9 +60,9 @@ Requires-Dist: requests-oauthlib ; extra == "standard" or extra == "oauth2" or e
 Requires-Dist: scikit-image ; extra == "test-images"
 Requires-Dist: sentry-sdk ; extra == "standard" or extra == "sentry" or extra == "all"
 Requires-Dist: ujson
-Requires-Dist: waitress ; extra == "dev" or extra == "all"
-Requires-Dist: zope.interface ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: zope.sqlalchemy ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: waitress ; extra == "standard" or extra == "dev" or extra == "waitress" or extra == "all"
+Requires-Dist: zope.interface ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: zope.sqlalchemy ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Project-URL: Repository, https://github.com/camptocamp/c2cwsgiutils
 Description-Content-Type: text/markdown
 
@@ -673,6 +676,64 @@ def hello_get(request):
     return {'hello': True}
 ```
 
+## Waitress
+
+In production mode we usually use Gunicorn but we can also use Waitress.
+
+The advantage to use Waitress is that it creates only one process, that makes it easier to manage especially on Kubernetes:
+
+- The memory is more stable.
+- The OOM killer will restart the container.
+- Prometheus didn't request trick to aggregate the metrics.
+
+Then to migrate from Gunicorn to Waitress you should do:
+
+Add call to `c2cwsgiutils.prometheus.start_single_process()` on your application main function.
+
+Changes to do in your docker file:
+
+```diff
+
+ ENV \
+-   GUNICORN_LOG_LEVEL=WARNING \
++   WAITRESS_LOG_LEVEL=WARNING \
++   WAITRESS_THREADS=10 \
+
+-RUN mkdir -p /prometheus-metrics \
+-  && chmod a+rwx /prometheus-metrics
+-ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+
+-CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"]
++CMD ["/venv/bin/pserve", "c2c:///app/production.ini"]
+```
+
+Remove the no more needed file `gunicorn.conf.py`.
+
+Update the `production.ini` file:
+
+```diff
+
+-# this file should be used by gunicorn.
+
+ [server:main]
++threads = %(WAITRESS_THREADS)s
++trusted_proxy = True
++clear_untrusted_proxy_headers = False
+
+ [loggers]
+-keys = root, gunicorn, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
++keys = root, waitress, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
+
+-[logger_gunicorn]
+-level = %(GUNICORN_LOG_LEVEL)s
++[logger_waitress]
++level = %(WAITRESS_LOG_LEVEL)s
+ handlers =
+-qualname = gunicorn.error
++qualname = waitress
+```
+
 # Exception handling
 
 c2cwsgiutils can install exception handling views that will catch any exception raised by the

{c2cwsgiutils-6.2.0.dev36 → c2cwsgiutils-6.2.0.dev41}/README.md

@@ -608,6 +608,64 @@ def hello_get(request):
     return {'hello': True}
 ```
 
+## Waitress
+
+In production mode we usually use Gunicorn but we can also use Waitress.
+
+The advantage to use Waitress is that it creates only one process, that makes it easier to manage especially on Kubernetes:
+
+- The memory is more stable.
+- The OOM killer will restart the container.
+- Prometheus didn't request trick to aggregate the metrics.
+
+Then to migrate from Gunicorn to Waitress you should do:
+
+Add call to `c2cwsgiutils.prometheus.start_single_process()` on your application main function.
+
+Changes to do in your docker file:
+
+```diff
+
+ ENV \
+-   GUNICORN_LOG_LEVEL=WARNING \
++   WAITRESS_LOG_LEVEL=WARNING \
++   WAITRESS_THREADS=10 \
+
+-RUN mkdir -p /prometheus-metrics \
+-  && chmod a+rwx /prometheus-metrics
+-ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+
+-CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"]
++CMD ["/venv/bin/pserve", "c2c:///app/production.ini"]
+```
+
+Remove the no more needed file `gunicorn.conf.py`.
+
+Update the `production.ini` file:
+
+```diff
+
+-# this file should be used by gunicorn.
+
+ [server:main]
++threads = %(WAITRESS_THREADS)s
++trusted_proxy = True
++clear_untrusted_proxy_headers = False
+
+ [loggers]
+-keys = root, gunicorn, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
++keys = root, waitress, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
+
+-[logger_gunicorn]
+-level = %(GUNICORN_LOG_LEVEL)s
++[logger_waitress]
++level = %(WAITRESS_LOG_LEVEL)s
+ handlers =
+-qualname = gunicorn.error
++qualname = waitress
+```
+
 # Exception handling
 
 c2cwsgiutils can install exception handling views that will catch any exception raised by the

c2cwsgiutils-6.2.0.dev41/c2cwsgiutils/loader.py

@@ -0,0 +1,39 @@
+import logging.config
+from typing import Optional, cast
+
+from plaster_pastedeploy import Loader as BaseLoader
+
+from c2cwsgiutils import get_config_defaults, get_logconfig_dict
+
+_LOG = logging.getLogger(__name__)
+
+
+class Loader(BaseLoader):  # type: ignore
+    """The application loader."""
+
+    def _get_defaults(self, defaults: Optional[dict[str, str]] = None) -> dict[str, str]:
+        d = get_config_defaults()
+        d.update(defaults or {})
+        return cast(dict[str, str], super()._get_defaults(d))
+
+    def __repr__(self) -> str:
+        """Get the object representation."""
+        return f'c2cwsgiutils.loader.Loader(uri="{self.uri}")'
+
+    def setup_logging(self, defaults: Optional[dict[str, str]] = None) -> None:
+        """
+        Set up logging via :func:`logging.config.fileConfig`.
+
+        Defaults are specified for the special ``__file__`` and ``here``
+        variables, similar to PasteDeploy config loading. Extra defaults can
+        optionally be specified as a dict in ``defaults``.
+
+        Arguments:
+            defaults: The defaults that will be used when passed to
+                :func:`logging.config.fileConfig`.
+
+        """
+        if "loggers" in self.get_sections():
+            logging.config.dictConfig(get_logconfig_dict(self.uri.path))
+        else:
+            logging.basicConfig()

{c2cwsgiutils-6.2.0.dev36 → c2cwsgiutils-6.2.0.dev41}/c2cwsgiutils/prometheus.py

@@ -2,6 +2,7 @@
 
 import os
 import re
+import resource
 from collections.abc import Generator, Iterable
 from typing import Any, Optional, TypedDict, cast
 
@@ -15,6 +16,12 @@ import pyramid.config
 from c2cwsgiutils import broadcast, redis_utils
 from c2cwsgiutils.debug.utils import dump_memory_maps
 
+PSUTILS = True
+try:
+    import psutil
+except ImportError:
+    PSUTILS = False
+
 _NUMBER_RE = re.compile(r"^[0-9]+$")
 MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
     "c2cwsgiutils_prometheus_collector_gc",
@@ -22,12 +29,21 @@ MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
 ]
 
 
+def start_single_process() -> None:
+    """Start separate HTTP server to provide the Prometheus metrics."""
+    if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
+        prometheus_client.REGISTRY.register(ResourceCollector())
+        prometheus_client.REGISTRY.register(MemoryInfoCollector())
+        prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"]))
+
+
 def start(registry: Optional[prometheus_client.CollectorRegistry] = None) -> None:
     """Start separate HTTP server to provide the Prometheus metrics."""
     if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
         broadcast.includeme()
 
         registry = prometheus_client.CollectorRegistry() if registry is None else registry
+        registry.register(ResourceCollector())
         registry.register(MemoryMapCollector())
         registry.register(prometheus_client.PLATFORM_COLLECTOR)
         registry.register(MultiProcessCustomCollector())
@@ -181,3 +197,43 @@ class MemoryMapCollector(prometheus_client.registry.Collector):
             for e in dump_memory_maps(pid):
                 gauge.add_metric([pid, e["name"]], e[self.memory_type + "_kb"] * 1024)
         yield gauge
+
+
+class ResourceCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_resource"),
+            "Python resources",
+            labels=["name"],
+        )
+        r = resource.getrusage(resource.RUSAGE_SELF)
+        for field in dir(r):
+            if field.startswith("ru_"):
+                gauge.add_metric([field[3:]], getattr(r, field))
+        yield gauge
+
+
+class MemoryInfoCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    process = psutil.Process(os.getpid())
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_memory_info"),
+            "Python memory info",
+            labels=["name"],
+        )
+        memory_info = self.process.memory_info()
+        gauge.add_metric(["rss"], memory_info.rss)
+        gauge.add_metric(["vms"], memory_info.vms)
+        gauge.add_metric(["shared"], memory_info.shared)
+        gauge.add_metric(["text"], memory_info.text)
+        gauge.add_metric(["lib"], memory_info.lib)
+        gauge.add_metric(["data"], memory_info.data)
+        gauge.add_metric(["dirty"], memory_info.dirty)
+        yield gauge

{c2cwsgiutils-6.2.0.dev36 → c2cwsgiutils-6.2.0.dev41}/c2cwsgiutils/setup_process.py

@@ -5,6 +5,7 @@ Must be imported at the very beginning of the process's life, before any other m
 """
 
 import argparse
+import logging
 import warnings
 from typing import Any, Callable, Optional, TypedDict, cast
 
@@ -97,4 +98,5 @@ def bootstrap_application(
     """
     loader = get_config_loader(config_uri)
     loader.setup_logging(options)
+    logging.getLogger(__name__).info("Loading the application from %s", config_uri)
     return cast(PyramidEnv, bootstrap(config_uri, options=options))

{c2cwsgiutils-6.2.0.dev36 → c2cwsgiutils-6.2.0.dev41}/pyproject.toml

@@ -16,7 +16,7 @@ strict = true
 
 [tool.poetry]
 name = "c2cwsgiutils"
-version = "6.2.0.dev36"
+version = "6.2.0.dev41"
 description = "Common utilities for Camptocamp WSGI applications"
 readme = "README.md"
 authors = ["Camptocamp <info@camptocamp.com>"]
@@ -93,6 +93,8 @@ scikit-image = { version = "0.24.0", optional = true }
 prometheus-client = { version = "0.21.0", optional = true}
 pyramid_mako = { version = "1.1.0", optional = true}
 psutil = { version = "6.1.0", optional = true}
+Paste = { version = "3.10.1", optional = true}
+coverage = { version = "7.6.4", optional = true}
 
 [tool.poetry.extras]
 standard = [
@@ -117,9 +119,12 @@ standard = [
   "zope.sqlalchemy",
   "prometheus_client",
   "pyramid_mako",
+  # waitress
+  "waitress",
+  "Paste",
 ]
 alembic = ["alembic"]
-debug = ["objgraph", "psutil"]
+debug = ["objgraph", "psutil", "coverage"]
 oauth2 = ["pyjwt", "requests-oauthlib"]
 sentry = ["sentry-sdk"]
 dev = ["waitress"]
@@ -136,6 +141,19 @@ webserver = [
   "zope.sqlalchemy",
   "prometheus_client",
   "pyramid_mako"]
+waitress = [
+  "waitress",
+  "Paste",
+  "cornice",
+  "psycopg2",
+  "pyramid",
+  "pyramid-tm",
+  "SQLAlchemy",
+  "SQLAlchemy-Utils",
+  "zope.interface",
+  "zope.sqlalchemy",
+  "prometheus_client",
+  "pyramid_mako"]
 tests = ["lxml", "boltons"]
 all = [
   # alembic
@@ -143,13 +161,12 @@ all = [
   # debug
   "objgraph",
   "psutil",
+  "coverage",
   # oauth2
   "pyjwt",
   "requests-oauthlib",
   # sentry
   "sentry-sdk",
-  # dev
-  "waitress",
   # broadcast
   "redis",
   # webserver
@@ -167,6 +184,9 @@ all = [
   # tests
   "lxml",
   "boltons",
+  # waitress
+  "waitress",
+  "Paste",
 ]
 test_images = ["scikit-image"]
 
@@ -175,7 +195,6 @@ test_images = ["scikit-image"]
 prospector = { version = "1.13.3", extras = ["with_bandit", "with_mypy", "with_pyroma", "with_ruff"] }
 prospector-profile-duplicated = "1.9.0"
 prospector-profile-utils = "1.14.0"
-coverage = "7.6.8"
 junit2html = "31.0.2"
 pytest = "8.3.4"
 pytest-cov = "6.0.0"

c2cwsgiutils-6.2.0.dev36/c2cwsgiutils/loader.py

@@ -1,21 +0,0 @@
-import logging
-from typing import Optional, cast
-
-from plaster_pastedeploy import Loader as BaseLoader
-
-from c2cwsgiutils import get_config_defaults
-
-_LOG = logging.getLogger(__name__)
-
-
-class Loader(BaseLoader):  # type: ignore
-    """The application loader."""
-
-    def _get_defaults(self, defaults: Optional[dict[str, str]] = None) -> dict[str, str]:
-        d = get_config_defaults()
-        d.update(defaults or {})
-        return cast(dict[str, str], super()._get_defaults(d))
-
-    def __repr__(self) -> str:
-        """Get the object representation."""
-        return f'c2cwsgiutils.loader.Loader(uri="{self.uri}")'