c2cwsgiutils 6.2.0.dev36__py3-none-any.whl → 6.2.0.dev41__py3-none-any.whl

c2cwsgiutils/loader.py

@@ -1,9 +1,9 @@
-import logging
+import logging.config
 from typing import Optional, cast
 
 from plaster_pastedeploy import Loader as BaseLoader
 
-from c2cwsgiutils import get_config_defaults
+from c2cwsgiutils import get_config_defaults, get_logconfig_dict
 
 _LOG = logging.getLogger(__name__)
 
@@ -19,3 +19,21 @@ class Loader(BaseLoader):  # type: ignore
     def __repr__(self) -> str:
         """Get the object representation."""
         return f'c2cwsgiutils.loader.Loader(uri="{self.uri}")'
+
+    def setup_logging(self, defaults: Optional[dict[str, str]] = None) -> None:
+        """
+        Set up logging via :func:`logging.config.fileConfig`.
+
+        Defaults are specified for the special ``__file__`` and ``here``
+        variables, similar to PasteDeploy config loading. Extra defaults can
+        optionally be specified as a dict in ``defaults``.
+
+        Arguments:
+            defaults: The defaults that will be used when passed to
+                :func:`logging.config.fileConfig`.
+
+        """
+        if "loggers" in self.get_sections():
+            logging.config.dictConfig(get_logconfig_dict(self.uri.path))
+        else:
+            logging.basicConfig()

c2cwsgiutils/prometheus.py

@@ -2,6 +2,7 @@
 
 import os
 import re
+import resource
 from collections.abc import Generator, Iterable
 from typing import Any, Optional, TypedDict, cast
 
@@ -15,6 +16,12 @@ import pyramid.config
 from c2cwsgiutils import broadcast, redis_utils
 from c2cwsgiutils.debug.utils import dump_memory_maps
 
+PSUTILS = True
+try:
+    import psutil
+except ImportError:
+    PSUTILS = False
+
 _NUMBER_RE = re.compile(r"^[0-9]+$")
 MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
     "c2cwsgiutils_prometheus_collector_gc",
@@ -22,12 +29,21 @@ MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
 ]
 
 
+def start_single_process() -> None:
+    """Start separate HTTP server to provide the Prometheus metrics."""
+    if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
+        prometheus_client.REGISTRY.register(ResourceCollector())
+        prometheus_client.REGISTRY.register(MemoryInfoCollector())
+        prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"]))
+
+
 def start(registry: Optional[prometheus_client.CollectorRegistry] = None) -> None:
     """Start separate HTTP server to provide the Prometheus metrics."""
     if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
         broadcast.includeme()
 
         registry = prometheus_client.CollectorRegistry() if registry is None else registry
+        registry.register(ResourceCollector())
         registry.register(MemoryMapCollector())
         registry.register(prometheus_client.PLATFORM_COLLECTOR)
         registry.register(MultiProcessCustomCollector())
@@ -181,3 +197,43 @@ class MemoryMapCollector(prometheus_client.registry.Collector):
             for e in dump_memory_maps(pid):
                 gauge.add_metric([pid, e["name"]], e[self.memory_type + "_kb"] * 1024)
         yield gauge
+
+
+class ResourceCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_resource"),
+            "Python resources",
+            labels=["name"],
+        )
+        r = resource.getrusage(resource.RUSAGE_SELF)
+        for field in dir(r):
+            if field.startswith("ru_"):
+                gauge.add_metric([field[3:]], getattr(r, field))
+        yield gauge
+
+
+class MemoryInfoCollector(prometheus_client.registry.Collector):
+    """Collect the resources used by Python."""
+
+    process = psutil.Process(os.getpid())
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name("python_memory_info"),
+            "Python memory info",
+            labels=["name"],
+        )
+        memory_info = self.process.memory_info()
+        gauge.add_metric(["rss"], memory_info.rss)
+        gauge.add_metric(["vms"], memory_info.vms)
+        gauge.add_metric(["shared"], memory_info.shared)
+        gauge.add_metric(["text"], memory_info.text)
+        gauge.add_metric(["lib"], memory_info.lib)
+        gauge.add_metric(["data"], memory_info.data)
+        gauge.add_metric(["dirty"], memory_info.dirty)
+        yield gauge

c2cwsgiutils/setup_process.py

@@ -5,6 +5,7 @@ Must be imported at the very beginning of the process's life, before any other m
 """
 
 import argparse
+import logging
 import warnings
 from typing import Any, Callable, Optional, TypedDict, cast
 
@@ -97,4 +98,5 @@ def bootstrap_application(
     """
     loader = get_config_loader(config_uri)
     loader.setup_logging(options)
+    logging.getLogger(__name__).info("Loading the application from %s", config_uri)
     return cast(PyramidEnv, bootstrap(config_uri, options=options))

{c2cwsgiutils-6.2.0.dev36.dist-info → c2cwsgiutils-6.2.0.dev41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: c2cwsgiutils
-Version: 6.2.0.dev36
+Version: 6.2.0.dev41
 Summary: Common utilities for Camptocamp WSGI applications
 Home-page: https://github.com/camptocamp/c2cwsgiutils
 License: BSD-2-Clause
@@ -33,23 +33,26 @@ Provides-Extra: sentry
 Provides-Extra: standard
 Provides-Extra: test-images
 Provides-Extra: tests
+Provides-Extra: waitress
 Provides-Extra: webserver
-Requires-Dist: SQLAlchemy ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: SQLAlchemy-Utils ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: Paste ; extra == "standard" or extra == "waitress" or extra == "all"
+Requires-Dist: SQLAlchemy ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: SQLAlchemy-Utils ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: alembic ; extra == "standard" or extra == "alembic" or extra == "all"
 Requires-Dist: boltons ; extra == "tests" or extra == "all"
 Requires-Dist: cee_syslog_handler
-Requires-Dist: cornice ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: cornice ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: coverage ; extra == "debug" or extra == "all"
 Requires-Dist: gunicorn ; extra == "standard" or extra == "webserver" or extra == "all"
 Requires-Dist: lxml ; extra == "tests" or extra == "all"
 Requires-Dist: objgraph ; extra == "debug" or extra == "all"
-Requires-Dist: prometheus-client ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: prometheus-client ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: psutil ; extra == "debug" or extra == "all"
-Requires-Dist: psycopg2 ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: psycopg2 ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: pyjwt ; extra == "standard" or extra == "oauth2" or extra == "all"
-Requires-Dist: pyramid ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: pyramid-tm ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: pyramid_mako ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: pyramid ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: pyramid-tm ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: pyramid_mako ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Requires-Dist: pyyaml
 Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all"
 Requires-Dist: requests
@@ -57,9 +60,9 @@ Requires-Dist: requests-oauthlib ; extra == "standard" or extra == "oauth2" or e
 Requires-Dist: scikit-image ; extra == "test-images"
 Requires-Dist: sentry-sdk ; extra == "standard" or extra == "sentry" or extra == "all"
 Requires-Dist: ujson
-Requires-Dist: waitress ; extra == "dev" or extra == "all"
-Requires-Dist: zope.interface ; extra == "standard" or extra == "webserver" or extra == "all"
-Requires-Dist: zope.sqlalchemy ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: waitress ; extra == "standard" or extra == "dev" or extra == "waitress" or extra == "all"
+Requires-Dist: zope.interface ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
+Requires-Dist: zope.sqlalchemy ; extra == "standard" or extra == "webserver" or extra == "waitress" or extra == "all"
 Project-URL: Repository, https://github.com/camptocamp/c2cwsgiutils
 Description-Content-Type: text/markdown
 
@@ -673,6 +676,64 @@ def hello_get(request):
     return {'hello': True}
 ```
 
+## Waitress
+
+In production mode we usually use Gunicorn but we can also use Waitress.
+
+The advantage to use Waitress is that it creates only one process, that makes it easier to manage especially on Kubernetes:
+
+- The memory is more stable.
+- The OOM killer will restart the container.
+- Prometheus didn't request trick to aggregate the metrics.
+
+Then to migrate from Gunicorn to Waitress you should do:
+
+Add call to `c2cwsgiutils.prometheus.start_single_process()` on your application main function.
+
+Changes to do in your docker file:
+
+```diff
+
+ ENV \
+-   GUNICORN_LOG_LEVEL=WARNING \
++   WAITRESS_LOG_LEVEL=WARNING \
++   WAITRESS_THREADS=10 \
+
+-RUN mkdir -p /prometheus-metrics \
+-  && chmod a+rwx /prometheus-metrics
+-ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+
+-CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"]
++CMD ["/venv/bin/pserve", "c2c:///app/production.ini"]
+```
+
+Remove the no more needed file `gunicorn.conf.py`.
+
+Update the `production.ini` file:
+
+```diff
+
+-# this file should be used by gunicorn.
+
+ [server:main]
++threads = %(WAITRESS_THREADS)s
++trusted_proxy = True
++clear_untrusted_proxy_headers = False
+
+ [loggers]
+-keys = root, gunicorn, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
++keys = root, waitress, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
+
+-[logger_gunicorn]
+-level = %(GUNICORN_LOG_LEVEL)s
++[logger_waitress]
++level = %(WAITRESS_LOG_LEVEL)s
+ handlers =
+-qualname = gunicorn.error
++qualname = waitress
+```
+
 # Exception handling
 
 c2cwsgiutils can install exception handling views that will catch any exception raised by the

{c2cwsgiutils-6.2.0.dev36.dist-info → c2cwsgiutils-6.2.0.dev41.dist-info}/RECORD

@@ -25,12 +25,12 @@ c2cwsgiutils/debug/utils.py,sha256=sIKZHQ8empzxE2OI3h7Uce8cvv4O7AD1y_VYeZfLVCA,2
 c2cwsgiutils/errors.py,sha256=miNVnEX9qBtwaRnny_k9IcBbbw-8vC3unFumi2w5Q9w,6679
 c2cwsgiutils/health_check.py,sha256=Wi3JQCKa7tqACuSgBi13oE69Xc-Vz1nheqBaxfOGXQU,20141
 c2cwsgiutils/index.py,sha256=ag7A8HJ9MgU72etDBuFMaci-xKjOD9R8OOm3UMEoQJY,18002
-c2cwsgiutils/loader.py,sha256=vU7yEobl9TlSFZwdLI2eF1ceUJ6xgiu2re31HA5sw1g,623
+c2cwsgiutils/loader.py,sha256=Hl9q-Ts-kXHLkV5PWFoO0jPBqhU2jKoCFvgDIdJaeF4,1333
 c2cwsgiutils/logging_view.py,sha256=wa2hCjGKkpRwrCA4K6Zk1Z3hPGUMB6SJAnki-_lLY00,3384
 c2cwsgiutils/models_graph.py,sha256=AROTzCHJCjjOiOmQj_Hk40WQdIPaHFLgN4Jech1aUzc,2683
 c2cwsgiutils/pretty_json.py,sha256=houw_oeU63CgzQTPlBrrYzinpE-UtqPeYLfTXdLHyu0,1711
 c2cwsgiutils/profiler.py,sha256=3902yp1rHpSNpsYKmqs_IrEpFg9P2QqNtjyolk53ZgQ,778
-c2cwsgiutils/prometheus.py,sha256=XIVnsI9zcIZsg0-tqrDSiHClDwdr3l5KMkmpFKyfUXk,6635
+c2cwsgiutils/prometheus.py,sha256=vTb0cDO30DQwjVdu0qCFEyTAOv7jeUFioDjTCbp1bYA,8712
 c2cwsgiutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 c2cwsgiutils/pyramid.py,sha256=kSCsBG5IPd0-tsXwdiJ3cj_yj-AxQYFusvTxPE6AaeQ,1388
 c2cwsgiutils/pyramid_logging.py,sha256=U1MD7xEAGH7QdX7FA6O12eHhzMx76i1WCXtf_EiJgVE,3911
@@ -44,7 +44,7 @@ c2cwsgiutils/scripts/stats_db.py,sha256=EaI1WAHGKj-sUWpznIOs8yMlgx2pnzwAksA8hA-n
 c2cwsgiutils/scripts/test_print.py,sha256=pLqqVZ9cVvjdD3MdJQ7Ul8i5OIBFvo1Gnr78rdH0Dh4,2130
 c2cwsgiutils/sentry.py,sha256=ZnWo0-8xgwWxDG_wo9gI8rDtZlKgcZXorG3b0tU5Hh0,6508
 c2cwsgiutils/services.py,sha256=AxdiH2S8gfEQpOY36R4WqKtJ-3P4OXhx0srTm5G5070,1569
-c2cwsgiutils/setup_process.py,sha256=19p-0ZRZQ08S6-hkZvAq_VJjtrLeWxsdoX-afRLJapg,3454
+c2cwsgiutils/setup_process.py,sha256=K4o3feB98zKoPiI2k7MnTsItwf_i-NSL8ziUmGnYg_Q,3553
 c2cwsgiutils/sql_profiler/__init__.py,sha256=YR8003lCNNGC2eZdyIt5gt76-1frG5szw8IYE_EsQB4,921
 c2cwsgiutils/sql_profiler/_impl.py,sha256=OdhAn-5x84Bz5RCldhgjX50b6yxSLnLh5nydNgHvgGc,3801
 c2cwsgiutils/sqlalchemylogger/README.md,sha256=qXfyhGC4Kbt-Ye_geTPm_1InpclhthmKZpc66rqB4as,2018
@@ -60,8 +60,8 @@ c2cwsgiutils/stats_pyramid/_db_spy.py,sha256=A61t6VKIrRRIjbyZTldmAUl_Q3ZDVFYqyxj
 c2cwsgiutils/stats_pyramid/_pyramid_spy.py,sha256=GnJRlLSAPl4PwQH5jfmc2OX5Hz9MSRRVJT2VcqkniVE,3518
 c2cwsgiutils/templates/index.html.mako,sha256=cK8qGBDeQG5SiJJCfvL0oKpgacr7dPKx634AAQivmjA,1416
 c2cwsgiutils/version.py,sha256=7H3URblj26Ql0bL3eXtP0LSRBeW4HbEsQ8O_BfWNr90,3124
-c2cwsgiutils-6.2.0.dev36.dist-info/LICENSE,sha256=r7ueGz9Fl2Bv3rmeQy0DEtohLmAiufRaCuv6Y5fyNhE,1304
-c2cwsgiutils-6.2.0.dev36.dist-info/METADATA,sha256=s18TzIHXyoYwmI7W0eKRRh2r9Ed4DNMxAUAntveyU20,34461
-c2cwsgiutils-6.2.0.dev36.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-c2cwsgiutils-6.2.0.dev36.dist-info/entry_points.txt,sha256=ujgqMTL1awN9qDg8WXmrF7m0fgR-hslUM6zKH86pvy0,703
-c2cwsgiutils-6.2.0.dev36.dist-info/RECORD,,
+c2cwsgiutils-6.2.0.dev41.dist-info/LICENSE,sha256=r7ueGz9Fl2Bv3rmeQy0DEtohLmAiufRaCuv6Y5fyNhE,1304
+c2cwsgiutils-6.2.0.dev41.dist-info/METADATA,sha256=AUqcjIsz5x6TK9kL03Q2EOlR2dTLvoneGmYseJDVy0s,36359
+c2cwsgiutils-6.2.0.dev41.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+c2cwsgiutils-6.2.0.dev41.dist-info/entry_points.txt,sha256=ujgqMTL1awN9qDg8WXmrF7m0fgR-hslUM6zKH86pvy0,703
+c2cwsgiutils-6.2.0.dev41.dist-info/RECORD,,