c2cwsgiutils 5.1.7.dev20230901073305__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl

c2cwsgiutils/prometheus.py

@@ -1,66 +1,184 @@
-from typing import Any, Mapping, MutableMapping, Optional  # noqa  # pylint: disable=unused-import
+"""Every thing we needs to have the metrics in Prometheus."""
 
-import requests
+import os
+import re
+from collections.abc import Generator, Iterable
+from typing import Any, Optional, TypedDict, cast
 
-LabelsType = Optional[Mapping[str, Any]]
+import prometheus_client
+import prometheus_client.core
+import prometheus_client.metrics_core
+import prometheus_client.multiprocess
+import prometheus_client.registry
+import pyramid.config
 
+from c2cwsgiutils import broadcast, redis_utils
+from c2cwsgiutils.debug.utils import dump_memory_maps
 
-class PushgatewayGroupPublisher:
-    """
-    Implement parts of the Prometheus Pushgateway protocol.
+_NUMBER_RE = re.compile(r"^[0-9]+$")
+MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS = [
+    "c2cwsgiutils_prometheus_collector_gc",
+    "c2cwsgiutils_prometheus_collector_process",
+]
 
-    As defined here:
 
-    https://github.com/prometheus/pushgateway
-    """
+def start(registry: Optional[prometheus_client.CollectorRegistry] = None) -> None:
+    """Start separate HTTP server to provide the Prometheus metrics."""
 
-    def __init__(
-        self, base_url: str, job: str, instance: Optional[str] = None, labels: LabelsType = None
-    ) -> None:
-        if not base_url.endswith("/"):
-            base_url += "/"
-        self._url = f"{base_url}metrics/job/{job}"
-        if instance is not None:
-            self._url += "/instance/" + instance
-        self._labels = labels
-        self._reset()
+    if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
+        broadcast.includeme()
 
-    def _merge_labels(self, labels: LabelsType) -> LabelsType:
-        if labels is None:
-            return self._labels
-        elif self._labels is None:
-            return labels
-        else:
-            tmp = dict(self._labels)
-            tmp.update(labels)
-            return tmp
-
-    def add(
-        self,
-        metric_name: str,
-        metric_value: Any,
-        metric_type: str = "gauge",
-        metric_labels: Optional[Mapping[str, str]] = None,
-    ) -> None:
-        if metric_name in self._types:
-            if self._types[metric_name] != metric_type:
-                raise ValueError("Cannot change the type of a given metric")
+        registry = prometheus_client.CollectorRegistry() if registry is None else registry
+        registry.register(MemoryMapCollector())
+        registry.register(prometheus_client.PLATFORM_COLLECTOR)
+        registry.register(MultiProcessCustomCollector())
+        prometheus_client.multiprocess.MultiProcessCollector(registry)  # type: ignore[no-untyped-call]
+        prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"]), registry=registry)
+
+
+def includeme(config: pyramid.config.Configurator) -> None:
+    """Initialize prometheus_client in pyramid context."""
+
+    broadcast.subscribe("c2cwsgiutils_prometheus_collector_gc", _broadcast_collector_gc)
+    broadcast.subscribe("c2cwsgiutils_prometheus_collector_process", _broadcast_collector_process)
+
+
+def build_metric_name(postfix: str) -> str:
+    """Build the metric name with the prefix from the environment variable."""
+
+    return os.environ.get("C2C_PROMETHEUS_PREFIX", "c2cwsgiutils_") + postfix
+
+
+def cleanup() -> None:
+    """Cleanup the prometheus_client registry."""
+
+    redis_utils.cleanup()
+    broadcast.cleanup()
+
+
+class SerializedSample(TypedDict):
+    """Represent the serialized sample."""
+
+    name: str
+    labels: dict[str, str]
+    value: float
+
+
+class SerializedMetric(TypedDict):
+    """Represent the serialized gauge."""
+
+    type: str
+    args: dict[str, Any]
+    samples: list[SerializedSample]
+
+
+def _broadcast_collector_gc() -> list[SerializedMetric]:
+    """Get the collected GC gauges."""
+
+    return serialize_collected_data(prometheus_client.GC_COLLECTOR)
+
+
+def _broadcast_collector_process() -> list[SerializedMetric]:
+    """Get the collected process gauges."""
+    return serialize_collected_data(prometheus_client.PROCESS_COLLECTOR)
+
+
+def serialize_collected_data(collector: prometheus_client.registry.Collector) -> list[SerializedMetric]:
+    """Serialize the data from the custom collector."""
+
+    gauges: list[SerializedMetric] = []
+    for process_gauge in collector.collect():
+        gauge: SerializedMetric = {
+            "type": "<to be defined>",
+            "args": {
+                "name": process_gauge.name,
+                "documentation": process_gauge.documentation,
+                "unit": process_gauge.unit,
+            },
+            "samples": [],
+        }
+
+        if isinstance(process_gauge, prometheus_client.core.GaugeMetricFamily):
+            gauge["type"] = "gauge"
+        elif isinstance(process_gauge, prometheus_client.core.CounterMetricFamily):
+            gauge["type"] = "counter"
         else:
-            self._types[metric_name] = metric_type
-            self._to_send += f"# TYPE {metric_name} {metric_type}\n"
-        self._to_send += metric_name
-        labels = self._merge_labels(metric_labels)
-        if labels is not None:
-            self._to_send += "{" + ", ".join(f'{k}="{v}"' for k, v in sorted(labels.items())) + "}"
-        self._to_send += f" {metric_value}\n"
-
-    def commit(self) -> None:
-        requests.put(self._url, data=self._to_send.encode("utf-8")).raise_for_status()
-        self._reset()
-
-    def _reset(self) -> None:
-        self._to_send = ""
-        self._types: MutableMapping[str, str] = {}
-
-    def __str__(self) -> str:
-        return self._url + " ->\n" + self._to_send
+            raise NotImplementedError()
+        for sample in process_gauge.samples:
+            gauge["samples"].append(
+                {
+                    "name": sample.name,
+                    "labels": {"pid": str(os.getpid()), **sample.labels},
+                    "value": sample.value,
+                },
+            )
+        gauges.append(gauge)
+    return gauges
+
+
+class MultiProcessCustomCollector(prometheus_client.registry.Collector):
+    """Get the metrics from the custom collectors."""
+
+    def collect(self) -> Generator[prometheus_client.core.Metric, None, None]:
+        results: list[list[SerializedMetric]] = []
+        for channel in MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS:
+            result = broadcast.broadcast(channel, expect_answers=True)
+            if result is not None:
+                results.extend(cast(Iterable[list[SerializedMetric]], result))
+        return _deserialize_collected_data(results)
+
+
+def _deserialize_collected_data(
+    results: list[list[SerializedMetric]],
+) -> Generator[prometheus_client.core.Metric, None, None]:
+    for serialized_collection in results:
+        for serialized_metric in serialized_collection:
+            if serialized_metric is None:
+                continue
+
+            if serialized_metric["type"] == "gauge":
+                metric: prometheus_client.core.Metric = prometheus_client.core.GaugeMetricFamily(
+                    **serialized_metric["args"]
+                )
+            elif serialized_metric["type"] == "counter":
+                metric = prometheus_client.core.CounterMetricFamily(**serialized_metric["args"])
+            else:
+                raise NotImplementedError()
+            for sample in serialized_metric["samples"]:
+                metric.samples.append(
+                    prometheus_client.metrics_core.Sample(**sample),  # type: ignore[attr-defined]
+                )
+            yield metric
+
+
+class MemoryMapCollector(prometheus_client.registry.Collector):
+    """The Linux memory map provider."""
+
+    def __init__(self, memory_type: str = "pss", pids: Optional[list[str]] = None):
+        """
+        Initialize.
+
+        Arguments:
+
+            memory_type: can be rss, pss or size
+            pids: the list of pids or none
+        """
+        super().__init__()
+        self.memory_type = memory_type
+        self.pids = pids
+
+    def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]:
+        """Get the gauge from smap file."""
+        gauge = prometheus_client.core.GaugeMetricFamily(
+            build_metric_name(f"process_smap_{self.memory_type}"),
+            f"Container smap used {self.memory_type.capitalize()}",
+            labels=["pid", "name"],
+            unit="bytes",
+        )
+
+        for pid in (
+            [p for p in os.listdir("/proc/") if _NUMBER_RE.match(p)] if self.pids is None else self.pids
+        ):
+            for e in dump_memory_maps(pid):
+                gauge.add_metric([pid, e["name"]], e[self.memory_type + "_kb"] * 1024)
+        yield gauge

c2cwsgiutils/pyramid.py

@@ -12,8 +12,8 @@ from c2cwsgiutils import (
     errors,
     index,
     logging_view,
-    metrics,
     pretty_json,
+    prometheus,
     redis_stats,
     request_tracking,
     sentry,
@@ -22,6 +22,8 @@ from c2cwsgiutils import (
     version,
 )
 
+_LOG = logging.getLogger(__name__)
+
 
 def includeme(config: pyramid.config.Configurator) -> None:
     """
@@ -46,7 +48,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
     config.include(logging_view.includeme)
     config.include(sql_profiler.includeme)
     config.include(version.includeme)
+    config.include(prometheus.includeme)
     config.include(debug.includeme)
-    config.include(metrics.includeme)
     config.include(errors.includeme)
     config.include(index.includeme)

c2cwsgiutils/pyramid_logging.py

@@ -14,7 +14,8 @@ import json
 import logging
 import logging.config
 import socket
-from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, Optional, TextIO
+from collections.abc import Mapping, MutableMapping
+from typing import TYPE_CHECKING, Any, Optional, TextIO
 
 import cee_syslog_handler
 from pyramid.threadlocal import get_current_request

c2cwsgiutils/redis_stats.py

@@ -1,25 +1,27 @@
 import logging
 import warnings
-from typing import Any, Callable, Dict, Optional  # noqa  # pylint: disable=unused-import
+from typing import Any, Callable, Optional
 
+import prometheus_client
 import pyramid.config
 
-from c2cwsgiutils import config_utils, stats
+from c2cwsgiutils import config_utils, prometheus
 
 LOG = logging.getLogger(__name__)
 ORIG: Optional[Callable[..., Any]] = None
 
+_PROMETHEUS_REDIS_SUMMARY = prometheus_client.Summary(
+    prometheus.build_metric_name("redis"),
+    "Number of redis commands",
+    ["command"],
+    unit="seconds",
+)
 
-def _execute_command_patch(self: Any, *args: Any, **options: Any) -> Any:
-    if stats.USE_TAGS:
-        key = ["redis"]
-        tags: Optional[Dict[str, str]] = dict(cmd=args[0])
-    else:
-        key = ["redis", args[0]]
-        tags = None
+
+def _execute_command_patch(self: Any, command: str, *args: Any, **options: Any) -> Any:
     assert ORIG is not None
-    with stats.outcome_timer_context(key, tags):
-        return ORIG(self, *args, **options)
+    with _PROMETHEUS_REDIS_SUMMARY.labels(command=command).time():
+        return ORIG(self, command, *args, **options)
 
 
 def init(config: Optional[pyramid.config.Configurator] = None) -> None:

c2cwsgiutils/redis_utils.py

@@ -1,7 +1,8 @@
 import logging
 import threading
 import time
-from typing import Any, Mapping, Optional, Tuple
+from collections.abc import Mapping
+from typing import Any, Optional
 
 import redis.client
 import redis.exceptions
@@ -29,9 +30,17 @@ _slave: Optional["redis.client.Redis[str]"] = None
 _sentinel: Optional[redis.sentinel.Sentinel] = None
 
 
+def cleanup() -> None:
+    """Cleanup the redis connections."""
+    global _master, _slave, _sentinel
+    _master = None
+    _slave = None
+    _sentinel = None
+
+
 def get(
     settings: Optional[Mapping[str, bytes]] = None,
-) -> Tuple[
+) -> tuple[
     Optional["redis.client.Redis[str]"],
     Optional["redis.client.Redis[str]"],
     Optional[redis.sentinel.Sentinel],
@@ -73,19 +82,11 @@ def _init(settings: Optional[Mapping[str, Any]]) -> None:
             db=db,
             **redis_options,
         )
-
-        try:
-            LOG.info("Redis setup using: %s, %s, %s", sentinels, service_name, redis_options_)
-            _master = _sentinel.master_for(service_name)
-            _slave = _sentinel.slave_for(service_name)
-            return
-        except redis.sentinel.MasterNotFoundError as error:
-            print(_sentinel.sentinels[0].sentinel_masters())
-            raise Exception(_sentinel.sentinels[0].sentinel_masters()) from error
+        LOG.info("Redis setup using: %s, %s, %s", sentinels, service_name, redis_options_)
+        _master = _sentinel.master_for(service_name)
+        _slave = _sentinel.slave_for(service_name)
+        return
     if url:
-        if "://" not in url:
-            url = "redis://" + url
-
         LOG.info("Redis setup using: %s, with options: %s", url, redis_options_)
         _master = redis.client.Redis.from_url(url, decode_responses=True, **redis_options)
         _slave = _master

c2cwsgiutils/request_tracking/__init__.py

@@ -4,22 +4,31 @@ Allows to track the request_id in the logs, the DB and others.
 Adds a c2c_request_id attribute to the Pyramid Request class to access it.
 """
 import logging
+import time
 import urllib.parse
 import uuid
 import warnings
-from typing import Any, Dict, List, Optional, Sequence  # noqa  # pylint: disable=unused-import
+from collections.abc import Mapping
+from typing import Optional, Union
 
+import prometheus_client
 import pyramid.request
 import requests.adapters
 import requests.models
 from pyramid.threadlocal import get_current_request
 
-from c2cwsgiutils import config_utils, stats
+from c2cwsgiutils import config_utils, prometheus
 
-ID_HEADERS: List[str] = []
+ID_HEADERS: list[str] = []
 _HTTPAdapter_send = requests.adapters.HTTPAdapter.send
 LOG = logging.getLogger(__name__)
 DEFAULT_TIMEOUT: Optional[float] = None
+_PROMETHEUS_REQUESTS_SUMMARY = prometheus_client.Summary(
+    prometheus.build_metric_name("requests"),
+    "Requests requests",
+    ["scheme", "hostname", "port", "method", "status", "group"],
+    unit="seconds",
+)
 
 
 def _gen_request_id(request: pyramid.request.Request) -> str:
@@ -33,8 +42,11 @@ def _patch_requests() -> None:
     def send_wrapper(
         self: requests.adapters.HTTPAdapter,
         request: requests.models.PreparedRequest,
-        timeout: Optional[float] = None,
-        **kwargs: Any,
+        stream: bool = False,
+        timeout: Union[None, float, tuple[float, float], tuple[float, None]] = None,
+        verify: Union[bool, str] = True,
+        cert: Union[None, bytes, str, tuple[Union[bytes, str], Union[bytes, str]]] = None,
+        proxies: Optional[Mapping[str, str]] = None,
     ) -> requests.Response:
         pyramid_request = get_current_request()
         header = ID_HEADERS[0]
@@ -47,31 +59,25 @@ def _patch_requests() -> None:
             else:
                 LOG.warning("Doing a %s request without timeout to %s", request.method, request.url)
 
-        status = 999
-        timer = stats.timer()
-        try:
-            response = _HTTPAdapter_send(self, request, timeout=timeout, **kwargs)
-            status = response.status_code
-            return response
-        finally:
-            if request.url is not None:
-                parsed = urllib.parse.urlparse(request.url)
-                port = parsed.port or (80 if parsed.scheme == "http" else 443)
-                if stats.USE_TAGS:
-                    key: Sequence[Any] = ["requests"]
-                    tags: Optional[Dict[str, Any]] = dict(
-                        scheme=parsed.scheme,
-                        host=parsed.hostname,
-                        port=port,
-                        method=request.method,
-                        status=status,
-                    )
-                else:
-                    key = ["requests", parsed.scheme, parsed.hostname, port, request.method, status]
-                    tags = None
-                timer.stop(key, tags)
-
-    requests.adapters.HTTPAdapter.send = send_wrapper  # type: ignore
+        assert request.url
+        parsed = urllib.parse.urlparse(request.url)
+        port = parsed.port or (80 if parsed.scheme == "http" else 443)
+        start = time.perf_counter()
+        response = _HTTPAdapter_send(
+            self, request, timeout=timeout, stream=stream, verify=verify, cert=cert, proxies=proxies
+        )
+
+        _PROMETHEUS_REQUESTS_SUMMARY.labels(
+            scheme=parsed.scheme,
+            hostname=parsed.hostname,
+            port=str(port),
+            method=request.method,
+            status=str(response.status_code),
+            group=str(response.status_code // 100 * 100),
+        ).observe(time.perf_counter() - start)
+        return response
+
+    requests.adapters.HTTPAdapter.send = send_wrapper  # type: ignore[method-assign]
 
 
 def init(config: Optional[pyramid.config.Configurator] = None) -> None:

c2cwsgiutils/request_tracking/_sql.py

@@ -8,7 +8,9 @@ from sqlalchemy.orm import Session
 def _add_session_id(session: Session, _transaction: Any, _connection: Any) -> None:
     request = get_current_request()
     if request is not None:
-        session.execute("set application_name=:session_id", params={"session_id": request.c2c_request_id})
+        session.execute(
+            sqlalchemy.text("set application_name=:session_id"), params={"session_id": request.c2c_request_id}
+        )
 
 
 def init() -> None:

c2cwsgiutils/scripts/genversion.py

@@ -6,14 +6,14 @@ import re
 import subprocess  # nosec
 import sys
 import warnings
-from typing import Dict, Optional, Tuple, cast
+from typing import Optional, cast
 
 SRC_VERSION_RE = re.compile(r"^.*\(([^=]*)===?([^=]*)\)$")
 VERSION_RE = re.compile(r"^([^=]*)==([^=]*)$")
 LOG = logging.getLogger(__name__)
 
 
-def _get_package_version(comp: str) -> Tuple[Optional[str], Optional[str]]:
+def _get_package_version(comp: str) -> tuple[Optional[str], Optional[str]]:
     """
     Parse plain and editable versions.
 
@@ -22,14 +22,14 @@ def _get_package_version(comp: str) -> Tuple[Optional[str], Optional[str]]:
     src_matcher = SRC_VERSION_RE.match(comp)
     matcher = src_matcher or VERSION_RE.match(comp)
     if matcher:
-        return cast(Tuple[str, str], matcher.groups())
+        return cast(tuple[str, str], matcher.groups())
     else:
         if len(comp) > 0 and not comp[:3] == "-e ":
             print("Cannot parse package version: " + comp)
         return None, None
 
 
-def _get_packages_version() -> Dict[str, str]:
+def _get_packages_version() -> dict[str, str]:
     result = {}
     with open(os.devnull, "w", encoding="utf-8") as devnull:
         for comp in (