c2cwsgiutils 5.2.1__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl

c2cwsgiutils/scripts/stats_db.py

@@ -1,21 +1,29 @@
 #!/usr/bin/env python3
-"""Emits statsd gauges for every tables of a database."""
+"""Provide prometheus gauges for every tables of a database."""
+
 import argparse
 import logging
 import os
 import sys
 import time
-from typing import Dict, List, Optional
+from typing import TYPE_CHECKING, Optional
+from wsgiref.simple_server import make_server
 
 import sqlalchemy
 import sqlalchemy.exc
 import sqlalchemy.orm
 import transaction
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+from prometheus_client.exposition import make_wsgi_app
 from zope.sqlalchemy import register
 
 import c2cwsgiutils.setup_process
-from c2cwsgiutils import stats
-from c2cwsgiutils.prometheus import PushgatewayGroupPublisher
+from c2cwsgiutils import prometheus
+
+if TYPE_CHECKING:
+    scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
+else:
+    scoped_session = sqlalchemy.orm.scoped_session
 
 LOG = logging.getLogger(__name__)
 
@@ -28,13 +36,17 @@ def _parse_args() -> argparse.Namespace:
         "--schema", type=str, action="append", required=True, default=["public"], help="schema to dump"
     )
     parser.add_argument(
-        "--extra", type=str, action="append", help="A SQL query that returns a metric name and a value"
-    )
-    parser.add_argument(
-        "--statsd-address", "--statsd_address", type=str, help="address:port for the statsd daemon"
+        "--extra",
+        type=str,
+        action="append",
+        help="A SQL query that returns a metric name and a value",
     )
     parser.add_argument(
-        "--statsd-prefix", "--statsd_prefix", type=str, default="c2c", help="prefix for the statsd metrics"
+        "--extra-gauge",
+        type=str,
+        action="append",
+        nargs=3,
+        help="A SQL query that returns a metric name and a value, with gauge name and help",
     )
     parser.add_argument(
         "--prometheus-url", "--prometheus_url", type=str, help="Base URL for the Prometheus Pushgateway"
@@ -54,43 +66,39 @@ class Reporter:
 
     def __init__(self, args: argparse.Namespace) -> None:
         self._error: Optional[Exception] = None
-        if args.statsd_address:
-            self.statsd: Optional[stats.StatsDBackend] = stats.StatsDBackend(
-                args.statsd_address, args.statsd_prefix, tags=stats.get_env_tags()
+        self.registry = CollectorRegistry()
+        self.prometheus_push = args.prometheus_url is not None
+        self.args = args
+        self.gauges: dict[str, Gauge] = {}
+
+    def get_gauge(self, kind: str, kind_help: str, labels: list[str]) -> Gauge:
+        if kind not in self.gauges:
+            self.gauges[kind] = Gauge(
+                prometheus.build_metric_name(f"database_{kind}"),
+                kind_help,
+                labels,
+                registry=self.registry,
             )
-        else:
-            self.statsd = None
-
-        if args.prometheus_url:
-            self.prometheus: Optional[PushgatewayGroupPublisher] = PushgatewayGroupPublisher(
-                args.prometheus_url,
-                "db_counts",
-                instance=args.prometheus_instance,
-                labels=stats.get_env_tags(),
-            )
-        else:
-            self.prometheus = None
+        return self.gauges[kind]
 
     def do_report(
-        self, metric: List[str], value: int, kind: str, tags: Optional[Dict[str, str]] = None
+        self, metric: list[str], value: int, kind: str, kind_help: str, tags: dict[str, str]
     ) -> None:
         LOG.debug("%s.%s -> %d", kind, ".".join(metric), value)
-        if value > 0:  # Don't export 0 values. We can always set null=0 in grafana...
-            if self.statsd is not None:
-                if stats.USE_TAGS and tags is not None:
-                    self.statsd.gauge([kind], value, tags=tags)
-                else:
-                    self.statsd.gauge([kind] + metric, value)
-            if self.prometheus is not None:
-                self.prometheus.add("database_table_" + kind, value, metric_labels=tags)
+        gauge = self.get_gauge(kind, kind_help, list(tags.keys()))
+        gauge.labels(**tags).set(value)
 
     def commit(self) -> None:
-        if self.prometheus is not None:
-            self.prometheus.commit()
+        if self.prometheus_push:
+            push_to_gateway(self.args.prometheus_url, job="db_counts", registry=self.registry)
+        else:
+            port = int(os.environ.get("C2C_PROMETHEUS_PORT", "9090"))
+            app = make_wsgi_app(self.registry)
+            with make_server("", port, app) as httpd:
+                LOG.info("Waiting that Prometheus get the metrics served on port %s...", port)
+                httpd.handle_request()
 
-    def error(self, metric: List[str], error_: Exception) -> None:
-        if self.statsd is not None:
-            self.statsd.counter(["error"] + metric, 1)
+    def error(self, metric: list[str], error_: Exception) -> None:
         if self._error is None:
             self._error = error_
 
@@ -100,7 +108,7 @@ class Reporter:
 
 
 def do_table(
-    session: sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session],
+    session: scoped_session,
     schema: str,
     table: str,
     reporter: Reporter,
@@ -114,7 +122,7 @@ def do_table(
 def _do_indexes(
     reporter: Reporter,
     schema: str,
-    session: sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session],
+    session: scoped_session,
     table: str,
 ) -> None:
     for index_name, size_main, size_fsm, number_of_scans, tuples_read, tuples_fetched in session.execute(
@@ -146,14 +154,16 @@ def _do_indexes(
             reporter.do_report(
                 [schema, table, index_name, fork],
                 value,
-                kind="index_size",
+                kind="table_index_size",
+                kind_help="Size of the index",
                 tags={"schema": schema, "table": table, "index": index_name, "fork": fork},
             )
         for action, value in (("scan", number_of_scans), ("read", tuples_read), ("fetch", tuples_fetched)):
             reporter.do_report(
                 [schema, table, index_name, action],
                 value,
-                kind="index_usage",
+                kind="table_index_usage",
+                kind_help="Usage of the index",
                 tags={"schema": schema, "table": table, "index": index_name, "action": action},
             )
 
@@ -161,7 +171,7 @@ def _do_indexes(
 def _do_table_size(
     reporter: Reporter,
     schema: str,
-    session: sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session],
+    session: scoped_session,
     table: str,
 ) -> None:
     result = session.execute(
@@ -178,35 +188,48 @@ def _do_table_size(
     assert result is not None
     size: int
     (size,) = result
-    reporter.do_report([schema, table], size, kind="size", tags={"schema": schema, "table": table})
+    reporter.do_report(
+        [schema, table],
+        size,
+        kind="table_size",
+        kind_help="Size of the table",
+        tags={"schema": schema, "table": table},
+    )
 
 
 def _do_table_count(
     reporter: Reporter,
     schema: str,
-    session: sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session],
+    session: scoped_session,
     table: str,
 ) -> None:
     # We request and estimation of the count as a real count is very slow on big tables
-    # and seems to cause replicatin lags. This estimate is updated on ANALYZE and VACUUM.
+    # and seems to cause replicating lags. This estimate is updated on ANALYZE and VACUUM.
     result = session.execute(
-        sqlalchemy.text(  # nosec
-            "SELECT reltuples::bigint AS count FROM pg_class "
-            f"WHERE oid = '{sqlalchemy.sql.quoted_name(schema, True)}."
-            f"{sqlalchemy.sql.quoted_name(table, True)}'::regclass;"
-        )
+        sqlalchemy.text(
+            "SELECT reltuples FROM pg_class where "
+            "oid=(quote_ident(:schema) || '.' || quote_ident(:table))::regclass;"
+        ),
+        params={"schema": schema, "table": table},
     ).fetchone()
     assert result is not None
     (count,) = result
-    reporter.do_report([schema, table], count, kind="count", tags={"schema": schema, "table": table})
+    reporter.do_report(
+        [schema, table],
+        count,
+        kind="table_count",
+        kind_help="The number of row in the table",
+        tags={"schema": schema, "table": table},
+    )
 
 
-def do_extra(
-    session: sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session], extra: str, reporter: Reporter
-) -> None:
+def do_extra(session: scoped_session, sql: str, kind: str, gauge_help: str, reporter: Reporter) -> None:
     """Do an extra report."""
-    for metric, count in session.execute(sqlalchemy.text(extra)):
-        reporter.do_report(str(metric).split("."), count, kind="count", tags={"metric": metric})
+
+    for metric, count in session.execute(sqlalchemy.text(sql)):
+        reporter.do_report(
+            str(metric).split("."), count, kind=kind, kind_help=gauge_help, tags={"metric": metric}
+        )
 
 
 def _do_dtats_db(args: argparse.Namespace) -> None:
@@ -241,10 +264,19 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
         for pos, extra in enumerate(args.extra):
             LOG.info("Process extra %s.", extra)
             try:
-                do_extra(session, extra, reporter)
+                do_extra(session, extra, "extra", "Extra metric", reporter)
             except Exception as e:  # pylint: disable=broad-except
                 LOG.exception("Process extra %s error.", extra)
                 reporter.error(["extra", str(pos + 1)], e)
+    if args.extra_gauge:
+        for pos, extra in enumerate(args.extra_gauge):
+            sql, gauge, gauge_help = extra
+            LOG.info("Process extra %s.", extra)
+            try:
+                do_extra(session, sql, gauge, gauge_help, reporter)
+            except Exception as e:  # pylint: disable=broad-except
+                LOG.exception("Process extra %s error.", extra)
+                reporter.error(["extra", str(len(args.extra) + pos + 1)], e)
 
     reporter.commit()
     transaction.abort()
@@ -255,12 +287,12 @@ def main() -> None:
     """Run the command."""
     success = False
     args = _parse_args()
-    c2cwsgiutils.setup_process.bootstrap_application_from_options(args)
+    c2cwsgiutils.setup_process.init(args.config_uri)
     for _ in range(int(os.environ.get("C2CWSGIUTILS_STATS_DB_TRYNUMBER", 10))):
         try:
             _do_dtats_db(args)
             success = True
-            continue
+            break
         except:  # pylint: disable=bare-except
             LOG.exception("Exception during run")
         time.sleep(float(os.environ.get("C2CWSGIUTILS_STATS_DB_SLEEP", 1)))

c2cwsgiutils/sentry.py

@@ -2,7 +2,8 @@ import contextlib
 import logging
 import os
 import warnings
-from typing import Any, Callable, Generator, MutableMapping, Optional
+from collections.abc import Generator, MutableMapping
+from typing import Any, Callable, Optional
 
 import pyramid.config
 import sentry_sdk

c2cwsgiutils/setup_process.py

@@ -7,7 +7,7 @@ Must be imported at the very beginning of the process's life, before any other m
 
 import argparse
 import warnings
-from typing import Any, Callable, Dict, Optional, TypedDict, cast
+from typing import Any, Callable, Optional, TypedDict, cast
 
 import pyramid.config
 import pyramid.registry
@@ -16,7 +16,7 @@ import pyramid.router
 from pyramid.paster import bootstrap
 from pyramid.scripts.common import get_config_loader, parse_vars
 
-from c2cwsgiutils import broadcast, coverage_setup, redis_stats, sentry, sql_profiler, stats
+from c2cwsgiutils import broadcast, coverage_setup, redis_stats, sentry, sql_profiler
 
 
 def fill_arguments(
@@ -51,7 +51,6 @@ def init(config_file: str = "c2c:///app/production.ini") -> None:
     coverage_setup.includeme()
     sentry.includeme(config)
     broadcast.includeme(config)
-    stats.init_backends(settings)
     redis_stats.includeme(config)
     sql_profiler.includeme(config)
 
@@ -63,18 +62,15 @@ def init_logging(config_file: str = "c2c:///app/production.ini") -> None:
     loader.setup_logging(None)
 
 
-PyramidEnv = TypedDict(
-    "PyramidEnv",
-    {
-        "root": Any,
-        "closer": Callable[..., Any],
-        "registry": pyramid.registry.Registry,
-        "request": pyramid.request.Request,
-        "root_factory": object,
-        "app": Callable[[Dict[str, str], Any], Any],
-    },
-    total=True,
-)
+class PyramidEnv(TypedDict, total=True):
+    """The return type of the bootstrap functions."""
+
+    root: Any
+    closer: Callable[..., Any]
+    registry: pyramid.registry.Registry
+    request: pyramid.request.Request
+    root_factory: object
+    app: Callable[[dict[str, str], Any], Any]
 
 
 def bootstrap_application_from_options(options: argparse.Namespace) -> PyramidEnv:
@@ -91,7 +87,7 @@ def bootstrap_application_from_options(options: argparse.Namespace) -> PyramidEn
 
 def bootstrap_application(
     config_uri: str = "c2c:///app/production.ini",
-    options: Optional[Dict[str, Any]] = None,
+    options: Optional[dict[str, Any]] = None,
 ) -> PyramidEnv:
     """
     Initialize all the application.

c2cwsgiutils/sql_profiler/_impl.py

@@ -5,8 +5,9 @@ That runs an "EXPLAIN ANALYZE" on every SELECT query going through SQLAlchemy.
 """
 import logging
 import re
+from collections.abc import Mapping
 from threading import Lock
-from typing import Any, Mapping, Set
+from typing import Any
 
 import pyramid.request
 import sqlalchemy.engine
@@ -22,7 +23,7 @@ class _Repository:
     def __init__(self) -> None:
         super().__init__()
         self._lock = Lock()
-        self._repo: Set[str] = set()
+        self._repo: set[str] = set()
 
     def profile(
         self,

c2cwsgiutils/sqlalchemylogger/_models.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Union
+from typing import Any, Union
 
 from sqlalchemy import Column
 from sqlalchemy.orm import declarative_base
@@ -8,7 +8,7 @@ from sqlalchemy.types import DateTime, Integer, String
 Base = declarative_base()
 
 
-def create_log_class(tablename: str = "logs", tableargs: Union[str, Dict[str, str]] = "") -> Any:
+def create_log_class(tablename: str = "logs", tableargs: Union[str, dict[str, str]] = "") -> Any:
     """Get the sqlalchemy lgo class."""
 
     class Log(Base):  # type: ignore

c2cwsgiutils/sqlalchemylogger/handlers.py

@@ -3,7 +3,7 @@ import queue
 import threading
 import time
 import traceback
-from typing import Any, Dict, List
+from typing import Any
 
 import sqlalchemy
 from sqlalchemy import create_engine
@@ -25,7 +25,7 @@ class SQLAlchemyHandler(logging.Handler):
 
     def __init__(
         self,
-        sqlalchemy_url: Dict[str, str],
+        sqlalchemy_url: dict[str, str],
         does_not_contain_expression: str = "",
         contains_expression: str = "",
     ) -> None:
@@ -54,7 +54,7 @@ class SQLAlchemyHandler(logging.Handler):
         LOG.debug("%s: starting processor thread", __name__)
         while True:
             logs = []
-            time_since_last = time.monotonic()
+            time_since_last = time.perf_counter()
             while True:
                 with self.condition:
                     self.condition.wait(timeout=self.MAX_TIMEOUT)
@@ -66,17 +66,17 @@ class SQLAlchemyHandler(logging.Handler):
                     # by writing chunks of self.MAX_NB_LOGS size,
                     # but also do not wait forever before writing stuff (self.MAX_TIMOUT)
                     if (len(logs) >= self.MAX_NB_LOGS) or (
-                        time.monotonic() >= (time_since_last + self.MAX_TIMEOUT)
+                        time.perf_counter() >= (time_since_last + self.MAX_TIMEOUT)
                     ):
                         self._write_logs(logs)
                         break
         LOG.debug("%s: stopping processor thread", __name__)
 
-    def _write_logs(self, logs: List[Any]) -> None:
+    def _write_logs(self, logs: list[Any]) -> None:
         try:
             self.session.bulk_save_objects(logs)
             self.session.commit()
-        except (SQLAlchemyError):
+        except SQLAlchemyError:
             try:
                 self.create_db()
                 self.session.rollback()

c2cwsgiutils/stats_pyramid/__init__.py

@@ -1,40 +1,36 @@
 """Generate statsd metrics for pyramid and SQLAlchemy events."""
+
 import warnings
 
 import pyramid.config
 import pyramid.request
 
-from c2cwsgiutils import stats
+from c2cwsgiutils.stats_pyramid import _pyramid_spy
 
 
 def init(config: pyramid.config.Configurator) -> None:
     """Initialize the whole stats module, for backward compatibility."""
+
     warnings.warn("init function is deprecated; use includeme instead")
     includeme(config)
 
 
 def includeme(config: pyramid.config.Configurator) -> None:
     """
-    Initialize the whole stats module.
+    Initialize the whole stats pyramid module.
 
     Arguments:
 
         config: The Pyramid config
     """
-    stats.init_backends(config.get_settings())
-    if stats.BACKENDS:  # pragma: nocover
-        if "memory" in stats.BACKENDS:  # pragma: nocover
-            from . import _views
 
-            _views.init(config)
-        from . import _pyramid_spy
-
-        _pyramid_spy.init(config)
-        init_db_spy()
+    _pyramid_spy.init(config)
+    init_db_spy()
 
 
 def init_db_spy() -> None:
     """Initialize the database spy."""
+
     from . import _db_spy
 
     _db_spy.init()

c2cwsgiutils/stats_pyramid/_db_spy.py

@@ -1,15 +1,24 @@
 import logging
 import re
-from typing import Any, Callable, Dict, Optional
+import time
+from typing import Any, Callable
 
+import prometheus_client
 import sqlalchemy.event
 from sqlalchemy.engine import Connection, Engine
 from sqlalchemy.orm import Session
 
-from c2cwsgiutils import stats
+from c2cwsgiutils import prometheus
 
 LOG = logging.getLogger(__name__)
 
+_PROMETHEUS_DB_SUMMARY = prometheus_client.Summary(
+    prometheus.build_metric_name("database"),
+    "Database requests",
+    ["what"],
+    unit="seconds",
+)
+
 
 def _jump_string(content: str, pos: int) -> int:
     quote_char = content[pos]
@@ -58,17 +67,11 @@ def _simplify_sql(sql: str) -> str:
 
 
 def _create_sqlalchemy_timer_cb(what: str) -> Callable[..., Any]:
-    if stats.USE_TAGS and what != "commit":
-        key = ["sql"]
-        tags: Optional[Dict[str, str]] = {"query": what}
-    else:
-        key = ["sql", what]
-        tags = None
-    measure = stats.timer(key, tags)
+    start = time.perf_counter()
 
     def after(*_args: Any, **_kwargs: Any) -> None:
-        duration = measure.stop()
-        LOG.debug("Execute statement '%s' in %d.", what, duration)
+        _PROMETHEUS_DB_SUMMARY.labels({"query": what}).observe(time.perf_counter() - start)
+        LOG.debug("Execute statement '%s' in %d.", what, time.perf_counter() - start)
 
     return after
 

c2cwsgiutils/stats_pyramid/_pyramid_spy.py

@@ -1,11 +1,26 @@
-from typing import Any, Callable, Dict, Optional
+import time
+from typing import Callable, Optional
 
+import prometheus_client
 import pyramid.config
 import pyramid.events
 import pyramid.request
 from pyramid.httpexceptions import HTTPException
 
-from c2cwsgiutils import stats
+from c2cwsgiutils import prometheus
+
+_PROMETHEUS_PYRAMID_ROUTES_SUMMARY = prometheus_client.Summary(
+    prometheus.build_metric_name("pyramid_routes"),
+    "Pyramid routes",
+    ["method", "route", "status", "group"],
+    unit="seconds",
+)
+_PROMETHEUS_PYRAMID_VIEWS_SUMMARY = prometheus_client.Summary(
+    prometheus.build_metric_name("pyramid_render"),
+    "Pyramid render",
+    ["method", "route", "status", "group"],
+    unit="seconds",
+)
 
 
 def _add_server_metric(
@@ -28,14 +43,16 @@ def _add_server_metric(
 
 
 def _create_finished_cb(
-    kind: str, measure: stats.Timer
+    kind: str, measure: prometheus_client.Summary
 ) -> Callable[[pyramid.request.Request], None]:  # pragma: nocover
+    start = time.process_time()
+
     def finished_cb(request: pyramid.request.Request) -> None:
         if request.exception is not None:
             if isinstance(request.exception, HTTPException):
                 status = request.exception.code
             else:
-                status = 500
+                status = 599
         else:
             status = request.response.status_code
         if request.matched_route is None:
@@ -44,35 +61,24 @@ def _create_finished_cb(
             name = request.matched_route.name
             if kind == "route":
                 _add_server_metric(request, "route", description=name)
-        if stats.USE_TAGS:
-            key = [kind]
-            tags: Optional[Dict[str, Any]] = {
-                "method": request.method,
-                "route": name,
-                "status": status,
-                "group": status // 100,
-            }
-        else:
-            key = [kind, request.method, name, status]
-            tags = None
-        duration = measure.stop(key, tags)
-        _add_server_metric(request, kind, duration=duration)
+        measure.labels(
+            method=request.method, route=name, status=status, group=str(status // 100 * 100)
+        ).observe(time.process_time() - start)
+        _add_server_metric(request, kind, duration=time.process_time() - start)
 
     return finished_cb
 
 
 def _request_callback(event: pyramid.events.NewRequest) -> None:  # pragma: nocover
     """Finish the callback called when a new HTTP request is incoming."""
-    measure = stats.timer()
-    event.request.add_finished_callback(_create_finished_cb("route", measure))
+    event.request.add_finished_callback(_create_finished_cb("route", _PROMETHEUS_PYRAMID_ROUTES_SUMMARY))
 
 
 def _before_rendered_callback(event: pyramid.events.BeforeRender) -> None:  # pragma: nocover
     """Finish the callback called when the rendering is starting."""
     request = event.get("request", None)
     if request:
-        measure = stats.timer()
-        request.add_finished_callback(_create_finished_cb("render", measure))
+        request.add_finished_callback(_create_finished_cb("render", _PROMETHEUS_PYRAMID_VIEWS_SUMMARY))
 
 
 def init(config: pyramid.config.Configurator) -> None:  # pragma: nocover

c2cwsgiutils/templates/index.html.mako

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+    <link
+      rel="icon"
+      type="image/png"
+      sizes="32x32"
+      href="${request.static_url('c2cwsgiutils:static/favicon-32x32.png')}"
+      referrerpolicy="no-referrer"
+    />
+    <link
+      rel="icon"
+      type="image/png"
+      sizes="16x16"
+      href="${request.static_url('c2cwsgiutils:static/favicon-16x16.png')}"
+      referrerpolicy="no-referrer"
+    />
+    <link
+        rel="stylesheet"
+        href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.2/css/bootstrap.min.css"
+        integrity="sha512-b2QcS5SsA8tZodcDtGRELiGv5SaKSk1vDHDaQRda0htPYWZ6046lr3kJ5bAAQdpV2mmA/4v0wQF9MyU6/pDIAg=="
+        crossorigin="anonymous"
+        referrerpolicy="no-referrer"
+    />
+    <title>c2cwsgiutils tools</title>
+    <style>
+      body {
+        margin-top: 0.5rem;
+      }
+      button, p  {
+        margin-bottom: 0.5rem;
+      }
+    </style>
+  </head>
+  <body>
+    <script>
+      (() => {
+        'use strict'
+        if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
+          document.documentElement.setAttribute('data-bs-theme', 'dark');
+        }
+      })()
+    </script>
+    <div class="container-fluid">
+      ${ body | n }
+    </div>
+  </body>
+</html>