c2cwsgiutils 5.1.7.dev20230901073305__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl

c2cwsgiutils/metrics.py

@@ -1,110 +0,0 @@
-"""Used to publish metrics to Prometheus."""
-
-import re
-import socket
-import warnings
-from os import listdir
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-import pyramid.request
-import pyramid.response
-
-from c2cwsgiutils.debug.utils import dump_memory_maps
-
-
-class Provider:
-    """The provider interface."""
-
-    def __init__(self, name: str, help_: str, type_: str = "gauge", extend: bool = True):
-        self.name = name
-        self.help = help_
-        self.type = type_
-        self.extend = extend
-
-    def get_data(self) -> List[Tuple[Dict[str, str], Union[int, float]]]:
-        """Get empty response, should be defined in the specific provider."""
-        return []
-
-
-_PROVIDERS = []
-
-
-POD_NAME = socket.gethostname()
-SERVICE_NAME = re.match("^(.+)-[0-9a-f]+-[0-9a-z]+$", POD_NAME)
-
-
-def add_provider(provider: Provider) -> None:
-    """Add the provider."""
-    _PROVIDERS.append(provider)
-
-
-def _metrics() -> pyramid.response.Response:
-    result: List[str] = []
-
-    for provider in _PROVIDERS:
-        result += [
-            f"# HELP {provider.name} {provider.help}",
-            f"# TYPE {provider.name} {provider.type}",
-        ]
-        for attributes, value in provider.get_data():
-            attrib = {}
-            if provider.extend:
-                attrib["pod_name"] = POD_NAME
-                if SERVICE_NAME is not None:
-                    attrib["service_name"] = SERVICE_NAME.group(1)
-            attrib.update(attributes)
-            dbl_quote = '"'
-            printable_attribs = ",".join([f'{k}="{v.replace(dbl_quote, "_")}"' for k, v in attrib.items()])
-            result.append(f"{provider.name}{{{printable_attribs}}} {value}")
-
-    return "\n".join(result)
-
-
-def _view(request: pyramid.request.Request) -> pyramid.response.Response:
-    request.response.text = _metrics()
-    return request.response
-
-
-NUMBER_RE = re.compile(r"^[0-9]+$")
-
-
-class MemoryMapProvider(Provider):
-    """The Linux memory map provider."""
-
-    def __init__(self, memory_type: str = "pss", pids: Optional[List[str]] = None):
-        """
-        Initialize.
-
-        Arguments:
-
-            memory_type: can be rss, pss or size
-            pids: the list of pids or none
-        """
-        super().__init__(
-            f"pod_process_smap_{memory_type}_kb",
-            f"Container smap used {memory_type.capitalize()}",
-        )
-        self.memory_type = memory_type
-        self.pids = pids
-
-    def get_data(self) -> List[Tuple[Dict[str, Any], Union[int, float]]]:
-        """Get empty response, should be defined in the specific provider."""
-        results: List[Tuple[Dict[str, Any], Union[int, float]]] = []
-        for pid in [p for p in listdir("/proc/") if NUMBER_RE.match(p)] if self.pids is None else self.pids:
-            results += [
-                ({"pid": pid, "name": e["name"]}, e[self.memory_type + "_kb"]) for e in dump_memory_maps(pid)
-            ]
-        return results
-
-
-def init(config: pyramid.config.Configurator) -> None:
-    """Initialize the metrics view, , for backward compatibility."""
-    warnings.warn("init function is deprecated; use includeme instead")
-    includeme(config)
-
-
-def includeme(config: pyramid.config.Configurator) -> None:
-    """Initialize the metrics view."""
-    config.add_route("c2c_metrics", r"/metrics", request_method="GET")
-    config.add_view(_view, route_name="c2c_metrics", http_cache=0)
-    add_provider(MemoryMapProvider())

c2cwsgiutils/scripts/check_es.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import datetime
-import logging
-import os
-import sys
-import time
-import uuid
-import warnings
-from typing import Any, List, Optional
-
-import requests.exceptions
-from dateutil import parser as dp
-
-import c2cwsgiutils.setup_process
-from c2cwsgiutils import stats
-
-
-def _ensure_slash(txt: Optional[str]) -> Optional[str]:
-    if txt is None:
-        return None
-    if txt.endswith("/"):
-        return txt
-    return txt + "/"
-
-
-LOGGER_NAME = "check_elasticsearch"
-LOG_TIMEOUT = int(os.environ["LOG_TIMEOUT"])
-LOG = logging.getLogger(LOGGER_NAME)
-ES_URL = _ensure_slash(os.environ.get("ES_URL"))
-ES_INDEXES = os.environ.get("ES_INDEXES")
-ES_AUTH = os.environ.get("ES_AUTH")
-ES_FILTERS = os.environ.get("ES_FILTERS", "")
-
-SEARCH_HEADERS = {"Content-Type": "application/json;charset=UTF-8", "Accept": "application/json"}
-if ES_AUTH is not None:
-    SEARCH_HEADERS["Authorization"] = ES_AUTH
-SEARCH_URL = f"{ES_URL}{ES_INDEXES}/_search"
-
-
-def _max_timestamp() -> datetime.datetime:
-    must: List[Any] = []
-    query = {
-        "aggs": {"max_timestamp": {"max": {"field": "@timestamp"}}},
-        "query": {"bool": {"must": must}},
-    }
-    if ES_FILTERS != "":
-        for filter_ in ES_FILTERS.split(","):
-            name, value = filter_.split("=")
-            must.append({"term": {name: value}})
-    else:
-        del query["query"]
-
-    r = requests.post(SEARCH_URL, json=query, headers=SEARCH_HEADERS)
-    r.raise_for_status()
-    json = r.json()
-    return dp.parse(json["aggregations"]["max_timestamp"]["value_as_string"])
-
-
-def _check_roundtrip() -> None:
-    check_uuid = str(uuid.uuid4())
-
-    # emit the log we are going to look for
-    logger_name = LOGGER_NAME + "." + check_uuid
-    logger = logging.getLogger(logger_name)
-    logger.setLevel(logging.INFO)
-    logger.info("Test roundtrip")
-
-    query = {"query": {"match_phrase": {"log.logger": logger_name}}}
-    start = time.monotonic()
-    while time.monotonic() < start + LOG_TIMEOUT:
-        exception = None
-        for _ in range(int(os.environ.get("C2CWSGIUTILS_CHECK_ES_TRYNUMBER", 10))):
-            try:
-                r = requests.post(SEARCH_URL, json=query, headers=SEARCH_HEADERS)
-                exception = None
-            except requests.exceptions.RequestException as e:
-                logger.exception("Error on querying Elasticsearch")
-                exception = e
-            if r.ok:
-                continue
-            time.sleep(float(os.environ.get("C2CWSGIUTILS_CHECK_ES_SLEEP", 1)))
-        if exception is not None:
-            raise exception
-        r.raise_for_status()
-        json = r.json()
-        found = json["hits"]["total"]
-        if isinstance(found, dict):
-            found = found["value"]
-        if found > 0:
-            LOG.info("Found the test log line.")
-            stats.set_gauge(["roundtrip"], time.monotonic() - start)
-            return
-        else:
-            LOG.info("Didn't find the test log line. Wait 1s...")
-            time.sleep(1)
-    LOG.warning("Timeout waiting for the test log line")
-    stats.set_gauge(["roundtrip"], LOG_TIMEOUT * 2)
-
-
-def deprecated() -> None:
-    """Run the command and print a deprecated notice."""
-    warnings.warn("c2cwsgiutils_check_es.py is deprecated; use c2cwsgiutils-check-es instead")
-    return main()
-
-
-def main() -> None:
-    """Run the command."""
-    try:
-        argparser = argparse.ArgumentParser(description="Check logs on Elasticsearch")
-        c2cwsgiutils.setup_process.fill_arguments(argparser)
-        args = argparser.parse_args()
-        c2cwsgiutils.setup_process.bootstrap_application_from_options(args)
-
-        with stats.outcome_timer_context(["get_max_timestamp"]):
-            max_ts = _max_timestamp()
-        now = datetime.datetime.now(max_ts.tzinfo)
-        age = round((now - max_ts).total_seconds())
-        LOG.info("Last log age: %ss", age)
-        stats.set_gauge(["max_age"], age)
-
-        if "LOG_TIMEOUT" in os.environ:
-            _check_roundtrip()
-    except:  # pylint: disable=bare-except
-        LOG.exception("Exception during run")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

c2cwsgiutils/scripts/coverage_report.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-import logging
-import os
-import shutil
-import sys
-import warnings
-
-import coverage
-
-LOG = logging.getLogger(__name__)
-
-
-def deprecated() -> None:
-    """Run the command and print a deprecated notice."""
-    warnings.warn("c2cwsgiutils_coverage_report.py is deprecated; use c2cwsgiutils-coverage-report instead")
-    return main()
-
-
-def main() -> None:
-    """Run the command."""
-    sources = sys.argv[1:]
-    report_dir = "/reports/coverage/api"
-    dest_dir = "/tmp/coverage/api"  # nosec
-    shutil.rmtree(dest_dir, ignore_errors=True)
-    shutil.copytree(report_dir, dest_dir)
-    cov = coverage.Coverage(
-        data_file=os.path.join(dest_dir, "coverage"), data_suffix=True, source=sources or None, branch=True
-    )
-    cov.combine([dest_dir], strict=True)
-    cov.html_report(directory=dest_dir, ignore_errors=True)
-    cov.xml_report(outfile=os.path.join(dest_dir, "coverage.xml"), ignore_errors=True)
-    cov.report(ignore_errors=True)
-
-
-if __name__ == "__main__":
-    main()

c2cwsgiutils/stats.py

@@ -1,355 +0,0 @@
-"""Generate statsd metrics."""
-
-import contextlib
-import logging
-import os
-import re
-import socket
-import threading
-import time
-from abc import ABCMeta, abstractmethod
-from typing import (  # noqa  # pylint: disable=unused-import
-    Any,
-    Callable,
-    Dict,
-    Generator,
-    List,
-    Mapping,
-    MutableMapping,
-    Optional,
-    Sequence,
-    Tuple,
-)
-
-import pyramid.request
-
-from c2cwsgiutils import config_utils
-
-LOG = logging.getLogger(__name__)
-USE_TAGS_ENV = "STATSD_USE_TAGS"
-TAG_PREFIX_ENV = "STATSD_TAG_"
-USE_TAGS = config_utils.config_bool(os.environ.get(USE_TAGS_ENV, "0"))
-TagType = Optional[Mapping[str, Any]]
-
-
-class _BaseBackend(metaclass=ABCMeta):
-    @abstractmethod
-    def timer(self, key: Sequence[Any], duration: float, tags: TagType = None) -> None:
-        pass
-
-    @abstractmethod
-    def gauge(self, key: Sequence[Any], value: float, tags: TagType = None) -> None:
-        pass
-
-    @abstractmethod
-    def counter(self, key: Sequence[Any], increment: int, tags: TagType = None) -> None:
-        pass
-
-
-BACKENDS: MutableMapping[str, _BaseBackend] = {}
-
-
-class Timer:
-    """Allow to measure the duration of some activity."""
-
-    def __init__(self, key: Optional[Sequence[Any]], tags: TagType) -> None:
-        self._key = key
-        self._tags = tags
-        self._start = time.monotonic()
-
-    def stop(self, key_final: Optional[Sequence[Any]] = None, tags_final: TagType = None) -> float:
-        duration = time.monotonic() - self._start
-        if key_final is not None:
-            self._key = key_final
-        if tags_final is not None:
-            self._tags = tags_final
-        assert self._key is not None
-        for backend in BACKENDS.values():
-            backend.timer(self._key, duration, self._tags)
-        return duration
-
-
-@contextlib.contextmanager
-def timer_context(key: Sequence[Any], tags: TagType = None) -> Generator[None, None, None]:
-    """
-    Add a duration measurement to the stats using the duration the context took to run.
-
-    Arguments:
-
-        key: The path of the key, given as a list.
-        tags: Some tags to attach to the metric.
-    """
-    measure = timer(key, tags)
-    yield
-    measure.stop()
-
-
-@contextlib.contextmanager
-def outcome_timer_context(key: List[Any], tags: TagType = None) -> Generator[None, None, None]:
-    """
-    Add a duration measurement to the stats using the duration the context took to run.
-
-    The given key is prepended with 'success' or 'failure' according to the context's outcome.
-
-    Arguments:
-
-        key: The path of the key, given as a list.
-        tags: Some tags to attach to the metric.
-    """
-    measure = timer()
-    try:
-        yield
-        if USE_TAGS:
-            opt_tags = dict(tags) if tags is not None else {}
-            opt_tags["success"] = 1
-            measure.stop(key, opt_tags)
-        else:
-            measure.stop(key + ["success"], tags)
-    except Exception:
-        if USE_TAGS:
-            opt_tags = dict(tags) if tags is not None else {}
-            opt_tags["success"] = 0
-            measure.stop(key, opt_tags)
-        else:
-            measure.stop(key + ["failure"], tags)
-        raise
-
-
-def timer(key: Optional[Sequence[Any]] = None, tags: TagType = None) -> Timer:
-    """
-    Create a timer for the given key.
-
-    The key can be omitted, but then need to be specified when stop is called.
-
-    Arguments:
-
-        key: The path of the key, given as a list.
-        tags: Some tags to attach to the metric.
-
-    Returns: An instance of _Timer
-    """
-    assert key is None or isinstance(key, list)
-    return Timer(key, tags)
-
-
-def set_gauge(key: Sequence[Any], value: float, tags: TagType = None) -> None:
-    """
-    Set a gauge value.
-
-    Arguments:
-
-        key: The path of the key, given as a list.
-        value: The new value of the gauge
-        tags: Some tags to attach to the metric.
-    """
-    for backend in BACKENDS.values():
-        backend.gauge(key, value, tags)
-
-
-def increment_counter(key: Sequence[Any], increment: int = 1, tags: TagType = None) -> None:
-    """
-    Increment a counter value.
-
-    Arguments:
-
-        key: The path of the key, given as a list.
-        increment: The increment
-        tags: Some tags to attach to the metric.
-    """
-    for backend in BACKENDS.values():
-        backend.counter(key, increment, tags)
-
-
-class MemoryBackend(_BaseBackend):
-    """Store stats in the memors."""
-
-    def __init__(self) -> None:
-        self._timers: MutableMapping[str, Tuple[int, float, float, float]] = {}
-        self._gauges: MutableMapping[str, float] = {}
-        self._counters: MutableMapping[str, int] = {}
-        self._stats_lock = threading.Lock()
-        LOG.info("Starting a MemoryBackend for stats")
-
-    @staticmethod
-    def _key_entry(key: str) -> str:
-        return str(key).replace("/", "_")
-
-    @staticmethod
-    def _key(key: Sequence[Any], tags: TagType) -> str:
-        result = "/".join(MemoryBackend._key_entry(v) for v in key)
-        result += _format_tags(
-            tags,
-            prefix="/",
-            tag_sep="/",
-            kv_sep="=",
-            key_formatter=MemoryBackend._key_entry,
-            value_formatter=MemoryBackend._key_entry,
-        )
-        return result
-
-    def timer(self, key: Sequence[Any], duration: float, tags: TagType = None) -> None:
-        """Add a duration measurement to the stats."""
-        the_key = self._key(key, tags)
-        with self._stats_lock:
-            cur = self._timers.get(the_key, None)
-            if cur is None:
-                self._timers[the_key] = (1, duration, duration, duration)
-            else:
-                self._timers[the_key] = (
-                    cur[0] + 1,
-                    cur[1] + duration,
-                    min(cur[2], duration),
-                    max(cur[3], duration),
-                )
-
-    def gauge(self, key: Sequence[Any], value: float, tags: TagType = None) -> None:
-        self._gauges[self._key(key, tags)] = value
-
-    def counter(self, key: Sequence[Any], increment: int, tags: TagType = None) -> None:
-        the_key = self._key(key, tags)
-        with self._stats_lock:
-            self._counters[the_key] = self._counters.get(the_key, 0) + increment
-
-    def get_stats(self, request: pyramid.request.Request) -> Mapping[str, Any]:
-        reset = request.params.get("reset", "0") == "1"
-        with self._stats_lock:
-            timers = {}
-            for key, value in self._timers.items():
-                timers[key] = {
-                    "nb": value[0],
-                    "avg_ms": int(round((value[1] / value[0]) * 1000.0)),
-                    "min_ms": int(round(value[2] * 1000.0)),
-                    "max_ms": int(round(value[3] * 1000.0)),
-                }
-            gauges = dict(self._gauges)
-            counters = dict(self._counters)
-
-            if reset:
-                self._timers.clear()
-                self._gauges.clear()
-                self._counters.clear()
-        return {"timers": timers, "gauges": gauges, "counters": counters}
-
-
-# https://github.com/prometheus/statsd_exporter/blob/master/mapper.go#L29
-INVALID_KEY_CHARS = re.compile(r"[^a-zA-Z0-9_]")
-INVALID_TAG_VALUE_CHARS = re.compile(r"[,#|]")
-
-
-class StatsDBackend(_BaseBackend):  # pragma: nocover
-    """Abstraction of the statd backend to sent some metrics."""
-
-    def __init__(self, address: str, prefix: str, tags: Optional[Dict[str, str]] = None) -> None:
-        self._prefix = prefix
-        self._tags = tags
-        if self._prefix != "" and not self._prefix.endswith("."):
-            self._prefix += "."
-
-        host, port = address.rsplit(":")
-        host = host.strip("[]")
-        addrinfo = socket.getaddrinfo(host, port, 0, 0, socket.IPPROTO_UDP)
-        family, socktype, protocol, _canonname, sock_addr = addrinfo[0]
-        LOG.info("Starting a StatsDBackend for %s stats: %s -> %s", prefix, address, repr(sock_addr))
-
-        self._socket = socket.socket(family, socktype, protocol)
-        self._socket.setblocking(False)
-        self._socket.connect(sock_addr)
-
-    @staticmethod
-    def _key_entry(key_entry: Any) -> str:
-        return INVALID_KEY_CHARS.sub("_", str(key_entry))
-
-    @staticmethod
-    def _tag_value(tag_value: Any) -> str:
-        return INVALID_TAG_VALUE_CHARS.sub("_", str(tag_value))
-
-    def _key(self, key: Sequence[Any]) -> str:
-        return (self._prefix + ".".join(map(StatsDBackend._key_entry, key)))[:450]
-
-    def _merge_tags(self, tags: TagType) -> TagType:
-        if tags is None:
-            return self._tags
-        elif self._tags is None:
-            return tags
-        else:
-            tmp = dict(self._tags)
-            tmp.update(tags)
-            return tmp
-
-    def _send(self, message: str, tags: TagType) -> None:
-        tags = self._merge_tags(tags)
-        message += _format_tags(
-            tags,
-            prefix="|#",
-            tag_sep=",",
-            kv_sep=":",
-            key_formatter=StatsDBackend._key_entry,
-            value_formatter=StatsDBackend._tag_value,
-        )
-        try:
-            self._socket.send(message.encode("utf-8"))
-        except Exception:  # nosec  # pylint: disable=broad-except
-            pass  # Ignore errors (must survive if stats cannot be sent)
-
-    def timer(self, key: Sequence[Any], duration: float, tags: TagType = None) -> None:
-        the_key = self._key(key)
-        ms_duration = int(round(duration * 1000.0))
-        ms_duration = max(ms_duration, 1)  # collectd would ignore events with zero durations
-        message = f"{the_key}:{ms_duration}|ms"
-        self._send(message, tags)
-
-    def gauge(self, key: Sequence[Any], value: float, tags: TagType = None) -> None:
-        the_key = self._key(key)
-        message = f"{the_key}:{value}|g"
-        self._send(message, tags)
-
-    def counter(self, key: Sequence[Any], increment: int, tags: TagType = None) -> None:
-        the_key = self._key(key)
-        message = f"{the_key}:{increment}|c"
-        self._send(message, tags)
-
-
-def init_backends(settings: Optional[Mapping[str, str]] = None) -> None:
-    """
-    Initialize the backends according to the configuration.
-
-    Arguments:
-
-        settings: The Pyramid config
-    """
-    if config_utils.env_or_settings(settings, "STATS_VIEW", "c2c.stats_view", False):  # pragma: nocover
-        BACKENDS["memory"] = MemoryBackend()
-
-    statsd_address = config_utils.env_or_settings(settings, "STATSD_ADDRESS", "c2c.statsd_address", None)
-    if statsd_address is not None:  # pragma: nocover
-        statsd_prefix = config_utils.env_or_settings(settings, "STATSD_PREFIX", "c2c.statsd_prefix", "")
-        statsd_tags = get_env_tags()
-        try:
-            BACKENDS["statsd"] = StatsDBackend(statsd_address, statsd_prefix, statsd_tags)
-        except Exception:  # pylint: disable=broad-except
-            LOG.error("Failed configuring the statsd backend. Will continue without it.", exc_info=True)
-
-
-def _format_tags(
-    tags: Optional[Mapping[str, Any]],
-    prefix: str,
-    tag_sep: str,
-    kv_sep: str,
-    key_formatter: Callable[[str], str],
-    value_formatter: Callable[[str], str],
-) -> str:
-    if tags:
-        return prefix + tag_sep.join(
-            key_formatter(k) + kv_sep + value_formatter(v) for k, v in sorted(tags.items())
-        )
-    else:
-        return ""
-
-
-def get_env_tags() -> Dict[str, str]:
-    """Get the tag from the environment variable."""
-    tags = {}
-    for name, value in os.environ.items():
-        if name.startswith(TAG_PREFIX_ENV):
-            tags[name[len(TAG_PREFIX_ENV) :].lower()] = value
-    return tags

c2cwsgiutils/stats_pyramid/_views.py

@@ -1,16 +0,0 @@
-from typing import cast
-
-import pyramid.config
-
-from c2cwsgiutils import config_utils, stats
-
-
-def init(config: pyramid.config.Configurator) -> None:
-    """Initialize the statistic view."""
-    config.add_route(
-        "c2c_read_stats_json", config_utils.get_base_path(config) + r"/stats.json", request_method="GET"
-    )
-    memory_backend = cast(stats.MemoryBackend, stats.BACKENDS["memory"])
-    config.add_view(
-        memory_backend.get_stats, route_name="c2c_read_stats_json", renderer="fast_json", http_cache=0
-    )

c2cwsgiutils-5.1.7.dev20230901073305.data/scripts/c2cwsgiutils-run

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-echo "Not for production usage!"
-
-_term() {
-  echo "Caught SIGTERM signal!"
-  kill -TERM "$child" 2> /dev/null
-  wait "$child"
-  exit 1
-}
-
-_int() {
-  echo "Caught SIGINT signal!"
-  kill -INT "$child" 2> /dev/null
-  wait "$child"
-  exit 1
-}
-
-trap _term SIGTERM
-trap _int SIGINT
-
-while true; do
-  /usr/local/bin/gunicorn --paste=/app/production.ini &
-  child=$!
-  wait "$child"
-  exit_status=$?
-  if [ $exit_status -eq 0 ]; then
-    exit 0
-  fi
-  echo "gunicorn exited with an error ($exit_status), restarting in 1s"
-  sleep 1
-done