qdb-prometheus-exporter 3.14.2__py3-none-any.whl

qdb_prometheus_exporter/collector.py

@@ -0,0 +1,141 @@
+import logging
+from typing import Generator
+
+import quasardb.stats as qdbst
+from prometheus_client.core import (
+    CounterMetricFamily,
+    GaugeMetricFamily,
+    InfoMetricFamily,
+)
+from prometheus_client.registry import Collector
+
+from .stats import fetch_qdb_stats
+import re
+from typing import Union
+
+
+class QdbStatsCollector(Collector):
+    def __init__(
+        self,
+        qdb_conn_args: dict,
+        filter_include: Union[list[str], None],
+        filter_exclude: Union[list[str], None],
+        logger: logging.Logger,
+    ):
+        self.qdb_conn_args = qdb_conn_args
+        self.filter_include = filter_include
+        self.filter_exclude = filter_exclude
+        self.logger = logger
+
+        self.qdbst_types_to_prometheus_types: dict[
+            qdbst.Type,
+            type[Union[GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily]],
+        ] = {
+            qdbst.Type.GAUGE: GaugeMetricFamily,
+            qdbst.Type.ACCUMULATOR: CounterMetricFamily,
+            qdbst.Type.LABEL: InfoMetricFamily,
+        }
+
+        self.suffix_regex = re.compile(r"(^|_)(total|count|ns)(?=_|$)")
+
+    def _conform_qdb_name_to_prometheus_name(self, name: str) -> str:
+        """
+        Converts QuasarDB statistic name to OpenMetrics/Prometheus compliant name.
+        """
+        # Prometheus will add suffixes based on metric type (_total for counters, _info for info metrics, no suffix for gauges), units are also added as suffixes for gauges
+        # e.g. metric with name `async_pipelines.pulled.total_count` will be transformed by Prometheus client internally to `async_pipelines_pulled_total_count_total`
+        # we want to avoid redundant suffixes in the final metric name. this will result is slightly different names compared to native QuasarDB stats, but is more compliant with Prometheus conventions.
+
+        # 1. Replace dots with underscores to conform to OpenMetrics hierarchy convention
+        name = name.replace(".", "_")
+        # 2. Remove suffixes that will be added by Prometheus
+        #    - total : Prometheus counter convention
+        #    - count : histogram / summary aggregation
+        #    - ns    : unit suffix (units belong in metadata, not names)
+        name = self.suffix_regex.sub("", name)
+        return name
+
+    def _parse_qdb_statistics_entry(
+        self, labels: dict[str, str], metric_name: str, metric_info: dict
+    ) -> Union[GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily, None]:
+        """
+        Converts single statistic from QuasarDB to Prometheus Metric.
+        """
+        metric_type = self.qdbst_types_to_prometheus_types.get(
+            metric_info["type"], None
+        )
+
+        if metric_type is None:
+            self.logger.error(
+                "Unknown metric type for metric '%s': %s",
+                metric_name,
+                str(metric_info["type"]),
+            )
+            return None
+
+        metric_name = self._conform_qdb_name_to_prometheus_name(metric_name)
+
+        # Info metrics dont have units in Prometheus, metric creation is different from others
+        #
+        # value for `documentation` is required by Prometheus but not available from QuasarDB stats module itself, we set it to an empty string
+        metric_kwargs = {
+            "name": metric_name,
+            "documentation": "",
+            "labels": list(labels.keys()),
+        }
+        add_metric_kwargs = {
+            "labels": list(labels.values()),
+            "value": metric_info["value"],
+        }
+
+        # depending on metric type, adjust metric creation parameters
+        if metric_type is InfoMetricFamily:
+            add_metric_kwargs["value"] = {"value": metric_info["value"]}
+        else:
+            metric_kwargs["unit"] = metric_info["unit"].name.lower()
+
+        metric = metric_type(**metric_kwargs)
+        metric.add_metric(**add_metric_kwargs)
+        return metric
+
+    def _parse_qdb_statistics(self, qdb_metrics: dict) -> Generator[
+        Union[GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily, None],
+        None,
+        None,
+    ]:
+        """
+        Yields Prometheus metrics parsed from QuasarDB statistics dictionary.
+        """
+        for endpoint, metric_types in qdb_metrics.items():
+            for metric_name, metric_info in metric_types["cumulative"].items():
+                labels = {"endpoint": endpoint, "stat_type": "cumulative"}
+                yield self._parse_qdb_statistics_entry(labels, metric_name, metric_info)
+
+            for user_id, metrics in metric_types["by_uid"].items():
+                for metric_name, metric_info in metrics.items():
+                    labels = {
+                        "endpoint": endpoint,
+                        "stat_type": "by_uid",
+                        "user_id": f"{user_id}",
+                    }
+                    yield self._parse_qdb_statistics_entry(
+                        labels, metric_name, metric_info
+                    )
+
+    def collect(self):
+        """
+        Method called by Prometheus to collect metrics.
+        Yields Prometheus metrics collected from QuasarDB.
+        """
+
+        try:
+            stats = fetch_qdb_stats(
+                self.qdb_conn_args,
+                self.filter_include,
+                self.filter_exclude,
+                self.logger,
+            )
+            for metric in self._parse_qdb_statistics(stats):
+                yield metric
+        except Exception:
+            self.logger.exception("Failed to collect metrics")

qdb_prometheus_exporter/main.py

@@ -0,0 +1,90 @@
+import logging
+
+import click
+import prometheus_client
+import uvicorn
+from fastapi import FastAPI
+from prometheus_client import make_asgi_app
+from prometheus_client.core import REGISTRY
+
+from .collector import QdbStatsCollector
+
+
+def _set_up_prometheus_metrics_app(
+    conn_args, filter_include, filter_exclude, logger: logging.Logger
+):
+    metrics_app = make_asgi_app()
+    # By default prometheus client will send metrics about the Python process itself. We want to forward QuasarDB metrics only
+    REGISTRY.unregister(prometheus_client.GC_COLLECTOR)
+    REGISTRY.unregister(prometheus_client.PLATFORM_COLLECTOR)
+    REGISTRY.unregister(prometheus_client.PROCESS_COLLECTOR)
+    REGISTRY.register(
+        QdbStatsCollector(conn_args, filter_include, filter_exclude, logger)
+    )
+    return metrics_app
+
+
+app = FastAPI(debug=False)
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+def _parse_list(x):
+    """
+    Parses a comma-separated string into a list.
+    """
+
+    if x is None or not x.strip():
+        return None
+
+    return [token.strip() for token in x.split(",") if token.strip()]
+
+
+@click.command()
+@click.option("--cluster", default="qdb://127.0.0.1:2836", type=str)
+@click.option("--cluster-public-key-file", default=None, type=str)
+@click.option("--user-security-file", default=None, type=str)
+@click.option(
+    "--filter-include",
+    default="",
+    type=str,
+    help="Optional comma-separated list of regex patterns to filter metrics. Only metrics that match at least one of the patterns will be reported.",
+)
+@click.option(
+    "--filter-exclude",
+    default="",
+    type=str,
+    help="Optional comma-separated list of regex patterns to filter metrics. Only metrics that contain none of the patterns will be reported.",
+)
+@click.option("--exporter-port", default=9000, type=int)
+@click.option("--listen-address", default="127.0.0.1", type=str, help="Address on which the exporter will listen.")
+def start_server(
+    cluster: str,
+    cluster_public_key_file: str,
+    user_security_file: str,
+    filter_include: str,
+    filter_exclude: str,
+    exporter_port: int,
+    listen_address: str,
+):
+    conn_args = {
+        "uri": cluster,
+    }
+
+    if cluster_public_key_file and user_security_file:
+        conn_args["cluster_public_key_file"] = cluster_public_key_file
+        conn_args["user_security_file"] = user_security_file
+
+    logger = logging.getLogger("uvicorn.error")
+    metrics_app = _set_up_prometheus_metrics_app(
+        conn_args, _parse_list(filter_include), _parse_list(filter_exclude), logger
+    )
+    app.mount("/metrics", metrics_app)
+    uvicorn.run(app, host=listen_address, port=exporter_port)
+
+
+if __name__ == "__main__":
+    start_server()

qdb_prometheus_exporter/stats.py

@@ -0,0 +1,175 @@
+import copy
+import logging
+import random
+import re
+import uuid
+
+import quasardb
+import quasardb.stats as qdbst
+
+from typing import Union
+
+
+def _do_filter_metrics(metrics: dict, fn):
+    return {key: metrics[key] for key in metrics if fn(key)}
+
+
+def _do_filter(stats: dict, fn):
+    """
+    Performs actual filtering of stats, keeping only those where fn(name) equals True
+    """
+
+    for node_id in stats:
+        for group_id in stats[node_id]:
+            if group_id == "cumulative":
+                stats[node_id][group_id] = _do_filter_metrics(
+                    stats[node_id][group_id], fn
+                )
+            elif group_id == "by_uid":
+                for uid in stats[node_id][group_id]:
+                    stats[node_id][group_id][uid] = _do_filter_metrics(
+                        stats[node_id][group_id][uid], fn
+                    )
+            else:
+                raise RuntimeError(
+                    "Internal error: unrecognized stats group id: {}".format(group_id)
+                )
+    return stats
+
+
+def filter_stats(
+    stats: dict,
+    include: Union[list[str], None],
+    exclude: Union[list[str], None],
+    logger: logging.Logger,
+):
+    logger.info("Filtering stats based on include/exclude filters")
+    stats_ = copy.deepcopy(stats)
+
+    if include is not None:
+        # Returns `true` if any of the `include` patterns is found in the metric name.
+        def _filter_include(metric_name):
+            return any(
+                pattern for pattern in include if re.search(pattern, metric_name)
+            )
+
+        stats_ = _do_filter(stats_, _filter_include)
+
+    if exclude is not None:
+        # Returns `false` if any of the `exclude` patterns is found in the metric name.
+        def _filter_exclude(metric_name):
+            return not any(
+                pattern for pattern in exclude if re.search(pattern, metric_name)
+            )
+
+        stats_ = _do_filter(stats_, _filter_exclude)
+
+    return stats_
+
+
+def _check_node_online(conn: quasardb.Cluster, logger: logging.Logger):
+    logger.info("Checking node online")
+
+    ret = {}
+
+    for endpoint in conn.endpoints():
+        ret[endpoint] = 0  # pessimistic
+        node = conn.node(endpoint)
+        entry = node.integer("$qdb.statistics.startup_epoch")  # entry always exists
+
+        try:
+            entry.get()
+            ret[endpoint] = 1
+        except quasardb.Error as e:
+            logger.error("[%s] Failed to read sample entry: %s", endpoint, str(e))
+
+    return ret
+
+
+def _check_node_writable(conn: quasardb.Cluster, logger: logging.Logger):
+    logger.info("Checking node writable")
+    key = f"_qdb_write_check_{uuid.uuid4().hex}"  # almost zero chance of collision
+    value = random.randint(-9223372036854775808, 9223372036854775807)
+    ret = {}
+
+    for endpoint in conn.endpoints():
+        ret[endpoint] = 0  # pessimistic
+        node = conn.node(endpoint)
+        entry = node.integer(key)
+
+        try:
+            entry.put(value)
+            if entry.get() == value:
+                ret[endpoint] = 1
+        except quasardb.Error as e:
+            logger.error("[%s] Failed to put/get test entry '%s': %s", endpoint, key, e)
+        finally:
+            try:
+                entry.remove()
+            except quasardb.AliasNotFoundError as e:
+                logger.error(
+                    "[%s] Failed to put/get test entry '%s': %s", endpoint, key, e
+                )
+            except quasardb.Error as e:
+                logger.error(
+                    "[%s] Failed to clean up test entry '%s': %s", endpoint, key, e
+                )
+
+    return ret
+
+
+def _get_base_qdb_metrics(conn: quasardb.Cluster, logger: logging.Logger):
+    """
+    Returns basic QuasarDB metrics such as node online and writable status.
+    Those metrics are most commonly used to determine QuasarDB cluster health.
+    """
+    logger.info("Getting base QuasarDB metrics")
+    ret = {endpoint: {"cumulative": {}, "by_uid": {}} for endpoint in conn.endpoints()}
+    online_stats = _check_node_online(conn, logger)
+    writable_stats = _check_node_writable(conn, logger)
+
+    for endpoint in conn.endpoints():
+        ret[endpoint]["cumulative"]["check.online"] = {
+            "value": online_stats.get(endpoint, 0),
+            "type": qdbst.Type.GAUGE,
+            "unit": qdbst.Unit.NONE,
+        }
+        ret[endpoint]["cumulative"]["node.writable"] = {
+            "value": writable_stats.get(endpoint, 0),
+            "type": qdbst.Type.GAUGE,
+            "unit": qdbst.Unit.NONE,
+        }
+
+    return ret
+
+
+def fetch_qdb_stats(
+    qdb_conn_args: dict,
+    include: Union[list[str], None],
+    exclude: Union[list[str], None],
+    logger: logging.Logger,
+):
+    base_stats, node_stats = {}, {}
+    logger.info("Getting QuasarDB connection")
+    try:
+        with quasardb.Cluster(**qdb_conn_args) as conn:
+            base_stats = _get_base_qdb_metrics(conn, logger)
+            node_stats = qdbst.by_node(conn)
+    except Exception as e:
+        logger.error("Failed to fetch stats from QuasarDB: %s", str(e))
+        raise
+
+    # Merge base_stats into node_stats
+    combined_stats = node_stats
+    for endpoint, data in base_stats.items():
+        if endpoint not in combined_stats:
+            combined_stats[endpoint] = data
+        else:
+            if "cumulative" in data:
+                combined_stats[endpoint].setdefault("cumulative", {}).update(
+                    data["cumulative"]
+                )
+            if "by_uid" in data:
+                combined_stats[endpoint].setdefault("by_uid", {}).update(data["by_uid"])
+
+    return filter_stats(combined_stats, include, exclude, logger)

qdb_prometheus_exporter-3.14.2.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: qdb-prometheus-exporter
+Version: 3.14.2
+Summary: Statistics exporter for Prometheus monitoring system for QuasarDB time-series database.
+Author: quasardb SAS
+Author-email: support@quasar.ai
+Project-URL: Homepage, https://www.quasar.ai
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: prometheus-client
+Requires-Dist: quasardb==3.14.2
+Requires-Dist: click
+Requires-Dist: uvicorn
+Requires-Dist: fastapi
+Dynamic: license-file

qdb_prometheus_exporter-3.14.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+qdb_prometheus_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+qdb_prometheus_exporter/collector.py,sha256=NZ_ihNiP3Lq1VUBbzRTRgeUzBLvdzX7R14Q02ss8Dhg,5493
+qdb_prometheus_exporter/main.py,sha256=DG-FP2OIXLN5Dd70fVE9g8zJeLB7PN5xCoDLHGespFo,2715
+qdb_prometheus_exporter/stats.py,sha256=UeBJFuLfJx_qOyf1rVcTk1IZ6Gx7oKrIUilJtcH0-78,5648
+qdb_prometheus_exporter-3.14.2.dist-info/licenses/LICENSE,sha256=D4FxifKokzhemZloJq4ytQjoDdWHF6Yf11itdMlqzDI,1496
+qdb_prometheus_exporter-3.14.2.dist-info/METADATA,sha256=D0Pv5Y32_UatMFY8fVoP6qIiVaQYNOugn6a_6KTeF_U,462
+qdb_prometheus_exporter-3.14.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+qdb_prometheus_exporter-3.14.2.dist-info/entry_points.txt,sha256=bTj7lJtxFFqs0W8NYv16phryQ_4M_4sHDT8hfQDYbpA,86
+qdb_prometheus_exporter-3.14.2.dist-info/top_level.txt,sha256=nZG-KKjflJ9MUQa5TzVpdqOR0GZlQwzll6vMOLJJ90M,24
+qdb_prometheus_exporter-3.14.2.dist-info/RECORD,,

qdb_prometheus_exporter-3.14.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

qdb_prometheus_exporter-3.14.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+qdb-prometheus-exporter = qdb_prometheus_exporter.main:start_server

qdb_prometheus_exporter-3.14.2.dist-info/licenses/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Quasar AI
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

qdb_prometheus_exporter-3.14.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+qdb_prometheus_exporter