snakemake-logger-plugin-prometheus 0.1.1__py3-none-any.whl

snakemake_logger_plugin_prometheus/__init__.py

@@ -0,0 +1,379 @@
+from dataclasses import dataclass, field
+import logging
+import threading
+import sys
+import uuid
+from typing import Any
+
+from snakemake_interface_logger_plugins.base import LogHandlerBase
+from snakemake_interface_logger_plugins.settings import LogHandlerSettingsBase
+from snakemake_interface_logger_plugins.common import LogEvent
+from prometheus_client import (
+    start_http_server,
+    Gauge,
+    Counter,
+    push_to_gateway,
+    REGISTRY,
+)
+
+
+@dataclass
+class LogHandlerSettings(LogHandlerSettingsBase):
+    port: int = field(
+        default=8000,
+        metadata={
+            "help": "Port to expose Prometheus metrics on",
+            "env_var": False,
+            "required": False,
+            "type": int,
+        },
+    )
+    push_gateway: str | None = field(
+        default=None,
+        metadata={
+            "help": "URL of the Prometheus Pushgateway (e.g. http://localhost:9091)",
+            "env_var": False,
+            "required": False,
+            "type": str,
+        },
+    )
+    push_job_name: str = field(
+        default="snakemake",
+        metadata={
+            "help": "Job name for the Pushgateway grouping",
+            "env_var": False,
+            "required": False,
+            "type": str,
+        },
+    )
+    run_id: str | None = field(
+        default=None,
+        metadata={
+            "help": "Unique identifier for this workflow run (label). Defaults to UUID.",
+            "env_var": False,
+            "required": False,
+            "type": str,
+        },
+    )
+
+
+@dataclass
+class JobMetadata:
+    """
+    Structured container for job information extracted from JOB_INFO logs.
+    """
+
+    job_id: int
+    rule_name: str
+    resources: dict[str, int | float]
+    threads: int
+
+
+class LogHandler(LogHandlerBase):
+    def __post_init__(self) -> None:
+        if self.common_settings.dryrun:
+            return
+        self.job_registry: dict[int, JobMetadata] = {}
+        self.running_jobs: set[int] = set()
+        self.deferred_starts: set[int] = set()
+
+        self.run_id = getattr(self.settings, "run_id", None)
+        if not self.run_id:
+            self.run_id = str(uuid.uuid4())
+            sys.stderr.write(f"[PrometheusPlugin] Auto-generated run_id: {self.run_id}\n")
+
+        self.metric_submitted = Gauge(
+            "snakemake_jobs_submitted",
+            "Number of jobs known to the scheduler (Queued + Running).",
+            ["run_id", "rule"],
+        )
+
+        self.metric_running = Gauge(
+            "snakemake_jobs_running",
+            "Number of jobs actively executing.",
+            ["run_id", "rule"],
+        )
+
+        self.metric_finished = Counter(
+            "snakemake_jobs_finished_total",
+            "Total number of jobs finished/failed.",
+            ["run_id", "status", "rule"],
+        )
+
+        self.metric_resource_usage = Gauge(
+            "snakemake_resource_usage_total",
+            "Total resources currently consumed by RUNNING jobs.",
+            ["run_id", "resource"],
+        )
+
+        self.metric_resource_limits = Gauge(
+            "snakemake_resource_limits",
+            "Maximum resources available to the workflow execution.",
+            ["run_id", "resource"],
+        )
+
+        self.metric_progress = Gauge(
+            "snakemake_workflow_progress",
+            "High-level workflow progress (jobs done vs total).",
+            ["run_id", "type"],
+        )
+
+        self.metric_planned = Gauge(
+            "snakemake_jobs_planned_total", "Jobs planned per rule. Updates on DAG re-evaluation.", ["run_id", "rule"]
+        )
+
+        self.metric_errors = Counter("snakemake_errors_total", "Total number of workflow errors.", ["run_id"])
+
+        self._setup_server()
+        self._setup_push_gateway()
+
+    def _setup_server(self) -> None:
+        port = getattr(self.settings, "port", 8000)
+        try:
+            start_http_server(port)
+            sys.stderr.write(f"[PrometheusPlugin] Metrics server started on port {port}\n")
+        except OSError as e:
+            sys.stderr.write(f"[PrometheusPlugin] ⚠️ Could not start server on port {port}: {e}\n")
+
+    def _setup_push_gateway(self) -> None:
+        self._stop_push_event = threading.Event()
+        push_gateway = getattr(self.settings, "push_gateway", None)
+
+        if push_gateway:
+            push_job_name = getattr(self.settings, "push_job_name", "snakemake")
+            sys.stderr.write(f"[PrometheusPlugin] Pushing metrics to {push_gateway} (job: {push_job_name})\n")
+            self._push_thread = threading.Thread(target=self._push_loop, daemon=True)
+            self._push_thread.start()
+
+    def _push_loop(self) -> None:
+        """Periodically pushes metrics to the Pushgateway."""
+        push_gateway = getattr(self.settings, "push_gateway", None)
+        assert push_gateway is not None
+        push_job_name = getattr(self.settings, "push_job_name", "snakemake")
+
+        while not self._stop_push_event.is_set():
+            try:
+                push_to_gateway(
+                    push_gateway, job=push_job_name, registry=REGISTRY, grouping_key={"run_id": self.run_id}
+                )
+            except Exception as e:
+                sys.stderr.write(f"[PrometheusPlugin] Push failed: {e}\n")
+
+            if self._stop_push_event.wait(15):
+                break
+
+    def close(self) -> None:
+        if hasattr(self, "_stop_push_event"):
+            self._stop_push_event.set()
+            if hasattr(self, "_push_thread") and self._push_thread.is_alive():
+                self._push_thread.join(timeout=1.0)
+
+        self._cleanup_registry()
+        super().close()
+
+    def _cleanup_registry(self) -> None:
+        """Unregister metrics to prevent state bleeding in test suites."""
+        metrics = [
+            self.metric_submitted,
+            self.metric_running,
+            self.metric_finished,
+            self.metric_resource_usage,
+            self.metric_resource_limits,
+            self.metric_progress,
+            self.metric_planned,
+            self.metric_errors,
+        ]
+        for m in metrics:
+            try:
+                REGISTRY.unregister(m)
+            except (KeyError, AttributeError, TypeError):
+                pass
+
+    @property
+    def writes_to_stream(self) -> bool:
+        return False
+
+    @property
+    def writes_to_file(self) -> bool:
+        return False
+
+    @property
+    def has_filter(self) -> bool:
+        return True
+
+    @property
+    def has_formatter(self) -> bool:
+        return True
+
+    @property
+    def needs_rulegraph(self) -> bool:
+        return False
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            if not hasattr(record, "event"):
+                return
+
+            event = record.event
+
+            if event == LogEvent.JOB_INFO:
+                self._handle_job_info(record)
+            elif event == LogEvent.JOB_STARTED:
+                self._handle_job_started(record)
+            elif event == LogEvent.JOB_FINISHED:
+                self._handle_job_finished(record)
+            elif event == LogEvent.JOB_ERROR:
+                self._handle_job_error(record)
+            elif event == LogEvent.ERROR:
+                self.metric_errors.labels(run_id=self.run_id).inc()
+            elif event == LogEvent.RUN_INFO:
+                self._handle_run_info(record)
+            elif event == LogEvent.PROGRESS:
+                self._handle_progress(record)
+            elif event == LogEvent.RESOURCES_INFO:
+                self._handle_resources_info(record)
+
+        except Exception:
+            self.handleError(record)
+
+    def _parse_resources(self, record: logging.LogRecord) -> dict[str, int | float]:
+        """
+        Robustly extracts resources from a LogRecord.
+        Handles: dicts, argparse.Namespace, and Snakemake's internal Resource objects.
+        """
+        raw_res = getattr(record, "resources", None)
+        resources: dict[str, Any] = {}
+
+        if raw_res:
+            if hasattr(raw_res, "_names"):
+                for name in raw_res._names:
+                    resources[name] = getattr(raw_res, name)
+            elif isinstance(raw_res, dict):
+                resources = raw_res.copy()
+            elif hasattr(raw_res, "__dict__"):
+                resources = vars(raw_res).copy()
+            elif hasattr(raw_res, "keys") and callable(raw_res.keys):
+                for k in raw_res.keys():
+                    resources[k] = raw_res[k]
+
+        filtered: dict[str, int | float] = {
+            k: v for k, v in resources.items() if k not in {"_cores", "_nodes"} and isinstance(v, (int, float))
+        }
+
+        if "threads" not in filtered:
+            threads = getattr(record, "threads", 1)
+            if isinstance(threads, (int, float)):
+                filtered["threads"] = threads
+
+        return filtered
+
+    def _handle_run_info(self, record: logging.LogRecord) -> None:
+        """Handles run info (DAG stats). Can be called multiple times if DAG is re-evaluated."""
+        counts = getattr(record, "per_rule_job_counts", {})
+        if counts:
+            for rule, count in counts.items():
+                self.metric_planned.labels(run_id=self.run_id, rule=rule).set(count)
+
+    def _handle_progress(self, record: logging.LogRecord) -> None:
+        """Handles progress updates. 'total' can change dynamically via checkpoints."""
+        done = getattr(record, "done", 0)
+        total = getattr(record, "total", 0)
+        self.metric_progress.labels(run_id=self.run_id, type="done").set(done)
+        self.metric_progress.labels(run_id=self.run_id, type="total").set(total)
+
+    def _handle_resources_info(self, record: logging.LogRecord) -> None:
+        """Handles the global resources info to set usage limits."""
+        cores = getattr(record, "cores", None)
+        if cores is not None:
+            self.metric_resource_limits.labels(run_id=self.run_id, resource="threads").set(cores)
+            self.metric_resource_limits.labels(run_id=self.run_id, resource="cores").set(cores)
+
+        provided = getattr(record, "provided_resources", {})
+        if provided:
+            for res, value in provided.items():
+                if isinstance(value, (int, float)):
+                    self.metric_resource_limits.labels(run_id=self.run_id, resource=res).set(value)
+
+    def _handle_job_info(self, record: logging.LogRecord) -> None:
+        job_id = getattr(record, "jobid", None)
+        if job_id is None:
+            return
+
+        rule_name = getattr(record, "rule_name", "unknown")
+        resources = self._parse_resources(record)
+        threads = int(resources.get("threads", 1))
+
+        metadata = JobMetadata(job_id=job_id, rule_name=rule_name, resources=resources, threads=threads)
+
+        self.job_registry[job_id] = metadata
+
+        self.metric_submitted.labels(run_id=self.run_id, rule=rule_name).inc()
+
+        if job_id in self.deferred_starts:
+            self.deferred_starts.remove(job_id)
+            self._record_job_start(metadata)
+
+    def _handle_job_started(self, record: logging.LogRecord) -> None:
+        job_ids = getattr(record, "job_ids", [])
+
+        if not job_ids:
+            job_ids = getattr(record, "jobs", [])
+
+        if isinstance(job_ids, int):
+            job_ids = [job_ids]
+        elif not job_ids and hasattr(record, "jobid"):
+            jid = getattr(record, "jobid")
+            if jid is not None:
+                job_ids = [jid]
+
+        for job_id in job_ids:
+            if not isinstance(job_id, int):
+                continue
+
+            if job_id in self.job_registry:
+                self._record_job_start(self.job_registry[job_id])
+            else:
+                self.deferred_starts.add(job_id)
+
+    def _record_job_start(self, metadata: JobMetadata) -> None:
+        """Internal helper to increment running metrics."""
+        if metadata.job_id in self.running_jobs:
+            return
+
+        self.metric_running.labels(run_id=self.run_id, rule=metadata.rule_name).inc()
+
+        for res_name, res_value in metadata.resources.items():
+            self.metric_resource_usage.labels(run_id=self.run_id, resource=res_name).inc(res_value)
+
+        self.running_jobs.add(metadata.job_id)
+
+    def _handle_job_finished(self, record: logging.LogRecord) -> None:
+        job_id = getattr(record, "job_id", getattr(record, "jobid", None))
+        if job_id is not None:
+            self._finalize_job(job_id, "finished")
+
+    def _handle_job_error(self, record: logging.LogRecord) -> None:
+        job_id = getattr(record, "jobid", None)
+        if job_id is not None:
+            self._finalize_job(job_id, "failed")
+
+    def _finalize_job(self, job_id: int, final_status: str) -> None:
+        job_info = self.job_registry.pop(job_id, None)
+
+        if job_id in self.deferred_starts:
+            self.deferred_starts.remove(job_id)
+
+        if job_info:
+            rule = job_info.rule_name
+
+            self.metric_finished.labels(run_id=self.run_id, status=final_status, rule=rule).inc()
+
+            self.metric_submitted.labels(run_id=self.run_id, rule=rule).dec()
+
+            if job_id in self.running_jobs:
+                self.metric_running.labels(run_id=self.run_id, rule=rule).dec()
+
+                for res_name, res_value in job_info.resources.items():
+                    self.metric_resource_usage.labels(run_id=self.run_id, resource=res_name).dec(res_value)
+
+                self.running_jobs.remove(job_id)

snakemake_logger_plugin_prometheus-0.1.1.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: snakemake-logger-plugin-prometheus
+Version: 0.1.1
+Summary: Snakemake logger plugin that exposes job info to prometheus.
+Project-URL: repository, https://github.com/tedil/snakemake-logger-plugin-prometheus
+Author-email: Till Hartmann <till.hartmann@bih-charite.de>
+License-File: LICENSE
+Requires-Python: <4.0,>=3.11
+Requires-Dist: prometheus-client<0.24,>=0.23.1
+Requires-Dist: snakemake-interface-logger-plugins<3,>=2.0
+Description-Content-Type: text/markdown
+
+# Snakemake Prometheus Logger Plugin
+
+A Snakemake logger plugin that exposes workflow metrics via a Prometheus-compatible HTTP endpoint.
+
+## Installation
+
+You can install this plugin via pip:
+
+```sh
+pip install snakemake-logger-plugin-prometheus
+```
+
+Or via Pixi/Conda if available in your channels.
+
+## Usage
+
+To use this logger, run Snakemake with the `--logger prometheus` flag.
+
+```sh
+snakemake --logger prometheus ...
+```
+
+### Pull Mode (Default)
+
+By default, the metrics server runs on port `8000`. Prometheus scrapes this address.
+You can change the port using the `--logger-prometheus-port` flag:
+
+```sh
+snakemake --logger prometheus --logger-prometheus-port 9090 ...
+```
+
+### Push Mode (Pushgateway)
+
+If you cannot expose a port or prefer pushing metrics, specify a Pushgateway URL.
+The plugin will push metrics every 15 seconds.
+
+```sh
+snakemake --logger prometheus --logger-prometheus-push-gateway http://localhost:9091
+```
+
+You can optionally set the job name used in the Pushgateway (default: `"snakemake"`):
+
+```sh
+snakemake --logger prometheus --logger-prometheus-push-gateway http://localhost:9091 --logger-prometheus-push-job-name my-workflow
+```
+
+## Available Metrics
+
+### Job Lifecycle
+
+| Metric Name                     | Type    | Labels                             | Description                                                                                               |
+|:--------------------------------|:--------|:-----------------------------------|:----------------------------------------------------------------------------------------------------------|
+| `snakemake_jobs_submitted`      | Gauge   | `rule`                             | Jobs known to the scheduler (Queued + Running). Increments when a job is ready, decrements when finished. |
+| `snakemake_jobs_running`        | Gauge   | `rule`                             | Jobs actively executing. Increments when execution starts, decrements when finished.                      |
+| `snakemake_jobs_finished_total` | Counter | `status` (finished/failed), `rule` | Cumulative count of completed jobs. Useful for throughput rates.                                          |
+| `snakemake_jobs_planned_total`  | Gauge   | `rule`                             | Total jobs planned for the workflow. Updates dynamically if checkpoints trigger DAG re-evaluation.        |
+
+### Resources
+
+| Metric Name                      | Type  | Labels     | Description                                                                         |
+|:---------------------------------|:------|:-----------|:------------------------------------------------------------------------------------|
+| `snakemake_resource_usage_total` | Gauge | `resource` | Total resources currently consumed by **Running** jobs (e.g., `threads`, `mem_mb`). |
+| `snakemake_resource_limits`      | Gauge | `resource` | Maximum resources available to the workflow (CLI limits or cluster capacity).       |
+
+### Workflow Status
+
+| Metric Name                   | Type    | Labels              | Description                            |
+|:------------------------------|:--------|:--------------------|:---------------------------------------|
+| `snakemake_workflow_progress` | Gauge   | `type` (done/total) | High-level progress counters.          |
+| `snakemake_errors_total`      | Counter | None                | Total number of workflow-level errors. |
+
+## Grafana Query Examples
+
+Here are common queries to visualize this data:
+
+| Description                  | Query                                                                                                                |
+|:-----------------------------|:---------------------------------------------------------------------------------------------------------------------|
+| **Queued Jobs**              | `sum(snakemake_jobs_submitted) - sum(snakemake_jobs_running)`                                                        |
+| **Running Jobs**             | `sum(snakemake_jobs_running)`                                                                                        |
+| **Throughput (Jobs/Minute)** | `sum(rate(snakemake_jobs_finished_total{status="finished"}[5m])) * 60`                                               |
+| **Resource Utilization %**   | `sum(snakemake_resource_usage_total{resource="threads"}) / sum(snakemake_resource_limits{resource="threads"}) * 100` |
+| **Failure Rate**             | `sum(rate(snakemake_jobs_finished_total{status="failed"}[5m]))`                                                      |
+
+## Development
+
+This project uses [pixi](https://pixi.sh) for dependency management and [uv](https://docs.astral.sh/uv/) for
+building/publishing.
+
+- Install dependencies: `pixi install`
+- Run tests: `pixi run test`
+- Run code formatting and linting: `pixi run qc`
+- Build package: `pixi run build`
+- Publish package: `pixi run publish`

snakemake_logger_plugin_prometheus-0.1.1.dist-info/RECORD

@@ -0,0 +1,5 @@
+snakemake_logger_plugin_prometheus/__init__.py,sha256=IasH6hqoYmhxeVN_iAgtkUtAk79VXFZYJxOBq949dyA,13387
+snakemake_logger_plugin_prometheus-0.1.1.dist-info/METADATA,sha256=eGCHpBS4woIP5Iserw5o94Qha9KvBhTf3AuMR6UuUSg,5360
+snakemake_logger_plugin_prometheus-0.1.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+snakemake_logger_plugin_prometheus-0.1.1.dist-info/licenses/LICENSE,sha256=y8AftXce0Ne8jKczrAZvysqhg9zE5lFB5g7gLZL9FdU,1070
+snakemake_logger_plugin_prometheus-0.1.1.dist-info/RECORD,,

snakemake_logger_plugin_prometheus-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

snakemake_logger_plugin_prometheus-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Till Hartmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.