async-runtime-auditor 0.1.0__py3-none-any.whl

async_auditor/cli.py

@@ -0,0 +1,350 @@
+import argparse
+import json
+import math
+import requests
+import shutil
+import sys
+import yaml
+
+from pathlib import Path
+
+# ---------------------------------------------------------
+# DEFAULT CONFIG PATH (Used ONLY for --init)
+# ---------------------------------------------------------
+
+DEFAULT_CONFIG_PATH = (
+    Path(__file__).parent / "default_metrics.yaml"
+)
+
+# ---------------------------------------------------------
+# CLI ARGUMENT PARSER
+# ---------------------------------------------------------
+
+def parse_args():
+
+    parser = argparse.ArgumentParser(
+        prog="async-auditor",
+        description=(
+            "Runtime audit CLI for detecting asyncio "
+            "event-loop starvation and threadpool saturation."
+        )
+    )
+
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=None,  # <-- STRICT CI: No silent fallback
+        help="Path to metrics configuration YAML"
+    )
+
+    parser.add_argument(
+        "--target",
+        type=str,
+        default=None,
+        help="Override Prometheus URL target"
+    )
+
+    parser.add_argument(
+        "--output-format",
+        type=str,
+        choices=["text", "json"],
+        default="text",
+        help="Output format"
+    )
+
+    parser.add_argument(
+        "--fail-on-critical",
+        action="store_true",
+        help=(
+            "Exit with code 1 if runtime "
+            "state is CRITICAL or COLLAPSE_RISK"
+        )
+    )
+
+    parser.add_argument(
+        "--init",
+        action="store_true",
+        help="Generate a template metrics.yaml in the current directory"
+    )
+
+    return parser.parse_args()
+
+# ---------------------------------------------------------
+# CONFIG PATH RESOLUTION
+# ---------------------------------------------------------
+
+def resolve_config_path(explicit_config, output_format):
+
+    # 1. EXPLICIT CONFIG OVERRIDE
+    if explicit_config:
+        return Path(explicit_config)
+
+    # 2. LOCAL metrics.yaml IN CURRENT DIRECTORY
+    local_config = Path.cwd() / "metrics.yaml"
+    if local_config.exists():
+        return local_config
+
+    # 3. NO CONFIG FOUND (HARD FAIL)
+    if output_format == "text":
+        print(
+            "[FATAL] No metrics configuration found.\n"
+            "Run:\n"
+            "  async-auditor --init\n"
+            "or provide explicit path:\n"
+            "  async-auditor --config <path>"
+        )
+    sys.exit(1)
+
+# ---------------------------------------------------------
+# CLI UX: INIT CONFIG
+# ---------------------------------------------------------
+
+def init_template_config():
+    target_path = Path.cwd() / "metrics.yaml"
+    
+    if target_path.exists():
+        print(f"[ERROR] {target_path.name} already exists in this directory.")
+        sys.exit(1)
+        
+    try:
+        shutil.copy(DEFAULT_CONFIG_PATH, target_path)
+        print(f"Successfully generated template: {target_path.absolute()}")
+        sys.exit(0)
+    except Exception as e:
+        print(f"[FATAL] Could not generate template: {e}")
+        sys.exit(1)
+
+# ---------------------------------------------------------
+# PRE-FLIGHT TELEMETRY CHECK
+# ---------------------------------------------------------
+
+def verify_telemetry_backend(prometheus_url, output_format):
+    health_endpoint = f"{prometheus_url}/-/healthy"
+    try:
+        response = requests.get(health_endpoint, timeout=5)
+        response.raise_for_status()
+    except Exception as e:
+        if output_format == "text":
+            print(f"[FATAL] Telemetry backend unreachable at {health_endpoint}")
+            print(f"Error: {e}")
+            print("Cannot audit runtime state without observability data.")
+        sys.exit(1)
+
+# ---------------------------------------------------------
+# SAFE NUMERIC HANDLING
+# ---------------------------------------------------------
+
+def sanitize_metric(value):
+    if value is None or math.isnan(value) or math.isinf(value):
+        return 0.0
+    return value
+
+# ---------------------------------------------------------
+# CONFIGURATION LOADER
+# ---------------------------------------------------------
+
+def load_config(config_path, output_format):
+    try:
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f)
+            if config is None:
+                raise ValueError("metrics config is empty")
+            return config
+    except Exception as e:
+        if output_format == "text":
+            print(f"[FATAL] Failed to load config {config_path}: {e}")
+        sys.exit(1)
+
+# ---------------------------------------------------------
+# PROMETHEUS QUERY HELPER
+# ---------------------------------------------------------
+
+def query_prometheus(query_key, queries, query_endpoint, output_format, config_path):
+    query = queries.get(query_key)
+    if not query:
+        if output_format == "text":
+            print(f"[WARNING] Query key '{query_key}' not found in {config_path}")
+        return 0.0
+
+    try:
+        response = requests.get(
+            query_endpoint,
+            params={"query": query},
+            timeout=10,
+        )
+        response.raise_for_status()
+        results = response.json()["data"]["result"]
+
+        if not results:
+            return 0.0
+
+        value = float(results[0]["value"][1])
+        return sanitize_metric(value)
+
+    except Exception as e:
+        if output_format == "text":
+            print(f"[ERROR] Prometheus query failed for '{query_key}': {e}")
+        return 0.0
+
+# ---------------------------------------------------------
+# RUNTIME HEALTH CLASSIFICATION
+# ---------------------------------------------------------
+
+def classify_runtime_health(score, thresholds):
+    score = sanitize_metric(score)
+    if score >= thresholds.get("health_score_collapse", 2500): return "COLLAPSE_RISK"
+    elif score >= thresholds.get("health_score_critical", 1200): return "CRITICAL"
+    elif score >= thresholds.get("health_score_degraded", 400): return "DEGRADED"
+    return "HEALTHY"
+
+# ---------------------------------------------------------
+# FAILURE MODE CLASSIFICATION
+# ---------------------------------------------------------
+
+def classify_failure_modes(max_lag, avg_blocking_duration, peak_active_requests, threadpool_queue_wait, threadpool_backlog, thresholds):
+    failure_modes = []
+
+    if max_lag > thresholds.get("max_event_loop_lag_s", 0.5) and avg_blocking_duration > thresholds.get("max_blocking_duration_s", 1.0):
+        failure_modes.append({"type": "EVENT_LOOP_STARVATION", "severity": "HIGH", "description": "Synchronous blocking is stalling the asyncio scheduler."})
+
+    if threadpool_queue_wait > thresholds.get("max_queue_wait_s", 1.0) and threadpool_backlog > thresholds.get("max_threadpool_backlog", 3):
+        failure_modes.append({"type": "THREADPOOL_SATURATION", "severity": "HIGH", "description": "Executor queue amplification detected."})
+
+    if peak_active_requests > thresholds.get("max_active_requests", 150):
+        failure_modes.append({"type": "LOAD_SATURATION", "severity": "MEDIUM", "description": "Concurrent request pressure is approaching runtime limits."})
+
+    return failure_modes
+
+# ---------------------------------------------------------
+# AUDIT REPORT GENERATOR
+# ---------------------------------------------------------
+
+def generate_audit_report(args):
+    config = load_config(args.config, args.output_format)
+    queries = config.get("queries", {})
+    thresholds = config.get("thresholds", {})
+
+    prometheus_url = args.target if args.target else config.get("target", "http://localhost:9090")
+    prometheus_url = prometheus_url.rstrip("/")
+    query_endpoint = f"{prometheus_url}/api/v1/query"
+    output_file = Path.cwd() / "audit_results.json"
+
+    # Pre-flight Check
+    verify_telemetry_backend(prometheus_url, args.output_format)
+
+    # -----------------------------------------------------
+    # METRIC INGESTION
+    # -----------------------------------------------------
+    p99_latency = query_prometheus("p99_latency", queries, query_endpoint, args.output_format, args.config)
+    max_lag = query_prometheus("event_loop_lag", queries, query_endpoint, args.output_format, args.config)
+    total_blocking_events = query_prometheus("blocking_events_total", queries, query_endpoint, args.output_format, args.config)
+    avg_blocking_duration = query_prometheus("blocking_duration_avg", queries, query_endpoint, args.output_format, args.config)
+    peak_active_requests = query_prometheus("peak_active_requests", queries, query_endpoint, args.output_format, args.config)
+    threadpool_queue_wait = query_prometheus("threadpool_queue_wait", queries, query_endpoint, args.output_format, args.config)
+    tasks_started = query_prometheus("threadpool_tasks_started", queries, query_endpoint, args.output_format, args.config)
+    tasks_completed = query_prometheus("threadpool_tasks_completed", queries, query_endpoint, args.output_format, args.config)
+    threadpool_backlog = sanitize_metric(max(0, tasks_started - tasks_completed))
+
+    # -----------------------------------------------------
+    # RUNTIME HEALTH SCORE
+    # -----------------------------------------------------
+    runtime_health_score = sanitize_metric(
+        (max_lag * 1000 * 1.5) +
+        (avg_blocking_duration * 400) +
+        (threadpool_queue_wait * 300) +
+        (threadpool_backlog * 8) +
+        (peak_active_requests * 1)
+    )
+    runtime_status = classify_runtime_health(runtime_health_score, thresholds)
+
+    # -----------------------------------------------------
+    # FAILURE MODE ANALYSIS
+    # -----------------------------------------------------
+    failure_modes = classify_failure_modes(
+        max_lag, avg_blocking_duration, peak_active_requests,
+        threadpool_queue_wait, threadpool_backlog, thresholds
+    )
+
+    # -----------------------------------------------------
+    # STRUCTURAL FINDINGS
+    # -----------------------------------------------------
+    findings = []
+    if max_lag > thresholds.get("max_event_loop_lag_s", 0.5): findings.append("Event-loop starvation detected.")
+    if avg_blocking_duration > thresholds.get("max_blocking_duration_s", 1.0): findings.append("Long-duration synchronous blocking observed.")
+    if max_lag > thresholds.get("latency_hide_threshold_s", 1.0) and p99_latency < thresholds.get("p99_latency_hide_threshold_s", 0.15):
+        findings.append("Customer-visible latency hides runtime instability.")
+    if peak_active_requests > thresholds.get("load_warning_threshold", 100): findings.append("Concurrent load saturation detected.")
+    if threadpool_backlog > thresholds.get("max_threadpool_backlog", 3): findings.append("Threadpool backlog accumulation detected.")
+    if threadpool_queue_wait > thresholds.get("max_queue_wait_s", 1.0): findings.append("Executor queue amplification detected.")
+
+    # -----------------------------------------------------
+    # REPORT OBJECT
+    # -----------------------------------------------------
+    report = {
+        "status": runtime_status,
+        "runtime_health_score": round(runtime_health_score, 2),
+        "p99_latency_ms": round(p99_latency * 1000, 2),
+        "max_event_loop_lag_ms": round(max_lag * 1000, 2),
+        "blocking_events": int(total_blocking_events),
+        "avg_blocking_duration_s": round(avg_blocking_duration, 2),
+        "peak_active_requests": int(peak_active_requests),
+        "threadpool_queue_wait_s": round(threadpool_queue_wait, 2),
+        "threadpool_backlog": int(threadpool_backlog),
+        "failure_modes": failure_modes,
+        "findings": findings,
+    }
+
+    # -----------------------------------------------------
+    # TEXT OUTPUT
+    # -----------------------------------------------------
+    if args.output_format == "text":
+        print("=" * 60)
+        print("ASYNC RUNTIME AUDITOR")
+        print("=" * 60)
+        print(f"Target: {prometheus_url}")
+        print(f"Config: {args.config}\n")
+        print(f"Runtime Status: {runtime_status}")
+        print(f"Runtime Health Score: {runtime_health_score:.0f}\n")
+        
+        if findings:
+            print("Findings:")
+            for finding in findings:
+                print(f"  - {finding}")
+        else:
+            print("No major runtime issues detected.")
+        print()
+
+        with open(output_file, "w") as f:
+            json.dump(report, f, indent=2)
+        print(f"JSON audit report written to:\n{output_file}")
+    else:
+        print(json.dumps(report, indent=2))
+
+    # -----------------------------------------------------
+    # CI EXIT SEMANTICS
+    # -----------------------------------------------------
+    if args.fail_on_critical and runtime_status in ["CRITICAL", "COLLAPSE_RISK"]:
+        sys.exit(1)
+
+    sys.exit(0)
+
+# ---------------------------------------------------------
+# CLI ENTRYPOINT
+# ---------------------------------------------------------
+
+def main():
+    args = parse_args()
+
+    # 1. Handle Init First
+    if args.init:
+        init_template_config()
+
+    # 2. Resolve Config (Enforce Determinism)
+    resolved_config = resolve_config_path(args.config, args.output_format)
+    args.config = str(resolved_config)
+
+    # 3. Execute Audit
+    generate_audit_report(args)
+
+if __name__ == "__main__":
+    main()

async_auditor/default_metrics.yaml

@@ -0,0 +1,21 @@
+target: "http://localhost:9090"
+
+queries:
+  p99_latency: "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))"
+  event_loop_lag: "max_over_time(asyncio_event_loop_lag_seconds[15m])"
+  blocking_events_total: "blocking_events_total"
+  blocking_duration_avg: "rate(blocking_duration_seconds_sum[15m]) / rate(blocking_duration_seconds_count[15m])"
+  peak_active_requests: "max_over_time(active_requests[15m])"
+  threadpool_queue_wait: "rate(threadpool_queue_wait_seconds_sum[15m]) / rate(threadpool_queue_wait_seconds_count[15m])"
+  threadpool_tasks_started: "threadpool_tasks_started_total"
+  threadpool_tasks_completed: "threadpool_tasks_completed_total"
+
+thresholds:
+  health_score_collapse: 2500
+  health_score_critical: 1200
+  health_score_degraded: 400
+  max_event_loop_lag_s: 0.5
+  max_blocking_duration_s: 1.0
+  max_queue_wait_s: 1.0
+  max_threadpool_backlog: 3
+  max_active_requests: 150

async_runtime_auditor-0.1.0.dist-info/METADATA

@@ -0,0 +1,261 @@
+Metadata-Version: 2.4
+Name: async-runtime-auditor
+Version: 0.1.0
+Summary: Runtime audit CLI for detecting asyncio degradation and threadpool saturation.
+Author: Priyanshu
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: System :: Monitoring
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: PyYAML>=6.0.1
+Dynamic: license-file
+
+
+# Async Runtime Auditor
+
+A lightweight CI/CD runtime audit CLI for Python `asyncio` applications.
+
+Standard APM metrics (like P99 HTTP latency) frequently hide severe async runtime degradation. Synchronous blocking calls and executor queue amplification can stall the event loop while external HTTP metrics still appear healthy.
+
+**Async Runtime Auditor** is a heuristic-driven operational tool designed to run in staging environments and deployment pipelines. It queries a telemetry backend (such as Prometheus), evaluates runtime state against configurable thresholds, and fails the build before blocking code reaches production.
+
+---
+
+# The Problem: Telemetry Asymmetry
+
+In many production Python systems:
+
+```text
+healthy HTTP latency != healthy runtime state
+```
+
+Standard infrastructure metrics frequently fail to detect:
+
+- synchronous I/O executed on the main async thread  
+- threadpool saturation and executor queue wait times  
+- asynchronous scheduler starvation  
+- hidden queue amplification  
+
+This tool exposes these blind spots by evaluating event-loop and executor telemetry directly.
+
+---
+
+# Installation
+
+Requires Python 3.9+
+
+```bash
+pip install async-runtime-auditor
+```
+
+For local development:
+
+```bash
+pip install -e .
+```
+
+from the repository root.
+
+---
+
+# Quick Start
+
+## 1. Initialize Configuration
+
+Generate a local `metrics.yaml` configuration file:
+
+```bash
+async-auditor --init
+```
+
+---
+
+## 2. Run an Audit
+
+Run the tool against a staging or local telemetry backend:
+
+```bash
+async-auditor --target http://localhost:9090
+```
+
+---
+
+## 3. CI/CD Pipeline Gating
+
+Run with strict exit semantics to fail the pipeline if critical degradation is detected:
+
+```bash
+async-auditor --target http://localhost:9090 --fail-on-critical
+```
+
+The CLI exits with code `1` if runtime state is:
+
+- `CRITICAL`  
+- `COLLAPSE_RISK`  
+
+---
+
+# Configuration (`metrics.yaml`)
+
+The auditor is intentionally decoupled from specific telemetry naming conventions.
+
+Prometheus queries are mapped into the heuristic engine through `metrics.yaml`.
+
+```yaml
+target: "http://localhost:9090"
+
+queries:
+  p99_latency: "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))"
+  event_loop_lag: "max_over_time(asyncio_event_loop_lag_seconds[15m])"
+  blocking_events_total: "blocking_events_total"
+  blocking_duration_avg: "rate(blocking_duration_seconds_sum[15m]) / rate(blocking_duration_seconds_count[15m])"
+  peak_active_requests: "max_over_time(active_requests[15m])"
+  threadpool_queue_wait: "rate(threadpool_queue_wait_seconds_sum[15m]) / rate(threadpool_queue_wait_seconds_count[15m])"
+  threadpool_tasks_started: "threadpool_tasks_started_total"
+  threadpool_tasks_completed: "threadpool_tasks_completed_total"
+
+thresholds:
+  health_score_collapse: 2500
+  health_score_critical: 1200
+  health_score_degraded: 400
+  max_event_loop_lag_s: 0.5
+  max_blocking_duration_s: 1.0
+  max_queue_wait_s: 1.0
+  max_threadpool_backlog: 3
+  max_active_requests: 150
+```
+
+---
+
+# Output Formats
+
+The CLI supports:
+
+- human-readable terminal output  
+- structured JSON output for CI pipelines  
+
+---
+
+## Standard Text Output
+
+```bash
+async-auditor --output-format text
+```
+
+---
+
+## JSON Output
+
+```bash
+async-auditor --output-format json
+```
+
+A structured report is also written to:
+
+```text
+audit_results.json
+```
+
+inside the current working directory.
+
+---
+
+# CI/CD Integration Example
+
+A reference GitHub Actions workflow is included in:
+
+```text
+examples/github-actions-gating.yml
+```
+
+This demonstrates how to use the auditor as a deployment quality gate inside pull request pipelines.
+
+---
+
+# Operational Model
+
+The auditor computes a deterministic runtime health score using:
+
+- event-loop lag  
+- blocking duration  
+- queue wait amplification  
+- active request pressure  
+- threadpool backlog behavior  
+
+The scoring model is intentionally threshold-driven and explainable.
+
+This project does not use:
+
+- machine learning classification  
+- anomaly detection systems  
+- probabilistic runtime forecasting  
+
+---
+
+# Intended Usage
+
+This tool is designed primarily for:
+
+- deployment pipeline gating  
+- staging environment validation  
+- runtime regression detection  
+- async infrastructure diagnostics  
+
+It is not intended to replace full observability platforms.
+
+---
+
+# Scope
+
+This project is intentionally narrow in scope.
+
+## Included
+
+- heuristic-based async runtime failure classification  
+- CI/CD exit-code semantics  
+- configurable operational thresholds  
+- Prometheus-compatible telemetry querying  
+- runtime degradation detection  
+
+---
+
+## Not Included
+
+- distributed tracing infrastructure  
+- observability data storage systems  
+- automated remediation  
+- Kubernetes orchestration  
+- OpenTelemetry collectors  
+- AI-driven diagnosis systems  
+
+---
+
+# Design Constraints
+
+The project intentionally prioritizes:
+
+- operational clarity  
+- deterministic output  
+- explainable runtime scoring  
+- low dependency overhead  
+- simple deployment integration  
+
+over:
+
+- infrastructure breadth  
+- platform extensibility  
+- distributed orchestration  
+- autonomous remediation  
+
+---
+
+# License
+
+MIT License

async_runtime_auditor-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+async_auditor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+async_auditor/cli.py,sha256=8rc6nfgsmxXxuWJUaRdpVoqQuIfpJ6rq4BtsZmZ376E,13540
+async_auditor/default_metrics.yaml,sha256=L2fcOA3KCmr7N7mUy3zub_uNLOT0EdmGtmoBvB-7CYs,933
+async_runtime_auditor-0.1.0.dist-info/licenses/LICENSE,sha256=ISMzH0EMJ7qQm0voHwx5hn6p0MFFWcFMcuSDtgn5Vqs,1093
+async_runtime_auditor-0.1.0.dist-info/METADATA,sha256=LgVqYRGZfYlPzhbpWNL2koBPHgtPbFMCQAG7hRtuJgs,6042
+async_runtime_auditor-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+async_runtime_auditor-0.1.0.dist-info/entry_points.txt,sha256=kmLXYS_YPMeT6O3yAzulN1qweMu5EM6TGxQRjD9lLeM,57
+async_runtime_auditor-0.1.0.dist-info/top_level.txt,sha256=hEkLzKd3KM2TVu3zbh0f8uqEWq6Ns2m7vtOfvgFNe-U,14
+async_runtime_auditor-0.1.0.dist-info/RECORD,,

async_runtime_auditor-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

async_runtime_auditor-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+async-auditor = async_auditor.cli:main

async_runtime_auditor-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 PRIYANSHU KUMAR
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

async_runtime_auditor-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+async_auditor