stackdoctor 0.1.0__py3-none-any.whl

stackdoctor/checks/postgres.py

@@ -0,0 +1,202 @@
+"""Light Postgres checks. For deep health checks use Postgres MCP Pro.
+
+Every connection is forced read-only with a statement timeout, at the server level.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+
+import psycopg
+from psycopg.rows import dict_row
+
+from ..config import get_config, skipped
+from ..safety import UnsafeError, preview, validate_select
+from . import event, utcnow
+
+SOURCE = "postgres"
+MAX_LIMIT = 1000
+_OPTIONS = "-c default_transaction_read_only=on -c statement_timeout=5000 -c lock_timeout=2000"
+
+
+@contextmanager
+def connect():
+    cfg = get_config()
+    # Keyword args override anything in DATABASE_URL, so these always apply.
+    conn = psycopg.connect(cfg.database_url, options=_OPTIONS, connect_timeout=5,
+                           application_name="stackdoctor", row_factory=dict_row)
+    try:
+        conn.read_only = True
+        ro = conn.execute("SHOW default_transaction_read_only").fetchone()
+        if ro["default_transaction_read_only"] != "on":
+            raise UnsafeError("Could not force a read-only session; refusing to run queries.")
+        yield conn
+    finally:
+        conn.close()
+
+
+def _not_configured():
+    return skipped("DATABASE_URL not set") if not get_config().database_url else None
+
+
+def active_queries(min_duration_s: float = 0) -> dict:
+    if (s := _not_configured()):
+        return s
+    cfg = get_config()
+    with connect() as conn:
+        rows = conn.execute(
+            """
+            SELECT pid, usename, application_name, client_addr::text, state,
+                   wait_event_type, wait_event, backend_type,
+                   xact_start, query_start, state_change,
+                   EXTRACT(EPOCH FROM now() - query_start)::float AS duration_s,
+                   EXTRACT(EPOCH FROM now() - xact_start)::float AS xact_age_s,
+                   query
+            FROM pg_stat_activity
+            WHERE state IS DISTINCT FROM 'idle'
+              AND pid <> pg_backend_pid()
+              AND backend_type = 'client backend'
+              AND COALESCE(EXTRACT(EPOCH FROM now() - query_start), 0) >= %s
+            ORDER BY query_start NULLS LAST
+            LIMIT 50
+            """,
+            (min_duration_s,),
+        ).fetchall()
+
+    events, queries = [], []
+    for r in rows:
+        r["query"] = preview(r["query"], 300)
+        r["duration_s"] = round(r["duration_s"] or 0, 1)
+        r["xact_age_s"] = round(r["xact_age_s"] or 0, 1)
+        queries.append(r)
+        if r["state"] == "idle in transaction" and r["xact_age_s"] >= cfg.long_query_s:
+            events.append(event(r["xact_start"], SOURCE, "idle_in_transaction",
+                                f"pid {r['pid']} idle in transaction for {r['xact_age_s']}s: {r['query']}",
+                                "warning", pid=r["pid"]))
+        elif r["state"] == "active" and r["duration_s"] >= cfg.long_query_s and r["wait_event_type"] != "Lock":
+            events.append(event(r["query_start"], SOURCE, "long_query",
+                                f"pid {r['pid']} running {r['duration_s']}s: {r['query']}",
+                                "warning", pid=r["pid"]))
+    return {"count": len(queries), "queries": queries, "events": events}
+
+
+def blocking_locks() -> dict:
+    if (s := _not_configured()):
+        return s
+    with connect() as conn:
+        blocked = conn.execute(
+            """
+            SELECT a.pid, a.usename, a.application_name, a.query_start,
+                   EXTRACT(EPOCH FROM now() - a.query_start)::float AS waiting_s,
+                   pg_blocking_pids(a.pid) AS blocked_by,
+                   (SELECT string_agg(DISTINCT l.relation::regclass::text, ', ')
+                      FROM pg_locks l WHERE l.pid = a.pid AND NOT l.granted
+                       AND l.relation IS NOT NULL) AS relation,
+                   a.query
+            FROM pg_stat_activity a
+            WHERE cardinality(pg_blocking_pids(a.pid)) > 0
+            ORDER BY a.query_start
+            LIMIT 50
+            """
+        ).fetchall()
+        blocker_pids = sorted({p for r in blocked for p in r["blocked_by"]})
+        blockers = conn.execute(
+            """
+            SELECT pid, usename, application_name, state, xact_start, state_change,
+                   EXTRACT(EPOCH FROM now() - xact_start)::float AS xact_age_s, query
+            FROM pg_stat_activity WHERE pid = ANY(%s)
+            """,
+            (blocker_pids,),
+        ).fetchall() if blocker_pids else []
+
+    events = []
+    for b in blockers:
+        b["query"] = preview(b["query"], 300)
+        b["xact_age_s"] = round(b["xact_age_s"] or 0, 1)
+        waiting = [r["pid"] for r in blocked if b["pid"] in r["blocked_by"]]
+        events.append(event(b["xact_start"], SOURCE, "lock_held",
+                            f"pid {b['pid']} ({b['state']}) holds a lock blocking {len(waiting)} "
+                            f"session(s) since transaction start: {b['query']}",
+                            "critical", pid=b["pid"]))
+    for r in blocked:
+        r["query"] = preview(r["query"], 300)
+        r["waiting_s"] = round(r["waiting_s"] or 0, 1)
+        events.append(event(r["query_start"], SOURCE, "query_blocked",
+                            f"pid {r['pid']} waiting {r['waiting_s']}s on {r['relation'] or 'a lock'} "
+                            f"held by {r['blocked_by']}: {r['query']}",
+                            "warning", pid=r["pid"]))
+    return {"blocked_count": len(blocked), "blocked": blocked, "blockers": blockers, "events": events}
+
+
+_PGSS_HELP = (
+    "pg_stat_statements is not installed in this database. To enable it (needs a superuser, "
+    "stackdoctor will not do it for you): 1) add `shared_preload_libraries = 'pg_stat_statements'` "
+    "to postgresql.conf (or `-c shared_preload_libraries=pg_stat_statements` in docker), "
+    "2) restart Postgres, 3) run `CREATE EXTENSION pg_stat_statements;` in this database. "
+    "On RDS/Cloud SQL, enable it via the parameter group / database flags."
+)
+
+
+def slow_queries(limit: int = 10) -> dict:
+    if (s := _not_configured()):
+        return s
+    limit = max(1, min(int(limit), 50))
+    with connect() as conn:
+        has_ext = conn.execute(
+            "SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'").fetchone()
+        seq = conn.execute(
+            """
+            SELECT schemaname || '.' || relname AS table, seq_scan, seq_tup_read,
+                   idx_scan, n_live_tup
+            FROM pg_stat_user_tables
+            WHERE seq_scan > 0
+            ORDER BY seq_tup_read DESC LIMIT 5
+            """
+        ).fetchall()
+        result: dict = {"seq_scan_heavy_tables": seq}
+        if not has_ext:
+            result.update(available=False, message=_PGSS_HELP)
+            return result
+        new_cols = conn.info.server_version >= 130000
+        mean, total = ("mean_exec_time", "total_exec_time") if new_cols else ("mean_time", "total_time")
+        try:
+            rows = conn.execute(
+                f"""
+                SELECT queryid, calls, round({mean}::numeric, 1) AS mean_ms,
+                       round({total}::numeric, 1) AS total_ms, rows,
+                       shared_blks_read, query
+                FROM pg_stat_statements
+                WHERE query NOT ILIKE '%%pg_stat_statements%%'
+                ORDER BY {mean} DESC LIMIT %s
+                """,
+                (limit,),
+            ).fetchall()
+        except psycopg.errors.ObjectNotInPrerequisiteState:
+            result.update(available=False, message=_PGSS_HELP)
+            return result
+    events = []
+    for r in rows:
+        r["query"] = preview(r["query"], 300)
+        if r["mean_ms"] >= 1000:
+            events.append(event(None, SOURCE, "slow_statement",
+                                f"{r['calls']} call(s) averaging {r['mean_ms']:.0f}ms: {r['query']}",
+                                "warning", queryid=r["queryid"]))
+    result.update(available=True, queries=rows, events=events,
+                  note="pg_stat_statements is cumulative since the last reset; it has no timestamps.")
+    return result
+
+
+def run_select(sql: str, limit: int = 100) -> dict:
+    if (s := _not_configured()):
+        return s
+    limit = max(1, min(int(limit), MAX_LIMIT))
+    try:
+        final_sql = validate_select(sql, limit)
+    except UnsafeError as e:
+        return {"rejected": True, "reason": str(e)}
+    with connect() as conn:
+        cur = conn.execute(final_sql)
+        rows = cur.fetchmany(limit) if cur.description else []
+        columns = [d.name for d in cur.description or []]
+    return {"columns": columns, "row_count": len(rows), "limit": limit, "rows": rows,
+            "executed_at": utcnow()}

stackdoctor/checks/redis.py

@@ -0,0 +1,154 @@
+"""Redis checks. Every command goes through the read-only allowlist."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+import redis as redis_lib
+
+from ..config import get_config, skipped
+from ..safety import UnsafeError, check_redis_command, preview
+from . import event
+
+SOURCE = "redis"
+
+
+class SafeRedis:
+    """A Redis client that can only run allowlisted read-only commands."""
+
+    def __init__(self, url: str):
+        self._r = redis_lib.Redis.from_url(
+            url, decode_responses=True, encoding_errors="replace",
+            socket_timeout=5, socket_connect_timeout=3)
+
+    def cmd(self, *args):
+        check_redis_command(*args)
+        return self._r.execute_command(*args)
+
+    def scan_iter(self, match: str, limit: int, count: int = 500, max_rounds: int = 200):
+        """SCAN (never KEYS) until `limit` keys or `max_rounds` round trips."""
+        cursor, found, rounds = 0, [], 0
+        while True:
+            cursor, keys = self.cmd("SCAN", cursor, "MATCH", match, "COUNT", count)
+            found.extend(keys)
+            rounds += 1
+            if int(cursor) == 0 or len(found) >= limit or rounds >= max_rounds:
+                return found[:limit], int(cursor) == 0
+
+
+@lru_cache(maxsize=4)
+def client(url: str) -> SafeRedis:
+    return SafeRedis(url)
+
+
+def glob_escape(text: str) -> str:
+    return "".join("\\" + c if c in "*?[]\\" else c for c in text)
+
+
+def memory_and_clients() -> dict:
+    cfg = get_config()
+    if not cfg.redis_url:
+        return skipped("REDIS_URL not set")
+    r = client(cfg.redis_url)
+    info = r.cmd("INFO")
+    used, maxmem = info.get("used_memory", 0), info.get("maxmemory", 0)
+    try:
+        maxclients = int(r.cmd("CONFIG GET", "maxclients").get("maxclients", 0))
+    except redis_lib.ResponseError:  # CONFIG is often disabled on managed Redis
+        maxclients = None
+    pct = round(used * 100 / maxmem, 1) if maxmem else None
+    result = {
+        "redis_version": info.get("redis_version"),
+        "uptime_s": info.get("uptime_in_seconds"),
+        "memory": {
+            "used": info.get("used_memory_human"), "used_bytes": used,
+            "peak": info.get("used_memory_peak_human"),
+            "maxmemory_bytes": maxmem or None, "used_pct_of_max": pct,
+            "maxmemory_policy": info.get("maxmemory_policy"),
+            "fragmentation_ratio": info.get("mem_fragmentation_ratio"),
+        },
+        "clients": {
+            "connected": info.get("connected_clients"), "blocked": info.get("blocked_clients"),
+            "maxclients": maxclients, "rejected_connections": info.get("rejected_connections"),
+        },
+        "stats": {k: info.get(k) for k in ("evicted_keys", "expired_keys", "keyspace_hits",
+                                           "keyspace_misses", "instantaneous_ops_per_sec")},
+        "keyspace": {k: v for k, v in info.items() if k.startswith("db") and isinstance(v, dict)},
+    }
+
+    events = []
+    if pct is not None and pct >= cfg.redis_mem_warn_pct:
+        events.append(event(None, SOURCE, "memory_high",
+                            f"Redis memory at {pct}% of maxmemory ({info.get('used_memory_human')}, "
+                            f"policy {info.get('maxmemory_policy')})", "critical" if pct >= 95 else "warning"))
+    if info.get("evicted_keys"):
+        events.append(event(None, SOURCE, "evictions",
+                            f"{info['evicted_keys']} keys evicted since start (policy {info.get('maxmemory_policy')})",
+                            "warning"))
+    if info.get("rejected_connections"):
+        events.append(event(None, SOURCE, "rejected_connections",
+                            f"{info['rejected_connections']} connections rejected (maxclients {maxclients})",
+                            "warning"))
+    try:
+        slow = r.cmd("SLOWLOG GET", 5)
+    except redis_lib.ResponseError:
+        slow = []
+    result["slowlog"] = [{"ts": s.get("start_time"), "duration_us": s.get("duration"),
+                          "command": preview(s.get("command"), 120)} for s in slow]
+    for s in result["slowlog"]:
+        if (s["duration_us"] or 0) >= 100_000:
+            events.append(event(s["ts"], SOURCE, "slow_command",
+                                f"slow Redis command {s['duration_us'] / 1000:.0f}ms: {s['command']}", "info"))
+    result["events"] = events
+    return result
+
+
+def scan_keys(pattern: str = "*", limit: int = 50) -> dict:
+    cfg = get_config()
+    if not cfg.redis_url:
+        return skipped("REDIS_URL not set")
+    limit = max(1, min(int(limit), 500))
+    keys, complete = client(cfg.redis_url).scan_iter(pattern or "*", limit)
+    return {"pattern": pattern, "count": len(keys), "keys": sorted(keys),
+            "scan_complete": complete,
+            "note": None if complete else "Stopped early (limit reached); more keys may match."}
+
+
+def key_info(key: str) -> dict:
+    cfg = get_config()
+    if not cfg.redis_url:
+        return skipped("REDIS_URL not set")
+    return describe_key(client(cfg.redis_url), key)
+
+
+def describe_key(r: SafeRedis, key: str) -> dict:
+    ktype = r.cmd("TYPE", key)
+    if ktype == "none":
+        return {"key": key, "exists": False}
+    out = {"key": key, "exists": True, "type": ktype, "ttl_ms": r.cmd("PTTL", key)}
+    for name, args in (("memory_bytes", ("MEMORY USAGE", key)), ("encoding", ("OBJECT ENCODING", key))):
+        try:
+            out[name] = r.cmd(*args)
+        except (redis_lib.ResponseError, UnsafeError):
+            pass
+    if ktype == "string":
+        out["length"] = r.cmd("STRLEN", key)
+        out["preview"] = preview(r.cmd("GETRANGE", key, 0, 300), 200)
+    elif ktype == "list":
+        out["length"] = r.cmd("LLEN", key)
+        out["preview"] = [preview(v, 200) for v in r.cmd("LRANGE", key, 0, 2)]
+    elif ktype == "hash":
+        out["length"] = r.cmd("HLEN", key)
+        _, items = r.cmd("HSCAN", key, 0, "COUNT", 10)
+        out["preview"] = {preview(k, 60): preview(v, 120) for k, v in list(items.items())[:10]}
+    elif ktype == "set":
+        out["length"] = r.cmd("SCARD", key)
+        _, members = r.cmd("SSCAN", key, 0, "COUNT", 10)
+        out["preview"] = [preview(m, 120) for m in members[:10]]
+    elif ktype == "zset":
+        out["length"] = r.cmd("ZCARD", key)
+        flat = r.cmd("ZRANGE", key, 0, 4, "WITHSCORES")
+        out["preview"] = [[preview(m, 120), s] for m, s in zip(flat[::2], flat[1::2])]
+    elif ktype == "stream":
+        out["length"] = r.cmd("XLEN", key)
+    return out

stackdoctor/config.py

@@ -0,0 +1,80 @@
+"""Configuration from environment variables (and an optional .env file)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from functools import lru_cache
+
+from dotenv import find_dotenv, load_dotenv
+
+
+def _float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, default))
+    except ValueError:
+        return default
+
+
+def _list(name: str) -> list[str]:
+    return [s.strip() for s in os.environ.get(name, "").split(",") if s.strip()]
+
+
+@dataclass(frozen=True)
+class Config:
+    database_url: str | None = None
+    redis_url: str | None = None
+    celery_broker_url: str | None = None
+    celery_result_backend: str | None = None
+    celery_app: str | None = None
+    celery_queues: list[str] = field(default_factory=list)
+    log_sources: list[str] = field(default_factory=list)
+
+    # Heuristic thresholds
+    queue_threshold: int = 100
+    expected_workers: int = 0
+    long_task_s: float = 60
+    long_query_s: float = 30
+    redis_mem_warn_pct: float = 85
+    log_error_spike: int = 10
+    chain_window_min: float = 30
+
+    # Limits
+    check_timeout_s: float = 8
+    celery_inspect_timeout_s: float = 1.0
+    max_output_chars: int = 20_000
+
+    @classmethod
+    def from_env(cls) -> "Config":
+        # Real environment variables win over the .env file.
+        load_dotenv(os.environ.get("STACKDOCTOR_ENV_FILE") or find_dotenv(usecwd=True))
+        get = lambda k: os.environ.get(k) or None  # noqa: E731
+        return cls(
+            database_url=get("DATABASE_URL"),
+            redis_url=get("REDIS_URL"),
+            celery_broker_url=get("CELERY_BROKER_URL"),
+            celery_result_backend=get("CELERY_RESULT_BACKEND"),
+            celery_app=get("CELERY_APP"),
+            celery_queues=_list("CELERY_QUEUES"),
+            log_sources=_list("LOG_SOURCES"),
+            queue_threshold=int(_float("QUEUE_THRESHOLD", 100)),
+            expected_workers=int(_float("EXPECTED_WORKERS", 0)),
+            long_task_s=_float("LONG_TASK_S", 60),
+            long_query_s=_float("LONG_QUERY_S", 30),
+            redis_mem_warn_pct=_float("REDIS_MEM_WARN_PCT", 85),
+            log_error_spike=int(_float("LOG_ERROR_SPIKE", 10)),
+            chain_window_min=_float("CHAIN_WINDOW_MIN", 30),
+            check_timeout_s=_float("CHECK_TIMEOUT_S", 8),
+            celery_inspect_timeout_s=_float("CELERY_INSPECT_TIMEOUT_S", 1.0),
+            max_output_chars=int(_float("MAX_OUTPUT_CHARS", 20_000)),
+        )
+
+
+@lru_cache(maxsize=1)
+def get_config() -> Config:
+    return Config.from_env()
+
+
+def skipped(reason: str) -> dict:
+    """Standard result for a check whose source isn't configured."""
+    return {"skipped": reason}