queue-aiops 0.1.0__py3-none-any.whl

queue_aiops/ops/overview.py

@@ -0,0 +1,90 @@
+"""One-shot broker overview (read-only).
+
+A single call an operator can lead with, platform-dispatched: for a redis
+target — version, memory posture, clients, ops/sec, hit rate; for a rabbitmq
+target — version, queue/message totals, connection/channel counts, rates.
+Resilient — a failing sub-call degrades to a partial summary with an
+``errors`` list.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from queue_aiops.ops import rabbit_reads, redis_reads
+from queue_aiops.platform import REDIS
+
+
+def queue_overview(conn: Any) -> dict:
+    """[READ] Summary: platform + version + backlog/memory/client health."""
+    if conn.target.platform == REDIS:
+        return _redis_overview(conn)
+    return _rabbitmq_overview(conn)
+
+
+def _redis_overview(conn: Any) -> dict:
+    errors: list[str] = []
+
+    info = redis_reads.server_info(conn)
+    if "error" in info:
+        errors.append(f"server: {info['error']}")
+        info = {}
+    mem = redis_reads.memory_stats(conn)
+    if "error" in mem:
+        errors.append(f"memory: {mem['error']}")
+        mem = {}
+    keys = redis_reads.keyspace(conn)
+    if "error" in keys:
+        errors.append(f"keyspace: {keys['error']}")
+        keys = {}
+
+    return {
+        "platform": conn.target.platform,
+        "target": conn.target.name,
+        "version": info.get("version"),
+        "role": info.get("role"),
+        "uptimeSeconds": info.get("uptimeSeconds"),
+        "connectedClients": info.get("connectedClients"),
+        "blockedClients": info.get("blockedClients"),
+        "opsPerSec": info.get("opsPerSec"),
+        "hitRatePct": info.get("hitRatePct"),
+        "usedMemoryBytes": mem.get("usedBytes"),
+        "maxmemoryBytes": mem.get("maxmemoryBytes"),
+        "usedPctOfMax": mem.get("usedPctOfMax"),
+        "maxmemoryPolicy": mem.get("maxmemoryPolicy"),
+        "fragmentationRatio": mem.get("fragmentationRatio"),
+        "totalKeys": keys.get("totalKeys"),
+        "errors": errors,
+    }
+
+
+def _rabbitmq_overview(conn: Any) -> dict:
+    errors: list[str] = []
+
+    ov = rabbit_reads.broker_overview(conn)
+    if "error" in ov:
+        errors.append(f"overview: {ov['error']}")
+        ov = {}
+    nodes = rabbit_reads.node_health(conn)
+    node_list = nodes.get("nodes", []) if "error" not in nodes else []
+    if "error" in nodes:
+        errors.append(f"nodes: {nodes['error']}")
+
+    return {
+        "platform": conn.target.platform,
+        "target": conn.target.name,
+        "version": ov.get("version"),
+        "clusterName": ov.get("clusterName"),
+        "queues": ov.get("queues"),
+        "messagesReady": ov.get("messagesReady"),
+        "messagesUnacked": ov.get("messagesUnacked"),
+        "connections": ov.get("connections"),
+        "channels": ov.get("channels"),
+        "consumers": ov.get("consumers"),
+        "publishRate": ov.get("publishRate"),
+        "deliverRate": ov.get("deliverRate"),
+        "nodesTotal": len(node_list),
+        "nodesRunning": sum(1 for n in node_list if n.get("running")),
+        "nodeAlarms": nodes.get("alarms") if "error" not in nodes else None,
+        "errors": errors,
+    }

queue_aiops/ops/rabbit_reads.py

@@ -0,0 +1,218 @@
+"""rabbitmq reads — overview, queues, connections, channels, policies, nodes.
+
+The day-to-day "is this broker healthy?" surface for rabbitmq targets, all via
+the management HTTP API. Every call is resilient — a transport/parse failure
+surfaces as ``{"error": ...}`` instead of raising — and all broker text is
+sanitised via ``s`` / the platform normaliser.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from queue_aiops.ops._util import as_obj, num, pick, rate, s
+
+MAX_ROWS = 200
+
+
+def broker_overview(conn: Any) -> dict:
+    """[READ] Broker identity + totals from /api/overview."""
+    try:
+        obj = as_obj(conn.platform.normalise(conn.get(conn.platform.path("overview"))))
+        totals = as_obj(obj.get("queue_totals"))
+        objects = as_obj(obj.get("object_totals"))
+        stats = as_obj(obj.get("message_stats"))
+        churn = as_obj(obj.get("churn_rates"))
+        return {
+            "platform": conn.target.platform,
+            "version": s(pick(obj, "rabbitmq_version", "product_version")),
+            "clusterName": s(obj.get("cluster_name")),
+            "node": s(obj.get("node")),
+            "messagesReady": num(totals.get("messages_ready")),
+            "messagesUnacked": num(totals.get("messages_unacknowledged")),
+            "messagesTotal": num(totals.get("messages")),
+            "queues": num(objects.get("queues")),
+            "connections": num(objects.get("connections")),
+            "channels": num(objects.get("channels")),
+            "consumers": num(objects.get("consumers")),
+            "publishRate": rate(stats.get("publish_details")),
+            "deliverRate": rate(stats.get("deliver_get_details")),
+            "ackRate": rate(stats.get("ack_details")),
+            "connectionChurn": {
+                "createdRate": rate(churn.get("connection_created_details")),
+                "closedRate": rate(churn.get("connection_closed_details")),
+                "channelCreatedRate": rate(churn.get("channel_created_details")),
+                "channelClosedRate": rate(churn.get("channel_closed_details")),
+            },
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def _queue_row(q: dict) -> dict:
+    stats = as_obj(q.get("message_stats"))
+    return {
+        "name": s(q.get("name"), 128),
+        "vhost": s(q.get("vhost"), 64),
+        "state": s(q.get("state"), 32),
+        "durable": bool(q.get("durable")),
+        "messages": num(q.get("messages")),
+        "messagesReady": num(q.get("messages_ready")),
+        "messagesUnacked": num(q.get("messages_unacknowledged")),
+        "consumers": num(q.get("consumers")),
+        "memoryBytes": num(q.get("memory")),
+        "publishRate": rate(stats.get("publish_details")),
+        "deliverRate": rate(stats.get("deliver_get_details")),
+        "ackRate": rate(stats.get("ack_details")),
+    }
+
+
+def list_queues(conn: Any, vhost: str | None = None) -> dict:
+    """[READ] Queues (optionally one vhost), deepest backlog first."""
+    try:
+        path = (
+            conn.platform.path("queues_vhost", vhost=vhost)
+            if vhost
+            else conn.platform.path("queues")
+        )
+        rows = [_queue_row(q) for q in conn.platform.rows(conn.get(path))]
+        rows.sort(key=lambda r: r["messages"], reverse=True)
+        return {"total": len(rows), "queues": rows[:MAX_ROWS]}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def queue_detail(conn: Any, vhost: str, name: str) -> dict:
+    """[READ] One queue's full detail (rates, consumers, memory, args)."""
+    try:
+        obj = as_obj(
+            conn.platform.normalise(
+                conn.get(conn.platform.path("queue", vhost=vhost, name=name))
+            )
+        )
+        row = _queue_row(obj)
+        row.update(
+            {
+                "autoDelete": bool(obj.get("auto_delete")),
+                "exclusive": bool(obj.get("exclusive")),
+                "arguments": as_obj(obj.get("arguments")),
+                "node": s(obj.get("node"), 64),
+                "idleSince": s(obj.get("idle_since"), 64),
+                "consumerUtilisation": num(
+                    pick(obj, "consumer_capacity", "consumer_utilisation", default=0)
+                ),
+            }
+        )
+        return row
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def list_connections(conn: Any) -> dict:
+    """[READ] Client connections grouped by peer host, busiest first."""
+    try:
+        rows = [
+            {
+                "name": s(c.get("name"), 128),
+                "peerHost": s(pick(c, "peer_host", default="(unknown)"), 64),
+                "user": s(c.get("user"), 64),
+                "state": s(c.get("state"), 32),
+                "channels": num(c.get("channels")),
+                "connectedAt": num(c.get("connected_at")),
+                "clientProduct": s(as_obj(c.get("client_properties")).get("product"), 64),
+            }
+            for c in conn.platform.rows(conn.get(conn.platform.path("connections")))
+        ]
+        by_host: dict[str, dict] = {}
+        for r in rows:
+            bucket = by_host.setdefault(
+                r["peerHost"], {"peerHost": r["peerHost"], "connections": 0, "channels": 0.0}
+            )
+            bucket["connections"] += 1
+            bucket["channels"] += r["channels"]
+        sources = sorted(by_host.values(), key=lambda e: e["connections"], reverse=True)
+        return {
+            "total": len(rows),
+            "byPeerHost": sources[:50],
+            "connections": rows[:MAX_ROWS],
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def list_channels(conn: Any) -> dict:
+    """[READ] Channels with unacked/prefetch pressure, most unacked first."""
+    try:
+        rows = [
+            {
+                "name": s(c.get("name"), 128),
+                "connection": s(as_obj(c.get("connection_details")).get("name"), 128),
+                "user": s(c.get("user"), 64),
+                "state": s(c.get("state"), 32),
+                "unacked": num(c.get("messages_unacknowledged")),
+                "prefetch": num(c.get("prefetch_count")),
+                "consumers": num(c.get("consumer_count")),
+            }
+            for c in conn.platform.rows(conn.get(conn.platform.path("channels")))
+        ]
+        rows.sort(key=lambda r: r["unacked"], reverse=True)
+        return {"total": len(rows), "channels": rows[:MAX_ROWS]}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def list_policies(conn: Any, vhost: str | None = None) -> dict:
+    """[READ] Policies (optionally one vhost)."""
+    try:
+        path = (
+            conn.platform.path("policies_vhost", vhost=vhost)
+            if vhost
+            else conn.platform.path("policies")
+        )
+        rows = [
+            {
+                "name": s(p.get("name"), 128),
+                "vhost": s(p.get("vhost"), 64),
+                "pattern": s(p.get("pattern"), 128),
+                "applyTo": s(pick(p, "apply-to", "apply_to", default="all"), 32),
+                "priority": num(p.get("priority")),
+                "definition": as_obj(p.get("definition")),
+            }
+            for p in conn.platform.rows(conn.get(path))
+        ]
+        return {"total": len(rows), "policies": rows[:MAX_ROWS]}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def node_health(conn: Any) -> dict:
+    """[READ] Node memory/disk/fd posture + alarms, most loaded first."""
+    try:
+        rows = []
+        for n in conn.platform.rows(conn.get(conn.platform.path("nodes"))):
+            mem_used = num(n.get("mem_used"))
+            mem_limit = num(n.get("mem_limit"))
+            rows.append(
+                {
+                    "name": s(n.get("name"), 64),
+                    "running": bool(n.get("running", True)),
+                    "memUsedBytes": mem_used,
+                    "memLimitBytes": mem_limit,
+                    "memAlarm": bool(n.get("mem_alarm")),
+                    "diskFreeBytes": num(n.get("disk_free")),
+                    "diskFreeLimitBytes": num(n.get("disk_free_limit")),
+                    "diskAlarm": bool(n.get("disk_free_alarm")),
+                    "fdUsed": num(n.get("fd_used")),
+                    "fdTotal": num(n.get("fd_total")),
+                    "socketsUsed": num(n.get("sockets_used")),
+                    "socketsTotal": num(n.get("sockets_total")),
+                }
+            )
+        rows.sort(key=lambda r: r["memUsedBytes"], reverse=True)
+        return {
+            "total": len(rows),
+            "alarms": sum(1 for r in rows if r["memAlarm"] or r["diskAlarm"]),
+            "nodes": rows,
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}

queue_aiops/ops/redis_reads.py

@@ -0,0 +1,237 @@
+"""redis reads — INFO, memory, clients, slowlog, config, keyspace, big keys.
+
+The day-to-day "is this cache healthy?" surface for redis targets. Every call
+is resilient — a transport/parse failure surfaces as ``{"error": ...}`` instead
+of raising — and all broker text is sanitised via ``s``.
+
+Big-key sampling is **SCAN-based with a hard budget** (never ``KEYS *``): at
+most ``SCAN_BUDGET_KEYS`` keys are walked in ``SCAN_PAGE``-sized pages and at
+most ``MEMORY_SAMPLE_MAX`` of them are sized with ``MEMORY USAGE``, so the read
+can never stall a production instance. Coverage is reported so the caller knows
+how partial the sample is.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from queue_aiops.ops._util import as_obj, num, pct, s
+
+# Hard bounds for the SCAN-based big-key sample (never KEYS *).
+SCAN_BUDGET_KEYS = 10_000  # max keys walked per call
+SCAN_PAGE = 500  # SCAN COUNT hint per page
+MEMORY_SAMPLE_MAX = 200  # max MEMORY USAGE calls per big-key sample
+TOP_KEYS = 20  # rows returned, largest first
+
+MAX_CLIENT_ROWS = 200
+MAX_SLOWLOG_ROWS = 128
+
+
+def server_info(conn: Any) -> dict:
+    """[READ] Server identity + health basics from INFO server/clients/stats."""
+    try:
+        info = as_obj(conn.redis_info())
+        return {
+            "platform": conn.target.platform,
+            "version": s(info.get("redis_version")),
+            "mode": s(info.get("redis_mode")),
+            "uptimeSeconds": num(info.get("uptime_in_seconds")),
+            "role": s(info.get("role")),
+            "connectedClients": num(info.get("connected_clients")),
+            "blockedClients": num(info.get("blocked_clients")),
+            "opsPerSec": num(info.get("instantaneous_ops_per_sec")),
+            "totalCommands": num(info.get("total_commands_processed")),
+            "keyspaceHits": num(info.get("keyspace_hits")),
+            "keyspaceMisses": num(info.get("keyspace_misses")),
+            "hitRatePct": pct(
+                num(info.get("keyspace_hits")),
+                num(info.get("keyspace_hits")) + num(info.get("keyspace_misses")),
+            ),
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def memory_stats(conn: Any) -> dict:
+    """[READ] Memory posture: used vs maxmemory, policy, fragmentation, peaks."""
+    try:
+        info = as_obj(conn.redis_info("memory"))
+        used = num(info.get("used_memory"))
+        maxmem = num(info.get("maxmemory"))
+        stats = as_obj(conn.redis_memory_stats())
+        return {
+            "usedBytes": used,
+            "usedHuman": s(info.get("used_memory_human")),
+            "maxmemoryBytes": maxmem,
+            "usedPctOfMax": pct(used, maxmem),
+            "maxmemoryPolicy": s(info.get("maxmemory_policy")),
+            "fragmentationRatio": num(info.get("mem_fragmentation_ratio")),
+            "rssBytes": num(info.get("used_memory_rss")),
+            "peakBytes": num(info.get("used_memory_peak")),
+            "keysCount": num(stats.get("keys.count")),
+            "overheadBytes": num(stats.get("overhead.total")),
+            "datasetBytes": num(stats.get("dataset.bytes")),
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def list_clients(conn: Any) -> dict:
+    """[READ] Connected clients grouped by source address, busiest first."""
+    try:
+        clients = conn.redis_client_list()
+        rows = [
+            {
+                "id": s(c.get("id"), 32),
+                "addr": s(c.get("addr"), 64),
+                "name": s(c.get("name"), 64),
+                "ageSeconds": num(c.get("age")),
+                "idleSeconds": num(c.get("idle")),
+                "lastCommand": s(c.get("cmd"), 64),
+                "db": s(c.get("db"), 8),
+                "flags": s(c.get("flags"), 16),
+            }
+            for c in clients
+        ]
+        by_source: dict[str, int] = {}
+        for r in rows:
+            source = r["addr"].rsplit(":", 1)[0] or "(unknown)"
+            by_source[source] = by_source.get(source, 0) + 1
+        sources = sorted(
+            ({"source": k, "clients": v} for k, v in by_source.items()),
+            key=lambda e: e["clients"],
+            reverse=True,
+        )
+        return {
+            "total": len(rows),
+            "bySource": sources[:50],
+            "clients": rows[:MAX_CLIENT_ROWS],
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def _command_text(raw: Any) -> str:
+    """Fold a slowlog command cell (str/bytes/list) into one bounded string."""
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8", "replace")
+    if isinstance(raw, (list, tuple)):
+        raw = " ".join(
+            p.decode("utf-8", "replace") if isinstance(p, bytes) else str(p) for p in raw
+        )
+    return s(raw, 128)
+
+
+def slowlog(conn: Any, count: int = MAX_SLOWLOG_ROWS) -> dict:
+    """[READ] Recent SLOWLOG entries, slowest first."""
+    try:
+        entries = conn.redis_slowlog(min(int(count), MAX_SLOWLOG_ROWS))
+        rows = [
+            {
+                "id": num(e.get("id")),
+                "startTime": num(e.get("start_time")),
+                "durationUs": num(e.get("duration")),
+                "command": _command_text(e.get("command")),
+                "clientAddr": s(e.get("client_address"), 64),
+                "clientName": s(e.get("client_name"), 64),
+            }
+            for e in entries
+            if isinstance(e, dict)
+        ]
+        rows.sort(key=lambda r: r["durationUs"], reverse=True)
+        return {"total": len(rows), "entries": rows}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def config_get(conn: Any, pattern: str = "*") -> dict:
+    """[READ] CONFIG GET for a glob pattern (bounded, values sanitised)."""
+    try:
+        raw = conn.redis_config_get(s(pattern, 64) or "*")
+        params = {s(k, 64): s(v, 128) for k, v in sorted(raw.items())}
+        return {"total": len(params), "parameters": params}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def keyspace(conn: Any) -> dict:
+    """[READ] Per-db key counts + expiry coverage from INFO keyspace."""
+    try:
+        info = as_obj(conn.redis_info("keyspace"))
+        dbs = []
+        for name, cell in info.items():
+            cell = as_obj(cell) if isinstance(cell, dict) else _parse_keyspace_cell(cell)
+            keys = num(cell.get("keys"))
+            expires = num(cell.get("expires"))
+            dbs.append(
+                {
+                    "db": s(name, 16),
+                    "keys": keys,
+                    "expires": expires,
+                    "expiresPct": pct(expires, keys),
+                    "avgTtlMs": num(cell.get("avg_ttl")),
+                }
+            )
+        return {"databases": dbs, "totalKeys": sum(d["keys"] for d in dbs)}
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}
+
+
+def _parse_keyspace_cell(cell: Any) -> dict:
+    """Parse a raw ``keys=1,expires=2,avg_ttl=3`` keyspace string."""
+    out: dict[str, str] = {}
+    for part in str(cell or "").split(","):
+        if "=" in part:
+            k, _, v = part.partition("=")
+            out[k.strip()] = v.strip()
+    return out
+
+
+def big_key_sample(conn: Any, top: int = TOP_KEYS) -> dict:
+    """[READ] SCAN-budgeted big-key sample — largest sampled keys first.
+
+    Walks at most ``SCAN_BUDGET_KEYS`` keys with SCAN (pages of ``SCAN_PAGE``),
+    sizes an evenly-spaced subset of at most ``MEMORY_SAMPLE_MAX`` keys with
+    MEMORY USAGE, and returns the largest ones. Never ``KEYS *`` — the budget
+    makes this safe on a big production keyspace, and ``coveragePct`` says how
+    partial the sample is.
+    """
+    try:
+        dbsize = conn.redis_dbsize()
+        scanned: list[str] = []
+        cursor = 0
+        while True:
+            cursor, keys = conn.redis_scan(cursor=cursor, count=SCAN_PAGE)
+            scanned.extend(keys)
+            if cursor == 0 or len(scanned) >= SCAN_BUDGET_KEYS:
+                break
+        scanned = scanned[:SCAN_BUDGET_KEYS]
+
+        step = max(1, len(scanned) // MEMORY_SAMPLE_MAX)
+        sampled = scanned[::step][:MEMORY_SAMPLE_MAX]
+        sized = []
+        for key in sampled:
+            usage = conn.redis_memory_usage(key)
+            if usage is not None:
+                sized.append({"key": s(key, 128), "bytes": float(usage)})
+        sized.sort(key=lambda e: e["bytes"], reverse=True)
+
+        return {
+            "dbsize": dbsize,
+            "scannedKeys": len(scanned),
+            "sampledKeys": len(sized),
+            "coveragePct": pct(len(scanned), dbsize),
+            "budget": {
+                "scanBudgetKeys": SCAN_BUDGET_KEYS,
+                "scanPage": SCAN_PAGE,
+                "memorySampleMax": MEMORY_SAMPLE_MAX,
+            },
+            "topKeys": sized[: max(1, int(top))],
+            "note": (
+                "SCAN-based sample under a hard budget (never KEYS *): sizes are "
+                "MEMORY USAGE on an evenly-spaced subset; coveragePct shows how "
+                "partial the walk was."
+            ),
+        }
+    except Exception as exc:  # noqa: BLE001 — report as partial
+        return {"error": s(exc, 200)}