holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/toolsets/prometheus/prometheus_instructions.jinja2

@@ -0,0 +1,38 @@
+
+# Prometheus/PromQL queries
+* ALWAYS call list_prometheus_rules to get the alert definition
+* Use Prometheus to query metrics from the alert promql
+* Use prometheus to execute promql queries with the tools `execute_prometheus_instant_query` and `execute_prometheus_range_query`
+* To create queries, use 'start_timestamp' and 'end_timestamp' as graphs start and end times
+* ALWAYS embed the execution results into your answer
+* You only need to embed the partial result in your response. Include the "tool_name" and "random_key". For example: << {"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "92jf2hf"} >>
+* Use these tools to generate charts that users can see. Here are standard metrics but you can use different ones:
+** For memory consumption: `container_memory_working_set_bytes`
+** For CPU usage: `container_cpu_usage_seconds_total`
+** For CPU throttling: `container_cpu_cfs_throttled_periods_total`
+** For latencies, prefer using `<metric>_sum` / `<metric>_count` over a sliding window
+** Avoid using `<metric>_bucket` unless you know the bucket's boundaries are configured correctly
+** Prefer individual averages like `rate(<metric>_sum) / rate(<metric>_count)`
+** Avoid global averages like `sum(rate(<metric>_sum)) / sum(rate(<metric>_count))` because it hides data and is not generally informative
+* Timestamps MUST be in string date format. For example: '2025-03-15 10:10:08.610862+00:00'
+* Post processing will parse your response, re-run the query from the tool output and create a chart visible to the user
+* Only generate and execute a prometheus query after checking what metrics are available with the `list_available_metrics` tool
+* Check that any node, service, pod, container, app, namespace, etc. mentioned in the query exist in the kubernetes cluster before making a query. Use any appropriate kubectl tool(s) for this
+* The toolcall will return no data to you. That is expected. You MUST however ensure that the query is successful.
+* When doing queries, always extend the time range, to 15 min before and after the alert start time
+* ALWAYS embed the execution results into your answer
+* ALWAYS embed a Prometheus graph in the response. The graph should visualize data related to the incident.
+* Embed at most 2 graphs
+* When embedding multiple graphs, always add line spacing between them
+    For example:
+    ```
+    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "lBaA"}>>
+
+    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "IKtq"}>>
+    ```
+{%- if config and config.additional_labels and config.additional_labels.keys()|list|length > 0 %}
+* ALWAYS add the following additional labels to ALL PromQL queries:
+{%- for key, value in config.additional_labels.items() %}
+  * {{ key }}="{{ value }}"
+{%- endfor -%}
+{%- endif -%}

holmes/plugins/toolsets/rabbitmq/api.py

@@ -0,0 +1,398 @@
+from enum import Enum
+import logging
+from typing import Any, Dict, List, Optional, Set
+from urllib.parse import urljoin, urlparse
+
+import backoff
+from pydantic import BaseModel
+import requests  # type: ignore
+from requests.auth import HTTPBasicAuth  # type: ignore
+
+# --- Enums and Pydantic Models (Mostly Unchanged) ---
+
+
+class ClusterConnectionStatus(str, Enum):
+    SUCCESS = "success"
+    ERROR = "error"
+
+
+class RabbitMQClusterConfig(BaseModel):
+    id: str = "rabbitmq"  # must be unique
+    management_url: str  # e.g., http://rabbitmq-service:15672
+    username: Optional[str] = None
+    password: Optional[str] = None
+    request_timeout_seconds: int = 30
+    verify_certs: bool = True
+
+    # For internal use
+    connection_status: Optional[ClusterConnectionStatus] = None
+    connection_error: Optional[str] = None
+
+
+class Partition(BaseModel):
+    node: str
+    unreachable_nodes: List[str]  # Nodes that 'node' cannot reach
+
+
+class NodeStatus(BaseModel):
+    node: str
+    running: bool  # Status as reported by the primary connected node
+
+
+class NodeInfo(BaseModel):
+    name: Optional[str] = "unknown"
+    type: Optional[str] = "unknown"
+    running: bool = False
+    mem_used: Optional[int] = None
+    mem_limit: Optional[int] = None
+    mem_alarm: Optional[bool] = None
+    disk_free: Optional[int] = None
+    disk_free_limit: Optional[int] = None
+    disk_free_alarm: Optional[bool] = None
+    fd_used: Optional[int] = None
+    fd_total: Optional[int] = None
+    sockets_used: Optional[int] = None
+    sockets_total: Optional[int] = None
+    uptime: Optional[int] = None
+    partitions: Optional[List[Any]] = None
+    error: Optional[str] = None
+
+
+class ClusterStatus(BaseModel):
+    nodes: List[NodeStatus]  # Overall node running status from primary view
+    network_partitions_detected: bool = False
+    partition_details: List[Partition]  # Combined list of detected partitions
+    raw_node_data: List[NodeInfo]  # Data from the primary connected node
+
+
+# --- Helper Functions (Slight modifications) ---
+
+
+def get_auth(config: RabbitMQClusterConfig) -> Optional[HTTPBasicAuth]:
+    if config.username or config.password:
+        return HTTPBasicAuth(
+            config.username or "guest",
+            config.password or "guest",
+        )
+    else:
+        return None
+
+
+def get_url(base_url: str, endpoint: str) -> str:
+    """Constructs a URL using a base and an endpoint."""
+    # Ensure base_url ends with '/' for urljoin to work predictably
+    if not base_url.endswith("/"):
+        base_url += "/"
+    return urljoin(base_url, endpoint.lstrip("/"))
+
+
+@backoff.on_exception(
+    backoff.expo,
+    requests.exceptions.RequestException,
+    max_tries=3,
+    giveup=lambda e: isinstance(e, requests.exceptions.HTTPError)
+    and e.response.status_code < 500,
+)
+def make_request(
+    config: RabbitMQClusterConfig,
+    method: str,
+    url: str,
+    params: Optional[Dict] = None,
+    data: Optional[Dict] = None,
+) -> Any:
+    """Makes an HTTP request to the RabbitMQ Management API."""
+    headers = {"Content-Type": "application/json"}
+    try:
+        response = requests.request(
+            method=method,
+            url=url,
+            headers=headers,
+            auth=get_auth(config),
+            params=params,
+            json=data,
+            timeout=config.request_timeout_seconds,
+            verify=config.verify_certs,
+        )
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Request failed for {method} {url}: {e}")
+        raise  # Re-raise after logging for upstream handling
+
+
+def node_data_to_node_info(node_data: Dict) -> NodeInfo:
+    """Converts raw node data dict to NodeInfo model."""
+    return NodeInfo(**node_data)
+
+
+def get_status_from_node(
+    config: RabbitMQClusterConfig, target_node_name: str
+) -> Optional[List[Dict]]:
+    """
+    Attempts to connect directly to the management API of a specific node.
+    Returns the raw node list from that node's perspective, or None on failure.
+    """
+    try:
+        # Extract hostname from node name (e.g., rabbit@hostname -> hostname)
+        parts = target_node_name.split("@")
+        if len(parts) != 2:
+            logging.debug(
+                f"Could not parse hostname from node name: {target_node_name}"
+            )
+            return None
+        hostname = parts[1]
+
+        # Construct the target node's management URL based on the original config's scheme/port
+        parsed_original_url = urlparse(config.management_url)
+        scheme = parsed_original_url.scheme or "http"
+        port = parsed_original_url.port or (
+            443 if scheme == "https" else 15672
+        )  # Default ports
+        base_target_url = f"{scheme}://{hostname}:{port}"
+
+        target_api_url = get_url(base_target_url, "api/nodes")
+        logging.debug(
+            f"Attempting direct connection to node {target_node_name} via {target_api_url}"
+        )
+
+        # Use the original config for auth, timeout, cert verification etc.
+        data = make_request(
+            config=config,
+            method="GET",
+            url=target_api_url,
+        )
+        # Ensure data is a list as expected from /api/nodes
+        if isinstance(data, list):
+            return data
+        else:
+            logging.debug(
+                f"Unexpected data format received from {target_api_url}: {type(data)}"
+            )
+            return None
+
+    except requests.exceptions.RequestException as e:
+        logging.debug(
+            f"Failed to directly connect to node {target_node_name} via management API: {e}"
+        )
+        return None
+    except Exception:
+        logging.debug(
+            f"Unexpected error trying to get status from node {target_node_name}",
+            exc_info=True,
+        )
+        return None
+
+
+# --- Main Logic Function (Updated) ---
+
+
+def find_node(nodes: List[NodeInfo], node_name: str) -> Optional[NodeInfo]:
+    for node in nodes:
+        if node.name == node_name:
+            return node
+    return None
+
+
+def upsert(nodes: List[NodeInfo], new_node_info: NodeInfo):
+    found_index = -1
+    for i, existing_node in enumerate(nodes):
+        if existing_node.name == new_node_info.name:
+            found_index = i
+            break
+
+    if found_index != -1:
+        nodes[found_index] = new_node_info
+    else:
+        nodes.append(new_node_info)
+
+
+def get_cluster_status(config: RabbitMQClusterConfig) -> ClusterStatus:
+    """
+    Gets cluster status, attempting direct connection to nodes reported as down
+    to detect potential hidden partitions.
+    """
+    raw_nodes_data: List[Dict] = []
+    try:
+        url = get_url(config.management_url, "api/nodes")
+        raw_nodes_data = make_request(
+            config=config,
+            method="GET",
+            url=url,
+        )
+        config.connection_status = ClusterConnectionStatus.SUCCESS
+        config.connection_error = None
+    except Exception as e:
+        logging.error(
+            f"Failed to get primary cluster status from {config.management_url}: {e}"
+        )
+        config.connection_status = ClusterConnectionStatus.ERROR
+        config.connection_error = str(e)
+        # Return an empty/error status if the primary connection fails
+        return ClusterStatus(
+            nodes=[],
+            network_partitions_detected=False,  # Cannot determine
+            partition_details=[],
+            raw_node_data=[],
+        )
+
+    # Process data from the primary connected node
+    detected_partitions: List[Partition] = []
+    primary_nodes: List[NodeInfo] = []
+    nodes_reported_down: Set[str] = set()
+    all_node_names_primary_view: Set[str] = set()
+
+    for node_data in raw_nodes_data:
+        node_info = node_data_to_node_info(node_data)
+        if not node_info.name:
+            continue
+
+        primary_nodes.append(node_info)
+        all_node_names_primary_view.add(node_info.name)
+
+        # Store partitions reported by RabbitMQ itself
+        if node_info.partitions:
+            # Ensure we don't add duplicates if multiple nodes report the same partition
+            partition_exists = any(
+                p.node == node_info.name
+                and set(p.unreachable_nodes) == set(node_info.partitions)
+                for p in detected_partitions
+            )
+            if not partition_exists:
+                detected_partitions.append(
+                    Partition(
+                        node=node_info.name,
+                        unreachable_nodes=list(node_info.partitions),
+                    )
+                )
+
+        # Keep track of nodes reported as down for later direct checks
+        if not node_info.running:
+            nodes_reported_down.add(node_info.name)
+
+    # --- Enhanced Partition Detection ---
+    artificially_detected_partitions: List[Partition] = []
+    for node_name in nodes_reported_down:
+        logging.debug(
+            f"Node {node_name} reported as down by primary. Attempting direct connection."
+        )
+        # Try connecting directly to the node reported as down
+        direct_nodes_data = get_status_from_node(config, node_name)
+        if not direct_nodes_data:
+            continue
+
+        direct_nodes = [
+            node_data_to_node_info(node_data) for node_data in direct_nodes_data
+        ]
+
+        logging.debug(
+            f"Direct connection to {node_name} succeeded. Analyzing its cluster view."
+        )
+        unreachable_by_this_node: List[str] = []
+        this_node = find_node(direct_nodes, node_name)
+
+        if not this_node or not this_node.running:
+            # Ignore this node if it's not running
+            # if this node reports another node as running that was not considered running before, we ignore that
+            # for simplicity as I expect we would get to that node by reaching out directly anyway
+            logging.debug(
+                f"Node {node_name} reported itself as not running upon direct connection."
+            )
+            continue
+        else:
+            # Node is running. Update the primary view with the updated node data
+            logging.info(
+                f"Node {node_name} reported itself as running upon direct connection. Updating primary view."
+            )
+            upsert(primary_nodes, this_node)
+
+        all_node_names_direct_view: Set[str] = set()
+
+        for node in direct_nodes:
+            all_node_names_direct_view.add(node.name)  # type: ignore
+            if not node.running:
+                unreachable_by_this_node.append(node.name)  # type: ignore
+
+        if unreachable_by_this_node:
+            unreachable_nodes_set = set(unreachable_by_this_node)
+
+            # Check if this specific partition view is already covered by RabbitMQ's reporting
+            is_already_reported = any(
+                partition.node == node_name
+                and set(partition.unreachable_nodes) == unreachable_nodes_set
+                for partition in detected_partitions
+            )
+
+            if not is_already_reported:
+                logging.debug(
+                    f"Artificially detecting partition: Node {node_name} cannot reach {unreachable_nodes_set}"
+                )
+                artificially_detected_partitions.append(
+                    Partition(
+                        node=node_name, unreachable_nodes=list(unreachable_nodes_set)
+                    )
+                )
+
+        # Check for nodes present in primary view but MISSING entirely from this node's direct view
+        missing_nodes = all_node_names_primary_view - all_node_names_direct_view
+        if missing_nodes:
+            # Combine missing nodes with those reported as down by this node
+            combined_unreachable = set(unreachable_by_this_node).union(missing_nodes)
+            is_already_reported = any(
+                p.node == node_name and set(p.unreachable_nodes) == combined_unreachable
+                for p in detected_partitions
+                + artificially_detected_partitions  # Check against RMQ and our own detections
+            )
+            if not is_already_reported:
+                logging.debug(
+                    f"Artificially detecting partition: Node {node_name} cannot see (missing or down) {combined_unreachable}"
+                )
+                # Avoid duplicate Partition entries if already added above based only on 'running=False'
+                existing_artificial = next(
+                    (
+                        p
+                        for p in artificially_detected_partitions
+                        if p.node == node_name
+                    ),
+                    None,
+                )
+                if existing_artificial:
+                    existing_artificial.unreachable_nodes = list(combined_unreachable)
+                else:
+                    artificially_detected_partitions.append(
+                        Partition(
+                            node=node_name, unreachable_nodes=list(combined_unreachable)
+                        )
+                    )
+
+        else:
+            logging.debug(
+                f"Direct connection to node {node_name} failed. Assuming it's unreachable."
+            )
+            pass
+
+    # Combine RabbitMQ-reported partitions and artificially detected ones
+    final_partitions = detected_partitions + artificially_detected_partitions
+
+    # Remove potential duplicates (same node reporting same unreachable set)
+    unique_partitions = []
+    seen_partitions = set()
+    for p in final_partitions:
+        # Create a unique key: node_name + sorted tuple of unreachable nodes
+        partition_key = (p.node, tuple(sorted(p.unreachable_nodes)))
+        if partition_key not in seen_partitions:
+            unique_partitions.append(p)
+            seen_partitions.add(partition_key)
+
+    node_statuses: List[NodeStatus] = [
+        NodeStatus(node=node_info.name, running=node_info.running)  # type: ignore
+        for node_info in primary_nodes
+    ]
+
+    cluster_status = ClusterStatus(
+        nodes=node_statuses,  # Keep original running status view
+        network_partitions_detected=True if len(unique_partitions) > 0 else False,
+        partition_details=unique_partitions,
+        raw_node_data=primary_nodes,  # Data from the primary node only
+    )
+
+    return cluster_status

holmes/plugins/toolsets/rabbitmq/rabbitmq_instructions.jinja2

@@ -0,0 +1,37 @@
+# RabbitMQ Troubleshooting Guidelines
+
+## Goal
+Your primary goal when using these tools is to diagnose RabbitMQ cluster health issues, with a specific focus on detecting
+**network partitions (split-brain scenarios)** and identifying potential causes like resource exhaustion on individual nodes.
+
+*   Use the tools to get the *current* state of the cluster and nodes.
+*   Clearly present the key findings from the tool outputs in your analysis.
+
+## Workflow for Split-Brain Diagnosis (Phase I)
+1.  **Check Cluster Status:** ALWAYS start by calling `get_rabbitmq_cluster_status`. This is the most important step.
+    *   Look for `"network_partitions_detected": true`.
+    *   Examine the `"partition_details"` to understand which nodes are reporting inability to reach others. This identifies
+        the members of different sides of the partition.
+    *   Check the `"running"` status of all nodes listed in `"nodes"`.
+2.  **Investigate Affected Nodes:** If a partition is detected, or if any nodes are reported as not running:
+    *   Analyze the node details: Pay close attention to `mem_alarm` and `disk_free_alarm`.
+        Resource exhaustion (memory, disk, file descriptors) is a common reason for nodes becoming unresponsive and causing
+        partitions. Also check if `running` is `false`.
+    *   Analyze the status of the kubernetes pods running RabbitMQ. There is typically one kubernetes pod per RabbitMQ node.
+        This can further indicate if a pod is running and if it is healthy or not.
+    *   Fetch the logs of any pod that is either partitioned or marked as not healthy by the RabbitMQ API.
+3.  **Synthesize Findings:** Based on the cluster status and node details, describe the situation clearly. For example:
+    *   "A network partition is detected in the RabbitMQ cluster '{cluster_name}'. Node 'rabbit@hostA' cannot reach
+        ['rabbit@hostB']. Node 'rabbit@hostB' reports a disk space alarm (`disk_free_alarm: true`)."
+    *   "Node 'rabbit@hostC' is reported as not running in the cluster status."
+4.  **Recommend Remediation Steps (Based on Docs):**
+    *   **CRITICAL:** Refer to the official RabbitMQ documentation for handling partitions:
+        *   Partitions: https://www.rabbitmq.com/docs/partitions, recovering: https://www.rabbitmq.com/docs/partitions#recovering
+        *   Clustering: https://www.rabbitmq.com/docs/clustering
+    *   **DO NOT invent recovery procedures.** Your role is to diagnose and *point* to the correct documentation or standard
+        procedures.
+    *   Based on the *type* of partition (e.g., resource issue vs. pure network), you can suggest which sections of the
+        documentation are most relevant. For example, if a node has a disk alarm, recommend investigating and resolving the
+        disk space issue on that node *before* attempting partition recovery procedures.
+    *   Common manual steps often involve deciding on a "winning" partition, restarting nodes in the "losing" partition(s),
+        and potentially resetting nodes, but **always defer to the official documentation for the exact commands and strategy.**