holmesgpt 0.13.2__py3-none-any.whl → 0.18.4__py3-none-any.whl

holmes/plugins/toolsets/prometheus/prometheus_instructions.jinja2

@@ -1,11 +1,40 @@
 
 # Prometheus/PromQL queries
-* ALWAYS call list_prometheus_rules to get the alert definition
+
+{%- if config and config.prometheus_url and "coralogix" in config.prometheus_url %}
+You are using Coralogix Prometheus.
+* Metrics/labels may differ; discover names with `get_metric_names` first.
+* For high-cardinality, wrap with `topk(5, <query>)`.
+* Always include explicit time ranges for range queries.
+* Example: `container_cpu_utilization{namespace="test-173"}` can fail if the label is named differently (e.g., `k8s_namespace_name`) or the metric is named differently. Do not assume names—only use labels you have seen returned or been told exist.
+{%- endif %}
+
+## Efficient Metric Discovery (when needed)
+* When you need to discover metrics, use `get_metric_names` with filters - it's the fastest method
+* Combine multiple patterns with regex OR (|) to reduce API calls:
+  - `{__name__=~"node_cpu.*|node_memory.*|node_disk.*"}` - get all node resource metrics in one call
+  - `{__name__=~"container.*|pod.*|kube.*"}` - get all Kubernetes-related metrics
+  - `{namespace=~"example1|example2|example3"}` - metrics from multiple namespaces
+* Use `get_metric_metadata` after discovering names to get types/descriptions if needed
+* Use `get_label_values` to discover pods, namespaces, jobs: e.g., get_label_values(label="pod")
+* Only use `get_series` when you need full label sets (slower than other methods)
+
+## Retrying queries that return too much data
+* When a Prometheus query returns too much data (e.g., truncation error), you MUST retry with a more specific query or less data points or topk/bottomk
+* NEVER EVER EVER answer a question based on Prometheus data that was truncated as you might be missing important information and give the totally wrong answer
+* Prefer telling the user you can't answer the question because of too much data rather than answering based on incomplete data
+* You are also able to show graphs to the user (using the promql embed functionality mentioned below) so you can show users graphs and THEY can interpret the data themselves, even if you can't answer.
+* Do NOT hestitate to try alternative queries and try to reduce the amount of data returned until you get a successful query
+* Be extremely, extremely cautious when answering based on get_label_values because the existence of a label value says NOTHING about the metric value itself (is it high, low, or perhaps the label exists in Prometheus but its an older series not present right now)
+* DO NOT give answers about metrics based on what 'is typically the case' or 'common knowledge' - if you can't see the actual metric value, you MUST NEVER EVER answer about it - just tell the user your limitations due to the size of the data
+
+## Alert Investigation & Query Execution
+* When investigating a Prometheus alert, ALWAYS call list_prometheus_rules to get the alert definition
 * Use Prometheus to query metrics from the alert promql
 * Use prometheus to execute promql queries with the tools `execute_prometheus_instant_query` and `execute_prometheus_range_query`
 * To create queries, use 'start_timestamp' and 'end_timestamp' as graphs start and end times
 * ALWAYS embed the execution results into your answer
-* You only need to embed the partial result in your response. Include the "tool_name" and "random_key". For example: << {"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "92jf2hf"} >>
+* You only need to embed the partial result in your response. Include the "tool_name" and "tool_call_id". For example: << {"type": "promql", "tool_name": "execute_prometheus_range_query", "tool_call_id": "92jf2hf"} >>
 * Use these tools to generate charts that users can see. Here are standard metrics but you can use different ones:
 ** For memory consumption: `container_memory_working_set_bytes`
 ** For CPU usage: `container_cpu_usage_seconds_total`
@@ -16,9 +45,29 @@
 ** Avoid global averages like `sum(rate(<metric>_sum)) / sum(rate(<metric>_count))` because it hides data and is not generally informative
 * Timestamps MUST be in string date format. For example: '2025-03-15 10:10:08.610862+00:00'
 * Post processing will parse your response, re-run the query from the tool output and create a chart visible to the user
-* Only generate and execute a prometheus query after checking what metrics are available with the `list_available_metrics` tool
+* When unsure about available metrics, use `get_metric_names` with appropriate filters (combine multiple patterns with | for efficiency). Then use `get_metric_metadata` if you need descriptions/types
 * Check that any node, service, pod, container, app, namespace, etc. mentioned in the query exist in the kubernetes cluster before making a query. Use any appropriate kubectl tool(s) for this
 * The toolcall will return no data to you. That is expected. You MUST however ensure that the query is successful.
+
+## Handling High-Cardinality Metrics
+* CRITICAL: When querying metrics that may return many time series (>10), ALWAYS use aggregation to limit results
+* ALWAYS use `topk()` or `bottomk()` to limit the number of series returned
+* Standard pattern for high-cardinality queries:
+  - Use `topk(5, <your_query>)` to get the top 5 series
+  - Example: `topk(5, rate(container_cpu_usage_seconds_total{namespace="example"}[5m]))`
+  - This prevents context overflow and focuses on the most relevant data
+* To also capture the aggregate of remaining series as "other":
+  ```
+  topk(5, rate(container_cpu_usage_seconds_total{namespace="example"}[5m])) or label_replace((sum(rate(container_cpu_usage_seconds_total{namespace="example"}[5m])) - sum(topk(5, rate(container_cpu_usage_seconds_total{namespace="example"}[5m])))), "pod", "other", "", "")
+  ```
+* Common high-cardinality scenarios requiring topk():
+  - Pod-level metrics in namespaces with many pods
+  - Container-level CPU/memory metrics
+  - HTTP metrics with many endpoints or status codes
+  - Any query returning more than 10 time series
+* For initial exploration, you may use instant queries with `count()` to check cardinality:
+  - Example: `count(count by (pod) (container_cpu_usage_seconds_total{namespace="example"}))`
+  - If count > 10, use topk() in your range query
 * When doing queries, always extend the time range, to 15 min before and after the alert start time
 * ALWAYS embed the execution results into your answer
 * ALWAYS embed a Prometheus graph in the response. The graph should visualize data related to the incident.
@@ -26,9 +75,9 @@
 * When embedding multiple graphs, always add line spacing between them
     For example:
 
-    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "lBaA"}>>
+    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "tool_call_id": "lBaA"}>>
 
-    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "IKtq"}>>
+    <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "tool_call_id": "IKtq"}>>
 
 {%- if config and config.additional_labels and config.additional_labels.keys()|list|length > 0 %}
 * ALWAYS add the following additional labels to ALL PromQL queries:

holmes/plugins/toolsets/prometheus/utils.py

@@ -0,0 +1,28 @@
+import re
+from typing import Optional, Union
+
+
+def parse_duration_to_seconds(v: Optional[Union[str, float, int]]) -> Optional[float]:
+    if v is None:
+        return None
+    if isinstance(v, (int, float)):
+        return float(v)
+    s = v.strip().lower()
+    if s.isdigit():
+        return float(int(s))
+
+    units = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+
+    # Check for partial time formats (e.g., 1h30m, 5m12s, 1d2h30m)
+    pattern = r"(\d+(?:\.\d+)?)(d|h|m|s)"
+    matches = re.findall(pattern, s)
+
+    if matches:
+        total_seconds = 0.0
+        for value_str, unit in matches:
+            value = float(value_str)
+            total_seconds += value * units[unit]
+        return float(int(total_seconds))
+
+    # fallback: try float seconds
+    return float(s)

holmes/plugins/toolsets/rabbitmq/api.py

@@ -1,11 +1,11 @@
-from enum import Enum
 import logging
+from enum import Enum
 from typing import Any, Dict, List, Optional, Set
 from urllib.parse import urljoin, urlparse
 
 import backoff
-from pydantic import BaseModel
 import requests  # type: ignore
+from pydantic import BaseModel, ConfigDict, model_validator
 from requests.auth import HTTPBasicAuth  # type: ignore
 
 # --- Enums and Pydantic Models (Mostly Unchanged) ---
@@ -17,12 +17,31 @@ class ClusterConnectionStatus(str, Enum):
 
 
 class RabbitMQClusterConfig(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
     id: str = "rabbitmq"  # must be unique
     management_url: str  # e.g., http://rabbitmq-service:15672
     username: Optional[str] = None
     password: Optional[str] = None
     request_timeout_seconds: int = 30
-    verify_certs: bool = True
+    verify_ssl: bool = True
+
+    @model_validator(mode="after")
+    def handle_deprecated_fields(self):
+        extra = self.model_extra or {}
+        deprecated = []
+
+        # Map old name to new name
+        if "verify_certs" in extra:
+            self.verify_ssl = extra["verify_certs"]
+            deprecated.append("verify_certs -> verify_ssl")
+
+        if deprecated:
+            logging.warning(
+                f"RabbitMQ config uses deprecated field names: {', '.join(deprecated)}. "
+                "Please update your configuration."
+            )
+        return self
 
     # For internal use
     connection_status: Optional[ClusterConnectionStatus] = None
@@ -111,7 +130,7 @@ def make_request(
             params=params,
             json=data,
             timeout=config.request_timeout_seconds,
-            verify=config.verify_certs,
+            verify=config.verify_ssl,
         )
         response.raise_for_status()
         return response.json()

holmes/plugins/toolsets/rabbitmq/toolset_rabbitmq.py

@@ -1,20 +1,21 @@
-import os
 import logging
+import os
 from typing import Any, List, Optional, Tuple
+from urllib.parse import urljoin
 
 from pydantic import BaseModel
+from requests import RequestException  # type: ignore
+
 from holmes.core.tools import (
     CallablePrerequisite,
     StructuredToolResult,
+    StructuredToolResultStatus,
     Tool,
+    ToolInvokeContext,
     ToolParameter,
-    ToolResultStatus,
     Toolset,
     ToolsetTag,
 )
-from requests import RequestException  # type: ignore
-from urllib.parse import urljoin
-
 from holmes.plugins.toolsets.rabbitmq.api import (
     ClusterConnectionStatus,
     RabbitMQClusterConfig,
@@ -63,9 +64,7 @@ class ListConfiguredClusters(BaseRabbitMQTool):
             toolset=toolset,
         )
 
-    def _invoke(
-        self, params: dict, user_approved: bool = False
-    ) -> StructuredToolResult:
+    def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult:
         if not self.toolset.config:
             raise ValueError("RabbitMQ is not configured.")
 
@@ -79,7 +78,7 @@ class ListConfiguredClusters(BaseRabbitMQTool):
             if c.connection_status == ClusterConnectionStatus.SUCCESS
         ]
         return StructuredToolResult(
-            status=ToolResultStatus.SUCCESS, data=available_clusters
+            status=StructuredToolResultStatus.SUCCESS, data=available_clusters
         )
 
     def get_parameterized_one_liner(self, params) -> str:
@@ -103,21 +102,21 @@ class GetRabbitMQClusterStatus(BaseRabbitMQTool):
             toolset=toolset,
         )
 
-    def _invoke(
-        self, params: dict, user_approved: bool = False
-    ) -> StructuredToolResult:
+    def _invoke(self, params: dict, context: ToolInvokeContext) -> StructuredToolResult:
         try:
             # Fetch node details which include partition info
             cluster_config = self._get_cluster_config(
                 cluster_id=params.get("cluster_id")
             )
             result = get_cluster_status(cluster_config)
-            return StructuredToolResult(status=ToolResultStatus.SUCCESS, data=result)
+            return StructuredToolResult(
+                status=StructuredToolResultStatus.SUCCESS, data=result
+            )
 
         except Exception as e:
             logging.info("Failed to process RabbitMQ cluster status", exc_info=True)
             return StructuredToolResult(
-                status=ToolResultStatus.ERROR,
+                status=StructuredToolResultStatus.ERROR,
                 error=f"Unexpected error fetching RabbitMQ cluster status: {str(e)}",
                 data=None,
             )