holmesgpt 0.12.3a1__py3-none-any.whl → 0.12.4__py3-none-any.whl

holmes/plugins/toolsets/datadog/toolset_datadog_metrics.py

@@ -27,6 +27,9 @@ from holmes.plugins.toolsets.utils import (
     process_timestamps_to_int,
     standard_start_datetime_tool_param_description,
 )
+from datetime import datetime
+
+from holmes.utils.keygen_utils import generate_random_key
 
 DEFAULT_TIME_SPAN_SECONDS = 3600
 
@@ -60,7 +63,7 @@ class ListActiveMetrics(BaseDatadogMetricsTool):
                     required=False,
                 ),
                 "tag_filter": ToolParameter(
-                    description="Filter metrics by tags in the format tag:value",
+                    description="Filter metrics by tags in the format tag:value. pod tag is pod_name. namespace tag is kube_namespace.",
                     type="string",
                     required=False,
                 ),
@@ -188,6 +191,16 @@ class QueryMetrics(BaseDatadogMetricsTool):
                     type="string",
                     required=False,
                 ),
+                "description": ToolParameter(
+                    description="Describes the query",
+                    type="string",
+                    required=True,
+                ),
+                "output_type": ToolParameter(
+                    description="Specifies how to interpret the Datadog result. Use 'Plain' for raw values, 'Bytes' to format byte values, 'Percentage' to scale 0–1 values into 0–100%, or 'CPUUsage' to convert values to cores (e.g., 500 becomes 500m, 2000 becomes 2).",
+                    type="string",
+                    required=False,
+                ),
             },
             toolset=toolset,
         )
@@ -230,6 +243,8 @@ class QueryMetrics(BaseDatadogMetricsTool):
             )
 
             series = data.get("series", [])
+            description = params.get("description", "")
+            output_type = params.get("output_type", "Plain")
 
             if not series:
                 return StructuredToolResult(
@@ -238,17 +253,58 @@ class QueryMetrics(BaseDatadogMetricsTool):
                     params=params,
                 )
 
+            # Transform Datadog series data to match Prometheus format
+            prometheus_result = []
+            for serie in series:
+                # Extract metric info from Datadog series
+                metric_info = {}
+                if "metric" in serie:
+                    metric_info["__name__"] = serie["metric"]
+
+                # Add other fields from scope/tag_set if available
+                if "scope" in serie and serie["scope"]:
+                    # Parse scope like "pod_name:robusta-runner-78599b764d-f847h" into labels
+                    scope_parts = serie["scope"].split(",")
+                    for part in scope_parts:
+                        if ":" in part:
+                            key, value = part.split(":", 1)
+                            metric_info[key.strip()] = value.strip()
+
+                # Transform pointlist to values format (timestamp, value as strings)
+                values = []
+                if "pointlist" in serie:
+                    for point in serie["pointlist"]:
+                        if len(point) >= 2:
+                            # Convert timestamp from milliseconds to seconds, format as string
+                            timestamp = int(point[0] / 1000)
+                            value = str(point[1])
+                            values.append([timestamp, value])
+
+                prometheus_result.append({"metric": metric_info, "values": values})
+
+            # Convert timestamps to RFC3339 format for start/end
+            start_rfc = datetime.fromtimestamp(from_time).strftime("%Y-%m-%dT%H:%M:%SZ")
+            end_rfc = datetime.fromtimestamp(to_time).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+            # Create response matching Prometheus format
             response_data = {
                 "status": "success",
+                "error_message": None,
+                "random_key": generate_random_key(),
+                "tool_name": self.name,
+                "description": description,
                 "query": query,
-                "from_time": from_time,
-                "to_time": to_time,
-                "series": series,
+                "start": start_rfc,
+                "end": end_rfc,
+                "step": 60,  # Default step, Datadog doesn't provide this directly
+                "output_type": output_type,
+                "data": {"resultType": "matrix", "result": prometheus_result},
             }
 
+            data_str = json.dumps(response_data, indent=2)
             return StructuredToolResult(
                 status=ToolResultStatus.SUCCESS,
-                data=json.dumps(response_data, indent=2),
+                data=data_str,
                 params=params,
             )
 
@@ -287,7 +343,8 @@ class QueryMetrics(BaseDatadogMetricsTool):
 
     def get_parameterized_one_liner(self, params) -> str:
         query = params.get("query", "<no query>")
-        return f"Query Datadog metrics: {query}"
+        description = params.get("description", "")
+        return f"Query Datadog metrics: query='{query}', description='{description}'"
 
 
 class QueryMetricsMetadata(BaseDatadogMetricsTool):
@@ -404,6 +461,92 @@ class QueryMetricsMetadata(BaseDatadogMetricsTool):
         return "Get Datadog metric metadata"
 
 
+class ListMetricTags(BaseDatadogMetricsTool):
+    def __init__(self, toolset: "DatadogMetricsToolset"):
+        super().__init__(
+            name="list_datadog_metric_tags",
+            description="List all available tags and aggregations for a specific metric. This helps in building queries by showing what dimensions are available for filtering.",
+            parameters={
+                "metric_name": ToolParameter(
+                    description="The name of the metric to get tags for (e.g., 'system.cpu.user', 'container.memory.usage')",
+                    type="string",
+                    required=True,
+                ),
+            },
+            toolset=toolset,
+        )
+
+    def _invoke(self, params: Any) -> StructuredToolResult:
+        if not self.toolset.dd_config:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=TOOLSET_CONFIG_MISSING_ERROR,
+                params=params,
+            )
+
+        url = None
+        query_params = None
+
+        try:
+            metric_name = get_param_or_raise(params, "metric_name")
+
+            url = f"{self.toolset.dd_config.site_api_url}/api/v2/metrics/{metric_name}/active-configurations"
+            headers = get_headers(self.toolset.dd_config)
+
+            data = execute_datadog_http_request(
+                url=url,
+                headers=headers,
+                timeout=self.toolset.dd_config.request_timeout,
+                method="GET",
+                payload_or_params={},
+            )
+
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=data,
+                params=params,
+            )
+
+        except DataDogRequestError as e:
+            logging.exception(e, exc_info=True)
+
+            if e.status_code == 404:
+                error_msg = f"Metric '{params.get('metric_name', 'unknown')}' not found. Please check the metric name."
+            elif e.status_code == 429:
+                error_msg = f"Datadog API rate limit exceeded. Failed after {MAX_RETRY_COUNT_ON_RATE_LIMIT} retry attempts."
+            elif e.status_code == 403:
+                error_msg = (
+                    f"Permission denied. Ensure your Datadog Application Key has the 'metrics_read' "
+                    f"permissions. Error: {str(e)}"
+                )
+            else:
+                error_msg = f"Exception while querying Datadog: {str(e)}"
+
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=error_msg,
+                params=params,
+                invocation=json.dumps({"url": url, "params": query_params})
+                if url and query_params
+                else None,
+            )
+
+        except Exception as e:
+            logging.exception(
+                f"Failed to query Datadog metric tags for params: {params}",
+                exc_info=True,
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"Exception while querying Datadog: {str(e)}",
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params) -> str:
+        metric_name = params.get("metric_name", "<no metric>")
+        return f"List available tags for Datadog metric: {metric_name}"
+
+
 class DatadogMetricsToolset(Toolset):
     dd_config: Optional[DatadogMetricsConfig] = None
 
@@ -418,6 +561,7 @@ class DatadogMetricsToolset(Toolset):
                 ListActiveMetrics(toolset=self),
                 QueryMetrics(toolset=self),
                 QueryMetricsMetadata(toolset=self),
+                ListMetricTags(toolset=self),
             ],
             experimental=True,
             tags=[ToolsetTag.CORE],

holmes/plugins/toolsets/kubernetes.yaml

@@ -199,8 +199,14 @@ toolsets:
     description: "Provides real-time metrics for pods and nodes"
     docs_url: "https://docs.robusta.dev/master/configuration/holmesgpt/toolsets/kubernetes.html#live-metrics"
     icon_url: "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRPKA-U9m5BxYQDF1O7atMfj9EMMXEoGu4t0Q&s"
+    llm_instructions: |
+      The kubectl_top_pods or kubectl_top_nodes do not return time series data or metrics that can be used for graphs
+      Do NOT use kubectl_top_pods or kubectl_top_nodes for graph generation - it only shows current snapshot data
+      kubectl_top_pods or kubectl_top_nodes are for current status checks, not historical graphs
     tags:
       - core
+    prerequisites:
+      - command: "kubectl top nodes"
     tools:
       - name: "kubectl_top_pods"
         description: "Retrieves real-time CPU and memory usage for each pod in the cluster."

holmes/plugins/toolsets/prometheus/prometheus.py

@@ -1,9 +1,7 @@
 import json
 import logging
 import os
-import random
 import re
-import string
 import time
 from typing import Any, Dict, List, Optional, Tuple, Union
 from urllib.parse import urljoin
@@ -31,6 +29,7 @@ from holmes.plugins.toolsets.utils import (
 from holmes.utils.cache import TTLCache
 from holmes.common.env_vars import IS_OPENSHIFT
 from holmes.common.openshift import load_openshift_token
+from holmes.utils.keygen_utils import generate_random_key
 
 PROMETHEUS_RULES_CACHE_KEY = "cached_prometheus_rules"
 DEFAULT_TIME_SPAN_SECONDS = 3600
@@ -79,10 +78,6 @@ class BasePrometheusTool(Tool):
     toolset: "PrometheusToolset"
 
 
-def generate_random_key():
-    return "".join(random.choices(string.ascii_letters + string.digits, k=4))
-
-
 def filter_metrics_by_type(metrics: Dict, expected_type: str):
     return {
         metric_name: metric_data
@@ -687,6 +682,7 @@ class ExecuteRangeQuery(BasePrometheusTool):
                 if self.toolset.config.tool_calls_return_data:
                     response_data["data"] = data.get("data")
                 data_str = json.dumps(response_data, indent=2)
+
                 return StructuredToolResult(
                     status=ToolResultStatus.SUCCESS,
                     data=data_str,

holmes/plugins/toolsets/prometheus/prometheus_instructions.jinja2

@@ -25,11 +25,11 @@
 * Embed at most 2 graphs
 * When embedding multiple graphs, always add line spacing between them
     For example:
-    ```
+
     <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "lBaA"}>>
 
     <<{"type": "promql", "tool_name": "execute_prometheus_range_query", "random_key": "IKtq"}>>
-    ```
+
 {%- if config and config.additional_labels and config.additional_labels.keys()|list|length > 0 %}
 * ALWAYS add the following additional labels to ALL PromQL queries:
 {%- for key, value in config.additional_labels.items() %}

holmes/plugins/toolsets/runbook/runbook_fetcher.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Any, Dict
+import textwrap
+from typing import Any, Dict, List, Optional
 
 from holmes.core.tools import (
     StructuredToolResult,
@@ -9,7 +10,8 @@ from holmes.core.tools import (
     Toolset,
     ToolsetTag,
 )
-from holmes.plugins.runbooks import get_runbook_by_path
+
+from holmes.plugins.runbooks import get_runbook_by_path, DEFAULT_RUNBOOK_SEARCH_PATH
 
 
 # TODO(mainred): currently we support fetch runbooks hosted internally, in the future we may want to support fetching
@@ -33,15 +35,66 @@ class RunbookFetcher(Tool):
         )
 
     def _invoke(self, params: Any) -> StructuredToolResult:
-        path: str = params["link"]
+        link: str = params["link"]
+
+        search_paths = [DEFAULT_RUNBOOK_SEARCH_PATH]
+        if self.toolset.config and "additional_search_paths" in self.toolset.config:
+            search_paths.extend(self.toolset.config["additional_search_paths"])
 
-        runbook_path = get_runbook_by_path(path)
+        runbook_path = get_runbook_by_path(link, search_paths)
+
+        if runbook_path is None:
+            err_msg = (
+                f"Runbook '{link}' not found in any of the search paths: {search_paths}"
+            )
+            logging.error(err_msg)
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=err_msg,
+                params=params,
+            )
+
+        # Read and return the runbook content
         try:
             with open(runbook_path, "r") as file:
                 content = file.read()
+                wrapped_content = textwrap.dedent(f"""\
+                    <runbook>
+{textwrap.indent(content, " " * 20)}
+                    </runbook>
+                    Note: the above are DIRECTIONS not ACTUAL RESULTS. You now need to follow the steps outlined in the runbook yourself USING TOOLS.
+                    Anything that looks like an actual result in the above <runbook> is just an EXAMPLE.
+                    Now follow those steps and report back what you find.
+                    You must follow them by CALLING TOOLS YOURSELF.
+                    If you are missing tools, follow your general instructions on how to enable them as present in your system prompt.
+
+                    Assuming the above runbook is relevant, you MUST start your response (after calling tools to investigate) with:
+                    "I found a runbook named [runbook name/description] and used it to troubleshoot:"
+
+                    Then list each step with ✅ for completed steps and ❌ for steps you couldn't complete.
+
+                    <example>
+                        I found a runbook named **Troubleshooting Erlang Issues** and used it to troubleshoot:
+
+                        1. ✅ *Check BEAM VM memory usage* - 87% allocated (3.2GB used of 4GB limit)
+                        2. ✅ *Review GC logs* - 15 full GC cycles in last 30 minutes, avg pause time 2.3s
+                        3. ✅ *Verify Erlang application logs* - `** exception error: out of memory in process <0.139.0> called by gen_server:handle_msg/6`
+                        4. ❌ *Could not analyze process mailbox sizes* - Observer tool not enabled in container. Enable remote shell or observer_cli for process introspection.
+                        5. ✅ *Check pod memory limits* - container limit 4Gi, requests 2Gi
+                        6. ✅ *Verify BEAM startup arguments* - `+S 4:4 +P 1048576`, no memory instrumentation flags enabled
+                        7. ❌ *Could not retrieve APM traces* - Datadog traces toolset is disabled. You can enable it by following https://robusta-dev.github.io/holmesgpt/data-sources/builtin-toolsets/datadog/
+                        8. ❌ *Could not query Erlang metrics* - Prometheus integration is not connected. Enable it via https://robusta-dev.github.io/holmesgpt/data-sources/builtin-toolsets/prometheus/
+                        9. ✅ *Examine recent deployments* - app version 2.1.3 deployed 4 hours ago, coincides with memory spike
+                        10. ❌ *Could not check Stripe API status* - No toolset for Stripe integration exists. To monitor Stripe or similar third-party APIs, add a [custom toolset](https://robusta-dev.github.io/holmesgpt/data-sources/custom-toolsets/) or use a [remote MCP server](https://robusta-dev.github.io/holmesgpt/data-sources/remote-mcp-servers/)
+
+                        **Root cause:** Memory leak in `gen_server` logic introduced in v2.1.3. BEAM VM hitting memory limit, causing out-of-memory crashes.
+
+                        **Fix:** Roll back to v2.1.2 or increase memory limit to 6GB as a temporary workaround.
+                    </example>
+                """)
                 return StructuredToolResult(
                     status=ToolResultStatus.SUCCESS,
-                    data=content,
+                    data=wrapped_content,
                     params=params,
                 )
         except Exception as e:
@@ -59,7 +112,12 @@ class RunbookFetcher(Tool):
 
 
 class RunbookToolset(Toolset):
-    def __init__(self):
+    def __init__(self, additional_search_paths: Optional[List[str]] = None):
+        # Store additional search paths in config
+        config = {}
+        if additional_search_paths:
+            config["additional_search_paths"] = additional_search_paths
+
         super().__init__(
             name="runbook",
             description="Fetch runbooks",
@@ -72,6 +130,7 @@ class RunbookToolset(Toolset):
                 ToolsetTag.CORE,
             ],
             is_default=True,
+            config=config,
         )
 
     def get_example_config(self) -> Dict[str, Any]:

holmes/plugins/toolsets/service_discovery.py

@@ -30,7 +30,7 @@ def find_service_url(label_selector):
         )
         if not svc_list.items:
             return None
-        svc: V1Service = svc_list.items[0]
+        svc: V1Service = svc_list.items[0]  # type: ignore
         name = svc.metadata.name
         namespace = svc.metadata.namespace
         port = svc.spec.ports[0].port

holmes/plugins/toolsets/slab.yaml

@@ -11,7 +11,7 @@ toolsets:
           - SLAB_API_KEY
     tools:
       - name: "fetch_slab_document"
-        description: "Fetch a document from slab. Use this to fetch runbooks if they are present before starting your investigation."
+        description: "Fetch a document from slab."
         command: |
           curl -X POST \
           -H "Authorization: ${SLAB_API_KEY}" \

holmes/utils/colors.py

@@ -0,0 +1,7 @@
+# Color constants for terminal output
+USER_COLOR = "#DEFCC0"  # light green
+AI_COLOR = "#00FFFF"  # cyan
+TOOLS_COLOR = "magenta"
+HELP_COLOR = "cyan"  # same as AI_COLOR for now
+ERROR_COLOR = "red"
+STATUS_COLOR = "yellow"

holmes/utils/console/consts.py

@@ -1,2 +1,7 @@
+import os
+
 # Common help texts
 system_prompt_help = "Advanced. System prompt for LLM. Values starting with builtin:// are loaded from holmes/plugins/prompts, values starting with file:// are loaded from the given path, other values are interpreted as a prompt string"
+
+# Agent name used in welcome banner, logging, and prompts
+agent_name: str = os.environ.get("AGENT_NAME", "HolmesGPT")

holmes/utils/console/result.py

@@ -6,6 +6,7 @@ from holmes.config import Config
 from holmes.core.tool_calling_llm import LLMResult
 from holmes.plugins.destinations import DestinationType
 from holmes.plugins.interfaces import Issue
+from holmes.utils.colors import AI_COLOR
 
 
 def handle_result(
@@ -27,7 +28,7 @@ def handle_result(
                     markup=False,
                 )
 
-        console.print("[bold green]AI:[/bold green]", end=" ")
+        console.print(f"[bold {AI_COLOR}]AI:[/bold {AI_COLOR}]", end=" ")
         console.print(Markdown(result.result))  # type: ignore
         if add_separator:
             console.print(Rule())

holmes/utils/keygen_utils.py

@@ -0,0 +1,6 @@
+import random
+import string
+
+
+def generate_random_key():
+    return "".join(random.choices(string.ascii_letters + string.digits, k=4))

holmes/version.py

@@ -90,7 +90,7 @@ def get_version() -> str:
         )
         dirty = "-dirty" if status else ""
 
-        return f"{tag}-{branch}{dirty}"
+        return f"dev-{tag}-{branch}{dirty}"
 
     except Exception:
         pass
@@ -149,7 +149,7 @@ def check_version() -> VersionCheckResult:
 
     update_message = None
     if not is_latest:
-        update_message = f"Update available: v{holmes_info.latest_version} (current: {current_version})"
+        update_message = f"Update available: {holmes_info.latest_version} (current: {current_version})"
 
     return VersionCheckResult(
         is_latest=is_latest,