holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/toolsets/grafana/base_grafana_toolset.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Any, ClassVar, Tuple, Type
+
+from holmes.core.tools import CallablePrerequisite, Tool, Toolset, ToolsetTag
+from holmes.plugins.toolsets.consts import TOOLSET_CONFIG_MISSING_ERROR
+from holmes.plugins.toolsets.grafana.common import GrafanaConfig
+
+from holmes.plugins.toolsets.grafana.grafana_api import grafana_health_check
+
+
+class BaseGrafanaToolset(Toolset):
+    config_class: ClassVar[Type[GrafanaConfig]] = GrafanaConfig
+
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        icon_url: str,
+        tools: list[Tool],
+        docs_url: str,
+    ):
+        super().__init__(
+            name=name,
+            description=description,
+            icon_url=icon_url,
+            docs_url=docs_url,
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            tools=tools,
+            tags=[
+                ToolsetTag.CORE,
+            ],
+            enabled=False,
+        )
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
+        if not config:
+            logging.debug(f"Grafana config not provided {self.name}")
+            return False, TOOLSET_CONFIG_MISSING_ERROR
+
+        try:
+            self._grafana_config = self.config_class(**config)
+            return grafana_health_check(self._grafana_config)
+
+        except Exception as e:
+            logging.exception(f"Failed to set up grafana toolset {self.name}")
+            return False, str(e)
+
+    def get_example_config(self):
+        example_config = GrafanaConfig(
+            api_key="YOUR API KEY",
+            url="YOUR GRAFANA URL",
+            grafana_datasource_uid="UID OF DATASOURCE IN GRAFANA",
+        )
+        return example_config.model_dump()

holmes/plugins/toolsets/grafana/common.py

@@ -0,0 +1,68 @@
+import json
+from typing import Dict, Optional
+from pydantic import BaseModel
+import datetime
+
+from holmes.core.tools import StructuredToolResult, ToolResultStatus
+
+
+class GrafanaConfig(BaseModel):
+    """A config that represents one of the Grafana related tools like Loki or Tempo
+    If `grafana_datasource_uid` is set, then it is assume that Holmes will proxy all
+    requests through grafana. In this case `url` should be the grafana URL.
+    If `grafana_datasource_uid` is not set, it is assumed that the `url` is the
+    systems' URL
+    """
+
+    api_key: Optional[str] = None
+    headers: Optional[Dict[str, str]] = None
+    url: str
+    grafana_datasource_uid: Optional[str] = None
+    external_url: Optional[str] = None
+    healthcheck: Optional[str] = "ready"
+
+
+def build_headers(api_key: Optional[str], additional_headers: Optional[Dict[str, str]]):
+    headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    if additional_headers:
+        headers.update(additional_headers)
+
+    return headers
+
+
+def format_log(log: Dict) -> str:
+    log_str = log.get("log", "")
+    timestamp_nanoseconds = log.get("timestamp")
+    if timestamp_nanoseconds:
+        timestamp_seconds = int(timestamp_nanoseconds) // 1_000_000_000
+        dt = datetime.datetime.fromtimestamp(timestamp_seconds)
+        log_str = dt.strftime("%Y-%m-%dT%H:%M:%SZ") + " " + log_str
+    else:
+        log_str = json.dumps(log)
+
+    return log_str
+
+
+def get_base_url(config: GrafanaConfig) -> str:
+    if config.grafana_datasource_uid:
+        return f"{config.url}/api/datasources/proxy/uid/{config.grafana_datasource_uid}"
+    else:
+        return config.url
+
+
+def ensure_grafana_uid_or_return_error_result(
+    config: GrafanaConfig,
+) -> Optional[StructuredToolResult]:
+    if not config.grafana_datasource_uid:
+        return StructuredToolResult(
+            status=ToolResultStatus.ERROR,
+            error="This tool only works when the toolset is configued ",
+        )
+    else:
+        return None

holmes/plugins/toolsets/grafana/grafana_api.py

@@ -0,0 +1,31 @@
+import logging
+import requests  # type: ignore
+from typing import Tuple
+import backoff
+
+from holmes.plugins.toolsets.grafana.common import (
+    GrafanaConfig,
+    build_headers,
+    get_base_url,
+)
+
+
+@backoff.on_exception(
+    backoff.expo,  # Exponential backoff
+    requests.exceptions.RequestException,  # Retry on request exceptions
+    max_tries=5,  # Maximum retries
+    giveup=lambda e: isinstance(e, requests.exceptions.HTTPError)
+    and e.response.status_code < 500,
+)
+def grafana_health_check(config: GrafanaConfig) -> Tuple[bool, str]:
+    base_url = get_base_url(config)
+    url = f"{base_url}/{config.healthcheck}"
+    try:
+        headers_ = build_headers(api_key=config.api_key, additional_headers=None)
+
+        response = requests.get(url, headers=headers_, timeout=10)  # Added timeout
+        response.raise_for_status()
+        return True, ""
+    except Exception as e:
+        logging.error(f"Failed to fetch grafana health status at {url}", exc_info=True)
+        return False, f"Failed to fetch grafana health status at {url}. {str(e)}"

holmes/plugins/toolsets/grafana/loki_api.py

@@ -0,0 +1,89 @@
+from typing import Dict, List, Optional, Union
+
+import backoff
+import requests  # type: ignore
+
+from holmes.plugins.toolsets.grafana.common import build_headers
+
+
+def parse_loki_response(results: List[Dict]) -> List[Dict]:
+    """
+    Parse Loki response into a more usable format
+
+    Args:
+        results: Raw results from Loki query
+
+    Returns:
+        List of formatted log entries
+    """
+    parsed_logs = []
+    for result in results:
+        stream = result.get("stream", {})
+        for value in result.get("values", []):
+            timestamp, log_line = value
+            parsed_logs.append(
+                {"timestamp": timestamp, "log": log_line, "labels": stream}
+            )
+    return parsed_logs
+
+
+@backoff.on_exception(
+    backoff.expo,  # Exponential backoff
+    requests.exceptions.RequestException,  # Retry on request exceptions
+    max_tries=5,  # Maximum retries
+    giveup=lambda e: isinstance(e, requests.exceptions.HTTPError)
+    and e.response.status_code < 500,
+)
+def execute_loki_query(
+    base_url: str,
+    api_key: Optional[str],
+    headers: Optional[Dict[str, str]],
+    query: str,
+    start: Union[int, str],
+    end: Union[int, str],
+    limit: int,
+) -> List[Dict]:
+    params = {"query": query, "limit": limit, "start": start, "end": end}
+    try:
+        url = f"{base_url}/loki/api/v1/query_range"
+        response = requests.get(
+            url,
+            headers=build_headers(api_key=api_key, additional_headers=headers),
+            params=params,  # type: ignore
+        )
+        response.raise_for_status()
+
+        result = response.json()
+        if "data" in result and "result" in result["data"]:
+            return parse_loki_response(result["data"]["result"])
+        return []
+
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Failed to query Loki logs: {str(e)}")
+
+
+def query_loki_logs_by_label(
+    base_url: str,
+    api_key: Optional[str],
+    headers: Optional[Dict[str, str]],
+    namespace: str,
+    label_value: str,
+    filter: Optional[str],
+    start: Union[int, str],
+    end: Union[int, str],
+    label: str,
+    namespace_search_key: str = "namespace",
+    limit: int = 200,
+) -> List[Dict]:
+    query = f'{{{namespace_search_key}="{namespace}", {label}="{label_value}"}}'
+    if filter:
+        query += f' |= "{filter}"'
+    return execute_loki_query(
+        base_url=base_url,
+        api_key=api_key,
+        headers=headers,
+        query=query,
+        start=start,
+        end=end,
+        limit=limit,
+    )

holmes/plugins/toolsets/grafana/tempo_api.py

@@ -0,0 +1,124 @@
+import requests  # type: ignore
+from typing import Dict, List, Optional
+import backoff
+
+from holmes.plugins.toolsets.grafana.common import build_headers
+from holmes.plugins.toolsets.grafana.trace_parser import process_trace
+
+
+def execute_tempo_query_with_retry(
+    base_url: str,
+    api_key: Optional[str],
+    headers: Optional[Dict[str, str]],
+    query_params: dict,
+    retries: int = 3,
+    timeout: int = 5,
+):
+    """
+    Execute a Tempo API query through Grafana with retries and timeout.
+
+    Args:
+        tempo_datasource_uid: The UID of the Tempo datasource.
+        query_params: Query parameters for the API.
+        retries: Number of retries for the request.
+        timeout: Timeout for each request in seconds.
+
+    Returns:
+        List of trace results.
+    """
+    url = f"{base_url}/api/search"
+
+    @backoff.on_exception(
+        backoff.expo,  # Exponential backoff
+        requests.exceptions.RequestException,  # Retry on request exceptions
+        max_tries=retries,  # Maximum retries
+        giveup=lambda e: isinstance(e, requests.exceptions.HTTPError)
+        and e.response.status_code < 500,
+    )
+    def make_request():
+        response = requests.post(
+            url,
+            headers=build_headers(api_key=api_key, additional_headers=headers),
+            json=query_params,
+            timeout=timeout,  # Set timeout for the request
+        )
+        response.raise_for_status()  # Raise an error for non-2xx responses
+        return response.json()
+
+    try:
+        return make_request()
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Request to Tempo API failed after retries: {e}")
+
+
+def query_tempo_traces(
+    base_url: str,
+    api_key: Optional[str],
+    headers: Optional[Dict[str, str]],
+    query: Optional[str],
+    start: int,
+    end: int,
+    limit: int,
+) -> Dict:
+    query_params = {
+        "start": str(start),
+        "end": str(end),
+        "limit": str(limit),
+    }
+
+    if query:
+        query_params["q"] = query
+    data = execute_tempo_query_with_retry(
+        base_url=base_url,
+        api_key=api_key,
+        headers=headers,
+        query_params=query_params,
+    )
+    return data
+
+
+def query_tempo_trace_by_id(
+    base_url: str,
+    api_key: Optional[str],
+    headers: Optional[Dict[str, str]],
+    trace_id: str,
+    key_labels: List[str],
+    retries: int = 3,
+    timeout: int = 5,
+) -> str:
+    """
+    Query Tempo for a specific trace by its ID with retries and backoff.
+
+    Args:
+        tempo_datasource_id: The ID of the Tempo datasource.
+        trace_id: The trace ID to retrieve.
+        retries: Number of retries for the request.
+        timeout: Timeout for each request in seconds.
+
+    Returns:
+        A formatted trace details string
+    """
+    url = f"{base_url}/api/traces/{trace_id}"
+
+    @backoff.on_exception(
+        backoff.expo,
+        requests.exceptions.RequestException,
+        max_tries=retries,
+        giveup=lambda e: isinstance(e, requests.exceptions.HTTPError)
+        and e.response.status_code < 500,
+    )
+    def make_request():
+        response = requests.get(
+            url,
+            headers=build_headers(api_key=api_key, additional_headers=headers),
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        return process_trace(response.json(), key_labels)
+
+    try:
+        return make_request()
+    except requests.exceptions.RequestException as e:
+        raise Exception(
+            f"Failed to retrieve trace by ID after retries: {e} \n for URL: {url}"
+        )

holmes/plugins/toolsets/grafana/toolset_grafana.py

@@ -0,0 +1,102 @@
+from typing import Dict, List
+from urllib.parse import urlencode, urljoin
+from holmes.core.tools import Tool, ToolParameter
+from holmes.plugins.toolsets.grafana.base_grafana_toolset import BaseGrafanaToolset
+import requests  # type: ignore
+import logging
+
+
+class ListAndBuildGrafanaDashboardURLs(Tool):
+    def __init__(self, toolset: BaseGrafanaToolset):
+        super().__init__(
+            name="list_and_build_grafana_dashboard_urls",
+            description="Lists all available Grafana dashboard urls",
+            parameters={
+                "cluster_name": ToolParameter(
+                    description="The cluster name. Defaults to None.",
+                    type="string",
+                    required=False,
+                ),
+                "namespace": ToolParameter(
+                    description="The namespace for filtering dashboards.",
+                    type="string",
+                    required=False,
+                ),
+                "node_name": ToolParameter(
+                    description="The node name to filter for node-related dashboards.",
+                    type="string",
+                    required=False,
+                ),
+                "pod_name": ToolParameter(
+                    description="The pod name to filter dashboards.",
+                    type="string",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(self, params: Dict) -> str:  # type: ignore
+        url = urljoin(
+            self._toolset._grafana_config.url, "/api/search?query=&type=dash-db"
+        )
+        headers = {"Authorization": f"Bearer {self._toolset._grafana_config.api_key}"}
+
+        try:
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            dashboards = response.json()
+            formatted_dashboards: List[str] = []
+            base_url = (
+                self._toolset._grafana_config.external_url
+                or self._toolset._grafana_config.url
+            )
+            for dash in dashboards:
+                dashboard_url = urljoin(
+                    base_url,
+                    f"/d/{dash['uid']}/{dash['uri'].split('/')[-1]}",
+                )
+
+                params_dict = {
+                    "var-cluster": params.get("cluster_name", ""),
+                    "var-namespace": params.get("namespace", ""),
+                    "var-pod": params.get("pod_name", ""),
+                    "var-node": params.get("node_name", ""),
+                    "var-datasource": self._toolset._grafana_config.grafana_datasource_uid,
+                    "refresh": "5s",
+                }
+
+                # If filtering for nodes, ensure only node-related dashboards are included
+                if params.get("node_name") and "node" not in dash["title"].lower():
+                    continue
+
+                # we add all params since if the dashboard isnt configured for a param it will ignore it if it is added
+                query_string = urlencode({k: v for k, v in params_dict.items() if v})
+                dashboard_url = (
+                    f"{dashboard_url}?{query_string}" if query_string else dashboard_url
+                )
+
+                formatted_dashboards.append(
+                    f"Title: {dash['title']}\nURL: {dashboard_url}\n"
+                )
+
+            return "\n".join(formatted_dashboards) or "No dashboards found."
+        except requests.RequestException as e:
+            logging.error(f"Error fetching dashboards: {str(e)}")
+            return f"Error fetching dashboards: {str(e)}"
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"Lists Grafana dashboards and builds URLs with parameters: {params}"
+
+
+class GrafanaToolset(BaseGrafanaToolset):
+    def __init__(self):
+        super().__init__(
+            name="grafana/grafana",
+            description="Provides tools for interacting with Grafana dashboards",
+            icon_url="https://w7.pngwing.com/pngs/434/923/png-transparent-grafana-hd-logo-thumbnail.png",
+            docs_url="",
+            tools=[
+                ListAndBuildGrafanaDashboardURLs(self),
+            ],
+        )

holmes/plugins/toolsets/grafana/toolset_grafana_loki.py

@@ -0,0 +1,102 @@
+from typing import Any, cast
+from pydantic import BaseModel
+
+from holmes.core.tools import CallablePrerequisite
+from holmes.plugins.toolsets.grafana.common import (
+    GrafanaConfig,
+    format_log,
+    get_base_url,
+)
+from holmes.plugins.toolsets.grafana.grafana_api import grafana_health_check
+from holmes.plugins.toolsets.logging_utils.logging_api import (
+    BasePodLoggingToolset,
+    FetchPodLogsParams,
+    PodLoggingTool,
+)
+from holmes.plugins.toolsets.utils import (
+    process_timestamps_to_rfc3339,
+)
+
+from holmes.plugins.toolsets.grafana.loki_api import (
+    query_loki_logs_by_label,
+)
+from holmes.core.tools import StructuredToolResult, ToolResultStatus
+
+DEFAULT_TIME_SPAN_SECONDS = 3600
+
+
+class GrafanaLokiLabelsConfig(BaseModel):
+    pod: str = "pod"
+    namespace: str = "namespace"
+
+
+class GrafanaLokiConfig(GrafanaConfig):
+    labels: GrafanaLokiLabelsConfig = GrafanaLokiLabelsConfig()
+
+
+class GrafanaLokiToolset(BasePodLoggingToolset):
+    def __init__(self):
+        super().__init__(
+            name="grafana/loki",
+            description="Fetches kubernetes pods logs from Loki",
+            icon_url="https://grafana.com/media/docs/loki/logo-grafana-loki.png",
+            docs_url="https://docs.robusta.dev/master/configuration/holmesgpt/toolsets/grafanaloki.html",
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            tools=[
+                PodLoggingTool(self),
+            ],
+        )
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> tuple[bool, str]:
+        if not config:
+            return False, "Missing Grafana Loki configuration. Check your config."
+
+        self.config = GrafanaLokiConfig(**config)
+
+        return grafana_health_check(self.config)
+
+    def get_example_config(self):
+        example_config = GrafanaLokiConfig(
+            api_key="YOUR API KEY",
+            url="YOUR GRAFANA URL",
+            grafana_datasource_uid="<UID of the loki datasource to use>",
+        )
+        return example_config.model_dump()
+
+    @property
+    def grafana_config(self) -> GrafanaLokiConfig:
+        return cast(GrafanaLokiConfig, self.config)
+
+    def fetch_pod_logs(self, params: FetchPodLogsParams) -> StructuredToolResult:
+        (start, end) = process_timestamps_to_rfc3339(
+            start_timestamp=params.start_time,
+            end_timestamp=params.end_time,
+            default_time_span_seconds=DEFAULT_TIME_SPAN_SECONDS,
+        )
+
+        base_url = get_base_url(self.grafana_config)
+        logs = query_loki_logs_by_label(
+            base_url=base_url,
+            api_key=self.grafana_config.api_key,
+            headers=self.grafana_config.headers,
+            filter=params.filter,
+            namespace=params.namespace,
+            namespace_search_key=self.grafana_config.labels.namespace,
+            label=self.grafana_config.labels.pod,
+            label_value=params.pod_name,
+            start=start,
+            end=end,
+            limit=params.limit or 2000,
+        )
+        if logs:
+            logs.sort(key=lambda x: x["timestamp"])
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data="\n".join([format_log(log) for log in logs]),
+                params=params.model_dump(),
+            )
+        else:
+            return StructuredToolResult(
+                status=ToolResultStatus.NO_DATA,
+                params=params.model_dump(),
+            )

holmes/plugins/toolsets/grafana/toolset_grafana_tempo.jinja2

@@ -0,0 +1,10 @@
+Use Tempo when investigating latency or performance issues. Tempo provides traces information for application running on the cluster.
+Assume every application provides tempo traces.
+1. Start by identifying an initial filter to use. This can be a pod name, a deployment name or a service name
+2. Use `fetch_tempo_traces` setting the appropriate query params
+    - Use the min_duration filter to ensure you get traces that trigger the alert when you are investigating a performance issue
+    - If possible, use start and end date to narrow down your search.
+        - Use fetch_finding_by_id if you are provided with a finding/alert id. It will contain details about when the alert was triggered
+    - Use at least one of the following argument to ensure you get relevant traces: `service_name`, `pod_name` or `deployment_name`.
+3. Look at the duration of each span in any single trace and deduct any issues.
+4. ALWAYS fetch the logs for a pod once you identify a span that is taking a long time. There may be an explanation for the slowness in the logs.