holmesgpt 0.13.3__py3-none-any.whl → 0.14.0a0__py3-none-any.whl

holmes/plugins/toolsets/grafana/toolset_grafana_tempo.py

@@ -1,10 +1,7 @@
 import os
-import re
-from typing import Any, Dict, List, cast
+from typing import Any, Dict, Tuple, cast, List
 
-import requests  # type: ignore
 import yaml  # type: ignore
-from pydantic import BaseModel
 
 from holmes.common.env_vars import load_bool
 from holmes.core.tools import (
@@ -15,43 +12,25 @@ from holmes.core.tools import (
 )
 from holmes.plugins.toolsets.grafana.base_grafana_toolset import BaseGrafanaToolset
 from holmes.plugins.toolsets.grafana.common import (
-    GrafanaConfig,
-    build_headers,
-    get_base_url,
+    GrafanaTempoConfig,
 )
-from holmes.plugins.toolsets.grafana.tempo_api import (
-    query_tempo_trace_by_id,
-    query_tempo_traces,
-)
-from holmes.plugins.toolsets.grafana.trace_parser import format_traces_list
+from holmes.plugins.toolsets.grafana.grafana_tempo_api import GrafanaTempoAPI
 from holmes.plugins.toolsets.logging_utils.logging_api import (
     DEFAULT_TIME_SPAN_SECONDS,
 )
 from holmes.plugins.toolsets.utils import (
-    get_param_or_raise,
-    process_timestamps_to_int,
     toolset_name_for_one_liner,
+    process_timestamps_to_int,
 )
 
 TEMPO_LABELS_ADD_PREFIX = load_bool("TEMPO_LABELS_ADD_PREFIX", True)
+TEMPO_API_USE_POST = False  # Use GET method for direct API mapping
 
 ONE_HOUR_IN_SECONDS = 3600
 DEFAULT_TRACES_TIME_SPAN_SECONDS = DEFAULT_TIME_SPAN_SECONDS  # 7 days
 DEFAULT_TAGS_TIME_SPAN_SECONDS = 8 * ONE_HOUR_IN_SECONDS  # 8 hours
 
 
-class GrafanaTempoLabelsConfig(BaseModel):
-    pod: str = "k8s.pod.name"
-    namespace: str = "k8s.namespace.name"
-    deployment: str = "k8s.deployment.name"
-    node: str = "k8s.node.name"
-    service: str = "service.name"
-
-
-class GrafanaTempoConfig(GrafanaConfig):
-    labels: GrafanaTempoLabelsConfig = GrafanaTempoLabelsConfig()
-
-
 class BaseGrafanaTempoToolset(BaseGrafanaToolset):
     config_class = GrafanaTempoConfig
 
@@ -67,6 +46,23 @@ class BaseGrafanaTempoToolset(BaseGrafanaToolset):
     def grafana_config(self) -> GrafanaTempoConfig:
         return cast(GrafanaTempoConfig, self._grafana_config)
 
+    def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
+        """Check Tempo connectivity using the echo endpoint."""
+        # First call parent to validate config
+        success, msg = super().prerequisites_callable(config)
+        if not success:
+            return success, msg
+
+        # Then check Tempo-specific echo endpoint
+        try:
+            api = GrafanaTempoAPI(self.grafana_config, use_post=TEMPO_API_USE_POST)
+            if api.query_echo_endpoint():
+                return True, "Successfully connected to Tempo"
+            else:
+                return False, "Failed to connect to Tempo echo endpoint"
+        except Exception as e:
+            return False, f"Failed to connect to Tempo: {str(e)}"
+
     def build_k8s_filters(
         self, params: Dict[str, Any], use_exact_match: bool
     ) -> List[str]:
@@ -107,9 +103,9 @@ class BaseGrafanaTempoToolset(BaseGrafanaToolset):
                     escaped_value = value.replace('"', '\\"')
                     filters.append(f'{prefix}{label}="{escaped_value}"')
                 else:
-                    # Escape regex special characters for partial match
-                    escaped_value = re.escape(value)
-                    filters.append(f'{prefix}{label}=~".*{escaped_value}.*"')
+                    # For partial match, use simple substring matching
+                    # Don't escape anything - let Tempo handle the regex
+                    filters.append(f'{prefix}{label}=~".*{value}.*"')
 
         return filters
 
@@ -122,207 +118,198 @@ def validate_params(params: Dict[str, Any], expected_params: List[str]):
     return f"At least one of the following argument is expected but none were set: {expected_params}"
 
 
-class GetTempoTraces(Tool):
-    def __init__(self, toolset: BaseGrafanaTempoToolset):
-        super().__init__(
-            name="fetch_tempo_traces",
-            description="""Lists Tempo traces. At least one of `service_name`, `pod_name` or `deployment_name` argument is required.""",
-            parameters={
-                "min_duration": ToolParameter(
-                    description="The minimum duration of traces to fetch, e.g., '5s' for 5 seconds.",
-                    type="string",
-                    required=True,
-                ),
-                "service_name": ToolParameter(
-                    description="Filter traces by service name",
-                    type="string",
-                    required=False,
-                ),
-                "pod_name": ToolParameter(
-                    description="Filter traces by pod name",
-                    type="string",
-                    required=False,
-                ),
-                "namespace_name": ToolParameter(
-                    description="Filter traces by namespace",
-                    type="string",
-                    required=False,
-                ),
-                "deployment_name": ToolParameter(
-                    description="Filter traces by deployment name",
-                    type="string",
-                    required=False,
-                ),
-                "node_name": ToolParameter(
-                    description="Filter traces by node",
-                    type="string",
-                    required=False,
-                ),
-                "start_datetime": ToolParameter(
-                    description=f"The beginning time boundary for the trace search period. String in RFC3339 format. If a negative integer, the number of seconds relative to the end_timestamp. Defaults to -{DEFAULT_TRACES_TIME_SPAN_SECONDS}",
-                    type="string",
-                    required=False,
-                ),
-                "end_datetime": ToolParameter(
-                    description="The ending time boundary for the trace search period. String in RFC3339 format. Defaults to NOW().",
-                    type="string",
-                    required=False,
-                ),
-                "limit": ToolParameter(
-                    description="Maximum number of traces to return. Defaults to 50",
-                    type="string",
-                    required=False,
-                ),
-                "sort": ToolParameter(
-                    description="One of 'descending', 'ascending' or 'none' for no sorting. Defaults to descending",
-                    type="string",
-                    required=False,
-                ),
-            },
-        )
-        self._toolset = toolset
-
-    def _invoke(
-        self, params: dict, user_approved: bool = False
-    ) -> StructuredToolResult:
-        api_key = self._toolset.grafana_config.api_key
-        headers = self._toolset.grafana_config.headers
-
-        invalid_params_error = validate_params(
-            params, ["service_name", "pod_name", "deployment_name"]
-        )
-        if invalid_params_error:
-            return StructuredToolResult(
-                status=ToolResultStatus.ERROR,
-                error=invalid_params_error,
-                params=params,
-            )
-
-        start, end = process_timestamps_to_int(
-            params.get("start_datetime"),
-            params.get("end_datetime"),
-            default_time_span_seconds=DEFAULT_TRACES_TIME_SPAN_SECONDS,
-        )
-
-        filters = self._toolset.build_k8s_filters(params, use_exact_match=True)
-
-        filters.append(f'duration>{get_param_or_raise(params, "min_duration")}')
-
-        query = " && ".join(filters)
-        query = f"{{{query}}}"
-
-        base_url = get_base_url(self._toolset.grafana_config)
-        traces = query_tempo_traces(
-            base_url=base_url,
-            api_key=api_key,
-            headers=headers,
-            query=query,
-            start=start,
-            end=end,
-            limit=params.get("limit", 50),
-        )
-        return StructuredToolResult(
-            status=ToolResultStatus.SUCCESS,
-            data=format_traces_list(traces),
-            params=params,
-            invocation=query,
-        )
-
-    def get_parameterized_one_liner(self, params: Dict) -> str:
-        return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo Traces (min_duration={params.get('min_duration')})"
-
-
-class GetTempoTags(Tool):
-    def __init__(self, toolset: BaseGrafanaTempoToolset):
-        super().__init__(
-            name="fetch_tempo_tags",
-            description="List the tags available in Tempo",
-            parameters={
-                "start_datetime": ToolParameter(
-                    description=f"The beginning time boundary for the search period. String in RFC3339 format. If a negative integer, the number of seconds relative to the end_timestamp. Defaults to -{DEFAULT_TAGS_TIME_SPAN_SECONDS}",
-                    type="string",
-                    required=False,
-                ),
-                "end_datetime": ToolParameter(
-                    description="The ending time boundary for the search period. String in RFC3339 format. Defaults to NOW().",
-                    type="string",
-                    required=False,
-                ),
-            },
-        )
-        self._toolset = toolset
-
-    def _invoke(
-        self, params: dict, user_approved: bool = False
-    ) -> StructuredToolResult:
-        api_key = self._toolset.grafana_config.api_key
-        headers = self._toolset.grafana_config.headers
-        start, end = process_timestamps_to_int(
-            start=params.get("start_datetime"),
-            end=params.get("end_datetime"),
-            default_time_span_seconds=DEFAULT_TAGS_TIME_SPAN_SECONDS,
-        )
-
-        base_url = get_base_url(self._toolset.grafana_config)
-        url = f"{base_url}/api/v2/search/tags?start={start}&end={end}"
-
-        try:
-            response = requests.get(
-                url,
-                headers=build_headers(api_key=api_key, additional_headers=headers),
-                timeout=60,
-            )
-            response.raise_for_status()  # Raise an error for non-2xx responses
-            data = response.json()
-            return StructuredToolResult(
-                status=ToolResultStatus.SUCCESS,
-                data=yaml.dump(data.get("scopes")),
-                params=params,
-            )
-        except requests.exceptions.RequestException as e:
-            raise Exception(f"Failed to retrieve tags: {e} \n for URL: {url}")
-
-    def get_parameterized_one_liner(self, params: Dict) -> str:
-        return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo tags"
-
-
-class GetTempoTraceById(Tool):
-    def __init__(self, toolset: BaseGrafanaTempoToolset):
-        super().__init__(
-            name="fetch_tempo_trace_by_id",
-            description="""Retrieves detailed information about a Tempo trace using its trace ID. Use this to investigate a trace.""",
-            parameters={
-                "trace_id": ToolParameter(
-                    description="The unique trace ID to fetch.",
-                    type="string",
-                    required=True,
-                ),
-            },
-        )
-        self._toolset = toolset
-
-    def _invoke(
-        self, params: dict, user_approved: bool = False
-    ) -> StructuredToolResult:
-        labels_mapping = self._toolset.grafana_config.labels
-        labels = list(labels_mapping.model_dump().values())
-
-        base_url = get_base_url(self._toolset.grafana_config)
-        trace_data = query_tempo_trace_by_id(
-            base_url=base_url,
-            api_key=self._toolset.grafana_config.api_key,
-            headers=self._toolset.grafana_config.headers,
-            trace_id=get_param_or_raise(params, "trace_id"),
-            key_labels=labels,
-        )
-        return StructuredToolResult(
-            status=ToolResultStatus.SUCCESS,
-            data=trace_data,
-            params=params,
-        )
-
-    def get_parameterized_one_liner(self, params: Dict) -> str:
-        return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo Trace (trace_id={params.get('trace_id')})"
+# class GetTempoTraces(Tool):
+#     def __init__(self, toolset: BaseGrafanaTempoToolset):
+#         super().__init__(
+#             name="fetch_tempo_traces",
+#             description="""Lists Tempo traces. At least one of `service_name`, `pod_name` or `deployment_name` argument is required.""",
+#             parameters={
+#                 "min_duration": ToolParameter(
+#                     description="The minimum duration of traces to fetch, e.g., '5s' for 5 seconds.",
+#                     type="string",
+#                     required=True,
+#                 ),
+#                 "service_name": ToolParameter(
+#                     description="Filter traces by service name",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "pod_name": ToolParameter(
+#                     description="Filter traces by pod name",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "namespace_name": ToolParameter(
+#                     description="Filter traces by namespace",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "deployment_name": ToolParameter(
+#                     description="Filter traces by deployment name",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "node_name": ToolParameter(
+#                     description="Filter traces by node",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "start_datetime": ToolParameter(
+#                     description=f"The beginning time boundary for the trace search period. String in RFC3339 format. If a negative integer, the number of seconds relative to the end_timestamp. Defaults to -{DEFAULT_TRACES_TIME_SPAN_SECONDS}",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "end_datetime": ToolParameter(
+#                     description="The ending time boundary for the trace search period. String in RFC3339 format. Defaults to NOW().",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "limit": ToolParameter(
+#                     description="Maximum number of traces to return. Defaults to 50",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "sort": ToolParameter(
+#                     description="One of 'descending', 'ascending' or 'none' for no sorting. Defaults to descending",
+#                     type="string",
+#                     required=False,
+#                 ),
+#             },
+#         )
+#         self._toolset = toolset
+#
+#     def _invoke(self, params: Dict, user_approved: bool = False) -> StructuredToolResult:
+#         # Create API instance
+#         api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+#
+#         invalid_params_error = validate_params(
+#             params, ["service_name", "pod_name", "deployment_name"]
+#         )
+#         if invalid_params_error:
+#             return StructuredToolResult(
+#                 status=ToolResultStatus.ERROR,
+#                 error=invalid_params_error,
+#                 params=params,
+#             )
+#
+#         start, end = process_timestamps_to_int(
+#             params.get("start_datetime"),
+#             params.get("end_datetime"),
+#             default_time_span_seconds=DEFAULT_TRACES_TIME_SPAN_SECONDS,
+#         )
+#
+#         filters = self._toolset.build_k8s_filters(params, use_exact_match=True)
+#
+#         filters.append(f'duration>{get_param_or_raise(params, "min_duration")}')
+#
+#         query = " && ".join(filters)
+#         query = f"{{{query}}}"
+#
+#         traces = api.search_traces_by_query(
+#             q=query,
+#             start=start,
+#             end=end,
+#             limit=params.get("limit", 50),
+#         )
+#         return StructuredToolResult(
+#             status=ToolResultStatus.SUCCESS,
+#             data=format_traces_list(traces),
+#             params=params,
+#             invocation=query,
+#         )
+#
+#     def get_parameterized_one_liner(self, params: Dict) -> str:
+#         return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo Traces (min_duration={params.get('min_duration')})"
+
+
+# class GetTempoTags(Tool):
+#     def __init__(self, toolset: BaseGrafanaTempoToolset):
+#         super().__init__(
+#             name="fetch_tempo_tags",
+#             description="List the tags available in Tempo",
+#             parameters={
+#                 "start_datetime": ToolParameter(
+#                     description=f"The beginning time boundary for the search period. String in RFC3339 format. If a negative integer, the number of seconds relative to the end_timestamp. Defaults to -{DEFAULT_TAGS_TIME_SPAN_SECONDS}",
+#                     type="string",
+#                     required=False,
+#                 ),
+#                 "end_datetime": ToolParameter(
+#                     description="The ending time boundary for the search period. String in RFC3339 format. Defaults to NOW().",
+#                     type="string",
+#                     required=False,
+#                 ),
+#             },
+#         )
+#         self._toolset = toolset
+#
+#     def _invoke(self, params: Dict, user_approved: bool = False) -> StructuredToolResult:
+#         # Create API instance
+#         api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+#
+#         start, end = process_timestamps_to_int(
+#             start=params.get("start_datetime"),
+#             end=params.get("end_datetime"),
+#             default_time_span_seconds=DEFAULT_TAGS_TIME_SPAN_SECONDS,
+#         )
+#
+#         try:
+#             data = api.search_tag_names_v2(start=start, end=end)
+#             return StructuredToolResult(
+#                 status=ToolResultStatus.SUCCESS,
+#                 data=yaml.dump(data.get("scopes")),
+#                 params=params,
+#             )
+#         except Exception as e:
+#             return StructuredToolResult(
+#                 status=ToolResultStatus.ERROR,
+#                 error=f"Failed to retrieve tags: {str(e)}",
+#                 params=params,
+#             )
+#
+#     def get_parameterized_one_liner(self, params: Dict) -> str:
+#         return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo tags"
+#
+
+
+# class GetTempoTraceById(Tool):
+#     def __init__(self, toolset: BaseGrafanaTempoToolset):
+#         super().__init__(
+#             name="fetch_tempo_trace_by_id",
+#             description="""Retrieves detailed information about a Tempo trace using its trace ID. Use this to investigate a trace.""",
+#             parameters={
+#                 "trace_id": ToolParameter(
+#                     description="The unique trace ID to fetch.",
+#                     type="string",
+#                     required=True,
+#                 ),
+#             },
+#         )
+#         self._toolset = toolset
+#
+#     def _invoke(self, params: Dict, user_approved: bool = False) -> StructuredToolResult:
+#         # Create API instance
+#         api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+#
+#         labels_mapping = self._toolset.grafana_config.labels
+#         labels = list(labels_mapping.model_dump().values())
+#
+#         # Get raw trace data
+#         trace_data = api.query_trace_by_id_v2(
+#             trace_id=get_param_or_raise(params, "trace_id")
+#         )
+#
+#         # Process the trace data (new API returns raw data)
+#         formatted_trace = process_trace(trace_data, labels)
+#
+#         return StructuredToolResult(
+#             status=ToolResultStatus.SUCCESS,
+#             data=formatted_trace,
+#             params=params,
+#         )
+#
+#     def get_parameterized_one_liner(self, params: Dict) -> str:
+#         return f"{toolset_name_for_one_liner(self._toolset.name)}: Fetched Tempo Trace (trace_id={params.get('trace_id')})"
+#
 
 
 class FetchTracesSimpleComparison(Tool):
@@ -426,20 +413,31 @@ Examples:
                 default_time_span_seconds=DEFAULT_TRACES_TIME_SPAN_SECONDS,
             )
 
-            base_url = get_base_url(self._toolset.grafana_config)
+            # Create API instance
+            api = GrafanaTempoAPI(
+                self._toolset.grafana_config, use_post=TEMPO_API_USE_POST
+            )
 
             # Step 1: Get all trace summaries
             stats_query = f"{{{base_query}}}"
-            all_traces_response = query_tempo_traces(
-                base_url=base_url,
-                api_key=self._toolset.grafana_config.api_key,
-                headers=self._toolset.grafana_config.headers,
-                query=stats_query,
+
+            # Debug log the query (useful for troubleshooting)
+            import logging
+
+            logger = logging.getLogger(__name__)
+            logger.info(f"Tempo query: {stats_query}")
+
+            logger.info(f"start: {start}, end: {end}")
+
+            all_traces_response = api.search_traces_by_query(
+                q=stats_query,
                 start=start,
                 end=end,
                 limit=1000,
             )
 
+            logger.info(f"Response: {all_traces_response}")
+
             traces = all_traces_response.get("traces", [])
             if not traces:
                 return StructuredToolResult(
@@ -488,39 +486,22 @@ Examples:
                     return None
 
                 try:
-                    url = f"{base_url}/api/traces/{trace_id}"
-                    response = requests.get(
-                        url,
-                        headers=build_headers(
-                            api_key=self._toolset.grafana_config.api_key,
-                            additional_headers=self._toolset.grafana_config.headers,
-                        ),
-                        timeout=5,
-                    )
-                    response.raise_for_status()
+                    trace_data = api.query_trace_by_id_v2(trace_id=trace_id)
                     return {
                         "traceID": trace_id,
                         "durationMs": trace_summary.get("durationMs", 0),
                         "rootServiceName": trace_summary.get(
                             "rootServiceName", "unknown"
                         ),
-                        "traceData": response.json(),  # Raw trace data
+                        "traceData": trace_data,  # Raw trace data
                     }
-                except requests.exceptions.RequestException as e:
+                except Exception as e:
                     error_msg = f"Failed to fetch full trace: {str(e)}"
-                    if hasattr(e, "response") and e.response is not None:
-                        error_msg += f" (Status: {e.response.status_code})"
                     return {
                         "traceID": trace_id,
                         "durationMs": trace_summary.get("durationMs", 0),
                         "error": error_msg,
                     }
-                except (ValueError, KeyError) as e:
-                    return {
-                        "traceID": trace_id,
-                        "durationMs": trace_summary.get("durationMs", 0),
-                        "error": f"Failed to parse trace data: {str(e)}",
-                    }
 
             # Fetch the selected traces
             result = {
@@ -553,6 +534,532 @@ Examples:
         return f"{toolset_name_for_one_liner(self._toolset.name)}: Simple Tempo Traces Comparison"
 
 
+# New tools matching GrafanaTempoAPI methods
+
+
+class SearchTracesByQuery(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="search_traces_by_query",
+            description=(
+                "Search for traces using TraceQL query language. "
+                "Uses the Tempo API endpoint: GET /api/search with 'q' parameter. "
+                'TraceQL allows complex filtering like: {resource.service.name="api"} && {span.http.status_code=500}'
+            ),
+            parameters={
+                "q": ToolParameter(
+                    description="TraceQL query (e.g., '{resource.service.name=\"api\" && span.http.status_code=500}')",
+                    type="string",
+                    required=True,
+                ),
+                "limit": ToolParameter(
+                    description="Maximum number of traces to return",
+                    type="integer",
+                    required=False,
+                ),
+                "start": ToolParameter(
+                    description="Start time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "spss": ToolParameter(
+                    description="Spans per span set",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.search_traces_by_query(
+                q=params["q"],
+                limit=params.get("limit"),
+                start=params.get("start"),
+                end=params.get("end"),
+                spss=params.get("spss"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Searched traces with TraceQL"
+
+
+class SearchTracesByTags(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="search_traces_by_tags",
+            description=(
+                "Search for traces using logfmt-encoded tags. "
+                "Uses the Tempo API endpoint: GET /api/search with 'tags' parameter. "
+                'Example: resource.service.name="api" http.status_code="500"'
+            ),
+            parameters={
+                "tags": ToolParameter(
+                    description='Logfmt-encoded span/process attributes (e.g., \'resource.service.name="api" http.status_code="500"\')',
+                    type="string",
+                    required=True,
+                ),
+                "min_duration": ToolParameter(
+                    description="Minimum trace duration (e.g., '5s', '100ms')",
+                    type="string",
+                    required=False,
+                ),
+                "max_duration": ToolParameter(
+                    description="Maximum trace duration (e.g., '10s', '1000ms')",
+                    type="string",
+                    required=False,
+                ),
+                "limit": ToolParameter(
+                    description="Maximum number of traces to return",
+                    type="integer",
+                    required=False,
+                ),
+                "start": ToolParameter(
+                    description="Start time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "spss": ToolParameter(
+                    description="Spans per span set",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.search_traces_by_tags(
+                tags=params["tags"],
+                min_duration=params.get("min_duration"),
+                max_duration=params.get("max_duration"),
+                limit=params.get("limit"),
+                start=params.get("start"),
+                end=params.get("end"),
+                spss=params.get("spss"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Searched traces with tags"
+
+
+class QueryTraceById(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="query_trace_by_id",
+            description=(
+                "Retrieve detailed trace information by trace ID. "
+                "Uses the Tempo API endpoint: GET /api/v2/traces/{trace_id}. "
+                "Returns the full trace data in OpenTelemetry format."
+            ),
+            parameters={
+                "trace_id": ToolParameter(
+                    description="The unique trace ID to fetch",
+                    type="string",
+                    required=True,
+                ),
+                "start": ToolParameter(
+                    description="Optional start time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="Optional end time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            trace_data = api.query_trace_by_id_v2(
+                trace_id=params["trace_id"],
+                start=params.get("start"),
+                end=params.get("end"),
+            )
+
+            # Return raw trace data as YAML for readability
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(trace_data, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Retrieved trace {params.get('trace_id')}"
+
+
+class SearchTagNames(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="search_tag_names",
+            description=(
+                "Discover available tag names across traces. "
+                "Uses the Tempo API endpoint: GET /api/v2/search/tags. "
+                "Returns tags organized by scope (resource, span, intrinsic)."
+            ),
+            parameters={
+                "scope": ToolParameter(
+                    description="Filter by scope: 'resource', 'span', or 'intrinsic'",
+                    type="string",
+                    required=False,
+                ),
+                "q": ToolParameter(
+                    description="TraceQL query to filter tags (e.g., '{resource.cluster=\"us-east-1\"}')",
+                    type="string",
+                    required=False,
+                ),
+                "start": ToolParameter(
+                    description="Start time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "limit": ToolParameter(
+                    description="Maximum number of tag names to return",
+                    type="integer",
+                    required=False,
+                ),
+                "max_stale_values": ToolParameter(
+                    description="Maximum stale values parameter",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.search_tag_names_v2(
+                scope=params.get("scope"),
+                q=params.get("q"),
+                start=params.get("start"),
+                end=params.get("end"),
+                limit=params.get("limit"),
+                max_stale_values=params.get("max_stale_values"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Discovered tag names"
+
+
+class SearchTagValues(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="search_tag_values",
+            description=(
+                "Get all values for a specific tag. "
+                "Uses the Tempo API endpoint: GET /api/v2/search/tag/{tag}/values. "
+                "Useful for discovering what values exist for a given tag."
+            ),
+            parameters={
+                "tag": ToolParameter(
+                    description="The tag name to get values for (e.g., 'resource.service.name', 'http.status_code')",
+                    type="string",
+                    required=True,
+                ),
+                "q": ToolParameter(
+                    description="TraceQL query to filter tag values (e.g., '{resource.cluster=\"us-east-1\"}')",
+                    type="string",
+                    required=False,
+                ),
+                "start": ToolParameter(
+                    description="Start time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time in Unix epoch seconds",
+                    type="integer",
+                    required=False,
+                ),
+                "limit": ToolParameter(
+                    description="Maximum number of values to return",
+                    type="integer",
+                    required=False,
+                ),
+                "max_stale_values": ToolParameter(
+                    description="Maximum stale values parameter",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.search_tag_values_v2(
+                tag=params["tag"],
+                q=params.get("q"),
+                start=params.get("start"),
+                end=params.get("end"),
+                limit=params.get("limit"),
+                max_stale_values=params.get("max_stale_values"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Retrieved values for tag '{params.get('tag')}'"
+
+
+class QueryMetricsInstant(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="query_metrics_instant",
+            description=(
+                "Compute a single TraceQL metric value across time range. "
+                "Uses the Tempo API endpoint: GET /api/metrics/query. "
+                "TraceQL metrics compute aggregated metrics from trace data. "
+                "Returns a single value for the entire time range. "
+                "Basic syntax: {selector} | function(attribute) [by (grouping)]\n\n"
+                "TraceQL metrics can help answer questions like:\n"
+                "- How many database calls across all systems are downstream of your application?\n"
+                "- What services beneath a given endpoint are failing?\n"
+                "- What services beneath an endpoint are slow?\n\n"
+                "TraceQL metrics help you answer these questions by parsing your traces in aggregate. "
+                "The instant version returns a single value for the query and is preferred over "
+                "query_metrics_range when you don't need the granularity of a full time-series but want "
+                "a total sum or single value computed across the whole time range."
+            ),
+            parameters={
+                "q": ToolParameter(
+                    description=(
+                        "TraceQL metrics query. Supported functions: rate, count_over_time, "
+                        "sum_over_time, max_over_time, min_over_time, avg_over_time, "
+                        "quantile_over_time, histogram_over_time, compare. "
+                        "Can use topk or bottomk modifiers. "
+                        "Syntax: {selector} | function(attribute) [by (grouping)]. "
+                        'Example: {resource.service.name="api"} | avg_over_time(duration)'
+                    ),
+                    type="string",
+                    required=True,
+                ),
+                "start": ToolParameter(
+                    description="Start time (Unix seconds/nanoseconds/RFC3339)",
+                    type="string",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time (Unix seconds/nanoseconds/RFC3339)",
+                    type="string",
+                    required=False,
+                ),
+                "since": ToolParameter(
+                    description="Duration string (e.g., '1h', '30m')",
+                    type="string",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.query_metrics_instant(
+                q=params["q"],
+                start=params.get("start"),
+                end=params.get("end"),
+                since=params.get("since"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return (
+            f"{toolset_name_for_one_liner(self._toolset.name)}: Computed TraceQL metric"
+        )
+
+
+class QueryMetricsRange(Tool):
+    def __init__(self, toolset: BaseGrafanaTempoToolset):
+        super().__init__(
+            name="query_metrics_range",
+            description=(
+                "Get time series data from TraceQL metrics queries. "
+                "Uses the Tempo API endpoint: GET /api/metrics/query_range. "
+                "Returns metrics computed at regular intervals (controlled by 'step' parameter). "
+                "Use this for graphing metrics over time or analyzing trends. "
+                "Basic syntax: {selector} | function(attribute) [by (grouping)]\n\n"
+                "TraceQL metrics can help answer questions like:\n"
+                "- How many database calls across all systems are downstream of your application?\n"
+                "- What services beneath a given endpoint are failing?\n"
+                "- What services beneath an endpoint are slow?\n\n"
+                "TraceQL metrics help you answer these questions by parsing your traces in aggregate."
+            ),
+            parameters={
+                "q": ToolParameter(
+                    description=(
+                        "TraceQL metrics query. Supported functions: rate, count_over_time, "
+                        "sum_over_time, max_over_time, min_over_time, avg_over_time, "
+                        "quantile_over_time, histogram_over_time, compare. "
+                        "Can use topk or bottomk modifiers. "
+                        "Syntax: {selector} | function(attribute) [by (grouping)]. "
+                        'Example: {resource.service.name="api"} | avg_over_time(duration)'
+                    ),
+                    type="string",
+                    required=True,
+                ),
+                "step": ToolParameter(
+                    description="Time series granularity (e.g., '1m', '5m', '1h')",
+                    type="string",
+                    required=False,
+                ),
+                "start": ToolParameter(
+                    description="Start time (Unix seconds/nanoseconds/RFC3339)",
+                    type="string",
+                    required=False,
+                ),
+                "end": ToolParameter(
+                    description="End time (Unix seconds/nanoseconds/RFC3339)",
+                    type="string",
+                    required=False,
+                ),
+                "since": ToolParameter(
+                    description="Duration string (e.g., '3h', '1d')",
+                    type="string",
+                    required=False,
+                ),
+                "exemplars": ToolParameter(
+                    description="Maximum number of exemplars to return",
+                    type="integer",
+                    required=False,
+                ),
+            },
+        )
+        self._toolset = toolset
+
+    def _invoke(
+        self, params: Dict, user_approved: bool = False
+    ) -> StructuredToolResult:
+        api = GrafanaTempoAPI(self._toolset.grafana_config, use_post=TEMPO_API_USE_POST)
+
+        try:
+            result = api.query_metrics_range(
+                q=params["q"],
+                step=params.get("step"),
+                start=params.get("start"),
+                end=params.get("end"),
+                since=params.get("since"),
+                exemplars=params.get("exemplars"),
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=yaml.dump(result, default_flow_style=False),
+                params=params,
+            )
+        except Exception as e:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=str(e),
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        return f"{toolset_name_for_one_liner(self._toolset.name)}: Retrieved TraceQL metrics time series"
+
+
 class GrafanaTempoToolset(BaseGrafanaTempoToolset):
     def __init__(self):
         super().__init__(
@@ -562,9 +1069,16 @@ class GrafanaTempoToolset(BaseGrafanaTempoToolset):
             docs_url="https://holmesgpt.dev/data-sources/builtin-toolsets/grafanatempo/",
             tools=[
                 FetchTracesSimpleComparison(self),
-                GetTempoTraces(self),
-                GetTempoTraceById(self),
-                GetTempoTags(self),
+                # GetTempoTraces(self),
+                # GetTempoTraceById(self),
+                # GetTempoTags(self),
+                SearchTracesByQuery(self),
+                SearchTracesByTags(self),
+                QueryTraceById(self),
+                SearchTagNames(self),
+                SearchTagValues(self),
+                QueryMetricsInstant(self),
+                QueryMetricsRange(self),
             ],
         )
         template_file_path = os.path.abspath(