holmesgpt 0.11.5__py3-none-any.whl → 0.12.0__py3-none-any.whl

holmes/plugins/toolsets/datadog/datadog_traces_formatter.py

@@ -0,0 +1,310 @@
+"""Formatting utilities for Datadog traces output."""
+
+from collections import defaultdict
+from datetime import datetime
+from typing import Any, Dict, List, Tuple
+
+from holmes.plugins.toolsets.utils import unix_nano_to_rfc3339
+
+
+def parse_datadog_span_timestamp(attrs: Dict[str, Any]) -> Tuple[int, int]:
+    """
+    Parse timestamp and duration from Datadog span attributes.
+
+    Returns:
+        Tuple of (start_ns, duration_ns)
+    """
+    custom = attrs.get("custom", {})
+
+    # Get timestamp and convert to nanoseconds
+    start_timestamp = attrs.get("start_timestamp", "")
+    # Check for duration in both custom and direct attributes
+    duration_ns = custom.get("duration", 0) or attrs.get("duration", 0)
+
+    # Check for start time in nanoseconds directly first
+    start_ns = attrs.get("start", 0)
+
+    # If not found, try to parse from timestamp string
+    if not start_ns and start_timestamp:
+        try:
+            dt = datetime.fromisoformat(start_timestamp.replace("Z", "+00:00"))
+            start_ns = int(dt.timestamp() * 1_000_000_000)
+        except (ValueError, TypeError):
+            start_ns = 0
+
+    return start_ns, duration_ns
+
+
+def format_traces_list(spans: List[Dict[str, Any]], limit: int = 50) -> str:
+    """
+    Format a list of spans grouped by trace ID into a readable output.
+    """
+    if not spans:
+        return ""
+
+    # Group spans by trace_id
+    traces = defaultdict(list)
+    for span in spans:
+        trace_id = span.get("attributes", {}).get("trace_id")
+        if trace_id:
+            traces[trace_id].append(span)
+
+    # Format output
+    output_lines = []
+    output_lines.append(f"Found {len(traces)} traces with matching spans")
+    output_lines.append("")
+
+    for trace_id, trace_spans in list(traces.items())[:limit]:
+        # Find root span and calculate trace duration
+        root_span = None
+        min_start = float("inf")
+        max_end = 0
+
+        for span in trace_spans:
+            attrs = span.get("attributes", {})
+            start_ns, duration_ns = parse_datadog_span_timestamp(attrs)
+            end_ns = start_ns + duration_ns
+
+            if start_ns > 0 and start_ns < min_start:
+                min_start = start_ns
+
+            if end_ns > max_end:
+                max_end = end_ns
+
+            # Check if this is a root span (no parent_id)
+            if not attrs.get("parent_id"):
+                root_span = span
+
+        # If no root span found, use the first span
+        if not root_span and trace_spans:
+            root_span = trace_spans[0]
+
+        # Calculate duration, handling edge cases
+        if min_start == float("inf") or max_end == 0:
+            trace_duration_ms = 0.0
+        else:
+            trace_duration_ms = (max_end - min_start) / 1_000_000
+
+        if root_span:
+            attrs = root_span.get("attributes", {})
+            service_name = attrs.get("service", "unknown")
+            operation_name = attrs.get("operation_name", "unknown")
+            start_time_str = (
+                unix_nano_to_rfc3339(min_start)
+                if min_start != float("inf")
+                else "unknown"
+            )
+
+            output_lines.append(
+                f"Trace (traceID={trace_id}) (durationMs={trace_duration_ms:.2f})"
+            )
+            output_lines.append(
+                f"\tstartTime={start_time_str} rootServiceName={service_name} rootTraceName={operation_name}"
+            )
+
+    return "\n".join(output_lines)
+
+
+def build_span_hierarchy(
+    spans: List[Dict[str, Any]],
+) -> Tuple[Dict[str, Dict], List[Dict]]:
+    """
+    Build a hierarchy of spans from a flat list.
+
+    Returns:
+        Tuple of (span_map, root_spans)
+    """
+    span_map = {}
+    root_spans = []
+
+    # First pass: create span objects
+    for span_data in spans:
+        attrs = span_data.get("attributes", {})
+        span_id = attrs.get("span_id", "")
+        parent_id = attrs.get("parent_id", "")
+
+        start_ns, duration_ns = parse_datadog_span_timestamp(attrs)
+
+        span_obj = {
+            "span_id": span_id,
+            "parent_id": parent_id,
+            "name": attrs.get("operation_name", "unknown"),
+            "service": attrs.get("service", "unknown"),
+            "resource": attrs.get("resource_name", ""),
+            "start_ns": start_ns,
+            "duration_ns": duration_ns,
+            "status": attrs.get("status", ""),
+            "tags": attrs.get("tags", []),
+            "children": [],
+            "attributes": attrs,
+        }
+
+        span_map[span_id] = span_obj
+
+        if not parent_id:
+            root_spans.append(span_obj)
+
+    # Second pass: build hierarchy
+    for span_obj in span_map.values():
+        parent_id = span_obj["parent_id"]
+        if parent_id and parent_id in span_map:
+            span_map[parent_id]["children"].append(span_obj)
+        elif parent_id and parent_id not in span_map:
+            # This is an orphaned span (parent not in trace)
+            root_spans.append(span_obj)
+
+    return span_map, root_spans
+
+
+def format_trace_hierarchy(trace_id: str, spans: List[Dict[str, Any]]) -> str:
+    """
+    Format a trace with its full span hierarchy.
+    """
+    if not spans:
+        return ""
+
+    span_map, root_spans = build_span_hierarchy(spans)
+
+    # Format output
+    output_lines = []
+    output_lines.append(f"Trace ID: {trace_id}")
+    output_lines.append("")
+
+    def format_span_tree(span: Dict[str, Any], level: int = 0) -> None:
+        indent = "  " * level
+        duration_ms = span["duration_ns"] / 1_000_000
+
+        output_lines.append(
+            f"{indent}├─ {span['name']} ({span['service']}) - {duration_ms:.2f}ms (span_id={span['span_id']})"
+        )
+
+        start_time_str = unix_nano_to_rfc3339(span["start_ns"])
+        end_time_ns = span["start_ns"] + span["duration_ns"]
+        end_time_str = unix_nano_to_rfc3339(end_time_ns)
+
+        output_lines.append(
+            f"{indent}│  Datetime: start={start_time_str} end={end_time_str}"
+        )
+
+        if span["resource"]:
+            output_lines.append(f"{indent}│  Resource: {span['resource']}")
+
+        if span["status"]:
+            output_lines.append(f"{indent}│  Status: {span['status']}")
+
+        # Show important tags
+        important_tags = [
+            "env",
+            "version",
+            "http.method",
+            "http.status_code",
+            "error.type",
+            "error.message",
+        ]
+        tags_to_show = {}
+
+        for tag in span["tags"]:
+            if isinstance(tag, str) and ":" in tag:
+                key, value = tag.split(":", 1)
+                if key in important_tags:
+                    tags_to_show[key] = value
+
+        if tags_to_show:
+            output_lines.append(f"{indent}│  Tags:")
+            for key, value in tags_to_show.items():
+                output_lines.append(f"{indent}│    {key}: {value}")
+
+        # Sort children by start time
+        sorted_children = sorted(span["children"], key=lambda s: s["start_ns"])
+        for child in sorted_children:
+            format_span_tree(child, level + 1)
+
+    # Format all root spans
+    for root_span in sorted(root_spans, key=lambda s: s["start_ns"]):
+        format_span_tree(root_span)
+
+    return "\n".join(output_lines)
+
+
+def format_spans_search(
+    spans: List[Dict[str, Any]], max_traces: int = 50, max_spans_per_trace: int = 10
+) -> str:
+    """
+    Format spans search results grouped by trace.
+    """
+    if not spans:
+        return ""
+
+    # Format output
+    output_lines = []
+    output_lines.append(f"Found {len(spans)} matching spans")
+    output_lines.append("")
+
+    # Group spans by trace for better readability
+    spans_by_trace = defaultdict(list)
+    for span in spans:
+        trace_id = span.get("attributes", {}).get("trace_id", "unknown")
+        spans_by_trace[trace_id].append(span)
+
+    output_lines.append(f"Spans grouped by {len(spans_by_trace)} traces:")
+    output_lines.append("")
+
+    for trace_id, trace_spans in list(spans_by_trace.items())[:max_traces]:
+        output_lines.append(f"Trace ID: {trace_id}")
+
+        # Sort spans by timestamp within each trace
+        sorted_spans = sorted(
+            trace_spans,
+            key=lambda s: parse_datadog_span_timestamp(s.get("attributes", {}))[0],
+        )
+
+        for span in sorted_spans[:max_spans_per_trace]:
+            attrs = span.get("attributes", {})
+
+            span_id = attrs.get("span_id", "unknown")
+            service = attrs.get("service", "unknown")
+            operation = attrs.get("operation_name", "unknown")
+            resource = attrs.get("resource_name", "")
+
+            start_ns, duration_ns = parse_datadog_span_timestamp(attrs)
+            duration_ms = duration_ns / 1_000_000
+            start_time_str = unix_nano_to_rfc3339(start_ns)
+
+            output_lines.append(f"  ├─ {operation} ({service}) - {duration_ms:.2f}ms")
+            output_lines.append(f"  │  span_id: {span_id}")
+            output_lines.append(f"  │  time: {start_time_str}")
+
+            if resource:
+                output_lines.append(f"  │  resource: {resource}")
+
+            # Show status if error
+            status = attrs.get("status", "")
+            if status and status != "ok":
+                output_lines.append(f"  │  status: {status}")
+
+            # Show important tags
+            tags = attrs.get("tags", [])
+            important_tags = {}
+            for tag in tags:
+                if isinstance(tag, str) and ":" in tag:
+                    key, value = tag.split(":", 1)
+                    if key in ["env", "version", "http.status_code", "error.type"]:
+                        important_tags[key] = value
+
+            if important_tags:
+                tags_str = ", ".join([f"{k}={v}" for k, v in important_tags.items()])
+                output_lines.append(f"  │  tags: {tags_str}")
+
+            output_lines.append("  │")
+
+        if len(trace_spans) > max_spans_per_trace:
+            output_lines.append(
+                f"  └─ ... and {len(trace_spans) - max_spans_per_trace} more spans in this trace"
+            )
+
+        output_lines.append("")
+
+    if len(spans_by_trace) > max_traces:
+        output_lines.append(f"... and {len(spans_by_trace) - max_traces} more traces")
+
+    return "\n".join(output_lines)

holmes/plugins/toolsets/datadog/instructions_datadog_traces.jinja2

@@ -0,0 +1,51 @@
+## Datadog Traces Toolset
+
+Tools to search and analyze distributed traces from Datadog APM.
+
+### Available Tools:
+- **fetch_datadog_traces** - List traces with filters (service, operation, duration)
+- **fetch_datadog_trace_by_id** - Get detailed span hierarchy for a specific trace
+- **fetch_datadog_spans** - Search spans with Datadog query syntax
+
+### Common Usage:
+
+```python
+# Find slow traces (>5s) for a service
+fetch_datadog_traces(service="backend-service", min_duration="5s")
+
+# Get trace details showing full span hierarchy
+fetch_datadog_trace_by_id(trace_id="6878d11e0000000064837efe7e97f5f8")
+
+# Search for errors using Datadog query syntax
+fetch_datadog_spans(query="@http.status_code:500")
+fetch_datadog_spans(service="api", query="status:error")
+
+# Time ranges (default: last hour)
+fetch_datadog_traces(
+    service="api",
+    start_datetime="-3600",  # 1 hour ago
+    end_datetime="0"         # now
+)
+```
+
+### Query Examples:
+
+```python
+# Performance issues
+fetch_datadog_traces(min_duration="2s", operation="GET /api/products")
+
+# Errors by service
+fetch_datadog_spans(service="payment", query="@http.status_code:5*")
+
+# Database queries
+fetch_datadog_spans(query="service:postgres @duration:>1000000000")
+
+# With tags
+fetch_datadog_spans(tags={"env": "production"}, query="error:true")
+```
+
+### Tips:
+- Duration units: ms, s, m (e.g., "500ms", "5s", "1m")
+- Time: RFC3339 format or negative seconds from now
+- Rate limit: 300 requests/hour
+- Default time range: 1 hour

holmes/plugins/toolsets/datadog/toolset_datadog_logs.py

@@ -0,0 +1,267 @@
+from enum import Enum
+import json
+import logging
+from typing import Any, Optional, Dict, Tuple
+from holmes.core.tools import (
+    CallablePrerequisite,
+    ToolsetTag,
+)
+from pydantic import BaseModel, Field
+from holmes.core.tools import StructuredToolResult, ToolResultStatus
+from holmes.plugins.toolsets.consts import TOOLSET_CONFIG_MISSING_ERROR
+from holmes.plugins.toolsets.datadog.datadog_api import (
+    DatadogBaseConfig,
+    DataDogRequestError,
+    execute_datadog_http_request,
+    get_headers,
+    MAX_RETRY_COUNT_ON_RATE_LIMIT,
+)
+from holmes.plugins.toolsets.logging_utils.logging_api import (
+    DEFAULT_TIME_SPAN_SECONDS,
+    BasePodLoggingToolset,
+    FetchPodLogsParams,
+    PodLoggingTool,
+)
+from holmes.plugins.toolsets.utils import process_timestamps_to_rfc3339
+
+
+class DataDogLabelsMapping(BaseModel):
+    pod: str = "pod_name"
+    namespace: str = "kube_namespace"
+
+
+class DataDogStorageTier(str, Enum):
+    INDEXES = "indexes"
+    ONLINE_ARCHIVES = "online-archives"
+    FLEX = "flex"
+
+
+DEFAULT_STORAGE_TIERS = [DataDogStorageTier.INDEXES]
+
+
+class DatadogLogsConfig(DatadogBaseConfig):
+    indexes: list[str] = ["*"]
+    # Ordered list of storage tiers. Works as fallback. Subsequent tiers are queried only if the previous tier yielded no result
+    storage_tiers: list[DataDogStorageTier] = Field(
+        default=DEFAULT_STORAGE_TIERS, min_length=1
+    )
+    labels: DataDogLabelsMapping = DataDogLabelsMapping()
+    page_size: int = 300
+    default_limit: int = 1000
+
+
+def calculate_page_size(
+    params: FetchPodLogsParams, dd_config: DatadogLogsConfig, logs: list
+) -> int:
+    logs_count = len(logs)
+
+    max_logs_count = dd_config.default_limit
+    if params.limit:
+        max_logs_count = params.limit
+
+    return min(dd_config.page_size, max(0, max_logs_count - logs_count))
+
+
+def fetch_paginated_logs(
+    params: FetchPodLogsParams,
+    dd_config: DatadogLogsConfig,
+    storage_tier: DataDogStorageTier,
+) -> list[dict]:
+    limit = params.limit or dd_config.default_limit
+
+    (from_time, to_time) = process_timestamps_to_rfc3339(
+        start_timestamp=params.start_time,
+        end_timestamp=params.end_time,
+        default_time_span_seconds=DEFAULT_TIME_SPAN_SECONDS,
+    )
+
+    url = f"{dd_config.site_api_url}/api/v2/logs/events/search"
+    headers = get_headers(dd_config)
+
+    query = f"{dd_config.labels.namespace}:{params.namespace}"
+    query += f" {dd_config.labels.pod}:{params.pod_name}"
+    if params.filter:
+        filter = params.filter.replace('"', '\\"')
+        query += f' "{filter}"'
+
+    payload: Dict[str, Any] = {
+        "filter": {
+            "from": from_time,
+            "to": to_time,
+            "query": query,
+            "indexes": dd_config.indexes,
+            "storage_tier": storage_tier.value,
+        },
+        "sort": "-timestamp",
+        "page": {"limit": calculate_page_size(params, dd_config, [])},
+    }
+
+    logs, cursor = execute_datadog_http_request(
+        url=url,
+        headers=headers,
+        payload_or_params=payload,
+        timeout=dd_config.request_timeout,
+    )
+
+    while cursor and len(logs) < limit:
+        payload["page"]["cursor"] = cursor
+        new_logs, cursor = execute_datadog_http_request(
+            url=url,
+            headers=headers,
+            payload_or_params=payload,
+            timeout=dd_config.request_timeout,
+        )
+        logs += new_logs
+        payload["page"]["limit"] = calculate_page_size(params, dd_config, logs)
+
+    # logs are fetched descending order. Unified logging API follows the pattern of kubectl logs where oldest logs are first
+    logs.reverse()
+
+    if len(logs) > limit:
+        logs = logs[-limit:]
+    return logs
+
+
+def format_logs(raw_logs: list[dict]) -> str:
+    logs = []
+
+    for raw_log_item in raw_logs:
+        message = raw_log_item.get("attributes", {}).get(
+            "message", json.dumps(raw_log_item)
+        )
+        logs.append(message)
+
+    return "\n".join(logs)
+
+
+class DatadogLogsToolset(BasePodLoggingToolset):
+    dd_config: Optional[DatadogLogsConfig] = None
+
+    def __init__(self):
+        super().__init__(
+            name="datadog/logs",
+            description="Toolset for interacting with Datadog to fetch logs",
+            docs_url="https://docs.datadoghq.com/api/latest/logs/",
+            icon_url="https://imgix.datadoghq.com//img/about/presskit/DDlogo.jpg",
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            tools=[
+                PodLoggingTool(self),
+            ],
+            experimental=True,
+            tags=[ToolsetTag.CORE],
+        )
+
+    def logger_name(self) -> str:
+        return "DataDog"
+
+    def fetch_pod_logs(self, params: FetchPodLogsParams) -> StructuredToolResult:
+        if not self.dd_config:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                data=TOOLSET_CONFIG_MISSING_ERROR,
+                params=params.model_dump(),
+            )
+
+        try:
+            raw_logs = []
+            for storage_tier in self.dd_config.storage_tiers:
+                raw_logs = fetch_paginated_logs(
+                    params, self.dd_config, storage_tier=storage_tier
+                )
+
+                if raw_logs:
+                    logs_str = format_logs(raw_logs)
+                    return StructuredToolResult(
+                        status=ToolResultStatus.SUCCESS,
+                        data=logs_str,
+                        params=params.model_dump(),
+                    )
+
+            return StructuredToolResult(
+                status=ToolResultStatus.NO_DATA,
+                params=params.model_dump(),
+            )
+
+        except DataDogRequestError as e:
+            logging.exception(e, exc_info=True)
+
+            # Provide more specific error message for rate limiting failures
+            if e.status_code == 429:
+                error_msg = f"Datadog API rate limit exceeded. Failed after {MAX_RETRY_COUNT_ON_RATE_LIMIT} retry attempts."
+            else:
+                error_msg = f"Exception while querying Datadog: {str(e)}"
+
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=error_msg,
+                params=params.model_dump(),
+                invocation=json.dumps(e.payload),
+            )
+
+        except Exception as e:
+            logging.exception(
+                f"Failed to query Datadog logs for params: {params}", exc_info=True
+            )
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"Exception while querying Datadog: {str(e)}",
+                params=params.model_dump(),
+            )
+
+    def _perform_healthcheck(self) -> Tuple[bool, str]:
+        """
+        Perform a healthcheck by fetching a single log from Datadog.
+        Returns (success, error_message).
+        """
+        try:
+            logging.info("Performing Datadog configuration healthcheck...")
+            healthcheck_params = FetchPodLogsParams(
+                namespace="*",
+                pod_name="*",
+                limit=1,
+                start_time="-172800",  # 48 hours in seconds
+            )
+
+            result = self.fetch_pod_logs(healthcheck_params)
+
+            if result.status == ToolResultStatus.ERROR:
+                error_msg = result.error or "Unknown error during healthcheck"
+                logging.error(f"Datadog healthcheck failed: {error_msg}")
+                return False, f"Datadog healthcheck failed: {error_msg}"
+            elif result.status == ToolResultStatus.NO_DATA:
+                error_msg = "No logs were found in the last 48 hours using wildcards for pod and namespace. Is the configuration correct?"
+                logging.error(f"Datadog healthcheck failed: {error_msg}")
+                return False, f"Datadog healthcheck failed: {error_msg}"
+
+            logging.info("Datadog healthcheck completed successfully")
+            return True, ""
+
+        except Exception as e:
+            logging.exception("Failed during Datadog healthcheck")
+            return False, f"Healthcheck failed with exception: {str(e)}"
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
+        if not config:
+            return (
+                False,
+                TOOLSET_CONFIG_MISSING_ERROR,
+            )
+
+        try:
+            dd_config = DatadogLogsConfig(**config)
+            self.dd_config = dd_config
+
+            # Perform healthcheck
+            success, error_msg = self._perform_healthcheck()
+            return success, error_msg
+
+        except Exception as e:
+            logging.exception("Failed to set up Datadog toolset")
+            return (False, f"Failed to parse Datadog configuration: {str(e)}")
+
+    def get_example_config(self) -> Dict[str, Any]:
+        return {
+            "dd_api_key": "your-datadog-api-key",
+            "dd_app_key": "your-datadog-application-key",
+            "site_api_url": "https://api.datadoghq.com",
+        }