holmesgpt 0.16.2a0__py3-none-any.whl → 0.18.4__py3-none-any.whl

holmes/plugins/toolsets/datadog/datadog_api.py

@@ -1,16 +1,17 @@
 import json
 import logging
 import re
+import threading
 from datetime import datetime, timedelta, timezone
-from typing import Any, Optional, Dict, Union, Tuple
+from typing import Any, Dict, Optional, Tuple, Union
 from urllib.parse import urlparse, urlunparse
+
 import requests  # type: ignore
 from pydantic import AnyUrl, BaseModel
 from requests.structures import CaseInsensitiveDict  # type: ignore
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_incrementing
 from tenacity.wait import wait_base
 
-
 START_RETRY_DELAY = (
     5.0  # Initial fallback delay if datadog does not return a reset_time
 )
@@ -22,6 +23,9 @@ RATE_LIMIT_REMAINING_SECONDS_HEADER = "X-RateLimit-Reset"
 # Cache for OpenAPI spec
 _openapi_spec_cache: Dict[str, Any] = {}
 
+# Global lock for Datadog API requests to prevent concurrent calls
+_datadog_request_lock = threading.Lock()
+
 # Relative time pattern (m = minutes, mo = months)
 RELATIVE_TIME_PATTERN = re.compile(r"^-?(\d+)([hdwsy]|min|m|mo)$|^now$", re.IGNORECASE)
 
@@ -237,6 +241,35 @@ def execute_datadog_http_request(
     payload_or_params: dict,
     timeout: int,
     method: str = "POST",
+) -> Any:
+    # from my limited testing doing 1 just request at a time is faster because the RATE_LIMIT_REMAINING_SECONDS_HEADER is shorter
+    # Serialize all Datadog API requests to avoid rate limits
+    with _datadog_request_lock:
+        return execute_datadog_http_request_with_retries(
+            url, headers, payload_or_params, timeout, method
+        )
+
+
+@retry(
+    retry=retry_if_http_429_error(),
+    wait=wait_for_retry_after_header(
+        fallback=wait_incrementing(
+            start=START_RETRY_DELAY, increment=INCREMENT_RETRY_DELAY
+        )
+    ),
+    stop=stop_after_attempt(MAX_RETRY_COUNT_ON_RATE_LIMIT),
+    before_sleep=lambda retry_state: logging.warning(
+        f"DataDog API rate limited. Retrying... "
+        f"(attempt {retry_state.attempt_number}/{MAX_RETRY_COUNT_ON_RATE_LIMIT})"
+    ),
+    reraise=True,
+)
+def execute_datadog_http_request_with_retries(
+    url: str,
+    headers: dict,
+    payload_or_params: dict,
+    timeout: int,
+    method: str,
 ) -> Any:
     logging.debug(
         f"Datadog API Request: Method: {method} URL: {url} Headers: {json.dumps(sanitize_headers(headers), indent=2)} {'Params' if method == 'GET' else 'Payload'}: {json.dumps(payload_or_params, indent=2)} Timeout: {timeout}s"
@@ -261,7 +294,7 @@ def execute_datadog_http_request(
         return response_data
 
     else:
-        logging.error(f"  Error Response Body: {response.text}")
+        logging.debug(f"Error Response Body: {response.text}")
         raise DataDogRequestError(
             payload=payload_or_params,
             status_code=response.status_code,

holmes/plugins/toolsets/datadog/datadog_logs_instructions.jinja2

@@ -44,7 +44,6 @@ Before running logs queries:
 ### Time Parameters
 - Use RFC3339 format: `2023-03-01T10:30:00Z`
 - Or relative seconds: `-3600` for 1 hour ago
-- Defaults to 1 hour window if not specified
 
 ### Common Investigation Patterns
 
@@ -52,3 +51,37 @@ Before running logs queries:
 1. User asks: "Show logs for my-workload"
 2. Use `kubectl_find_resource` → find pod "my-workload-abc123-xyz"
 3. Query Datadog for pod "my-workload-abc123-xyz" logs
+
+
+### Search Query Guidelines
+
+1. Avoid using @timestamp Attribute in the search queries (e.g  for example  @timestamp:[2025-12-10T01:00:00.000Z TO 2025-12-10T04:00:00.000Z)
+   Rely on the fetch_datadog_logs function start_datetime and end_datetime parameters for that.
+2. Datadog default TAGS for kubernetes are *kube_namespace* and *pod_name*, if a user specificy custom TAGS used in his environment please use them in your search queries.
+3. If you see a useful TAG in your Old fetch_datadog_logs query use it for further queries.
+
+### CRITICAL: Cursor Usage Rules
+**NEVER parallelize cursor-based calls or reuse cursor values!**
+
+Cursors are stateful pointers - each one is single-use and represents a unique position in the data stream.
+
+**WRONG (causes duplicate data):**
+```
+Batch 1 → cursor_A
+Then call Batch 2, 3, 4 ALL with cursor_A in parallel ❌
+Result: Duplicate data, incomplete results
+```
+
+**CORRECT (sequential pagination):**
+```
+Batch 1 → cursor_A
+Wait for response → use cursor_A for Batch 2 → cursor_B  
+Wait for response → use cursor_B for Batch 3 → cursor_C
+Result: Complete unique data ✅
+```
+
+**Key Rules:**
+- Each response provides a NEW cursor for the NEXT request
+- NEVER reuse the same cursor value multiple times
+- NEVER make parallel calls with the same cursor
+- Always wait for response before using the returned cursor

holmes/plugins/toolsets/datadog/datadog_metrics_instructions.jinja2

@@ -73,7 +73,7 @@ When investigating metrics-related issues:
 
 # Handling queries results
 * ALWAYS embed the execution results into your answer
-* You only need to embed the partial result in your response. Include the "tool_name" and "random_key". For example: << {"type": "datadogql", "tool_name": "query_datadog_metrics", "random_key": "92jf2hf"} >>
+* You only need to embed the partial result in your response. Include the "tool_name" and "tool_call_id". For example: << {"type": "datadogql", "tool_name": "query_datadog_metrics", "tool_call_id": "92jf2hf"} >>
 * Post processing will parse your response, re-run the query from the tool output and create a chart visible to the user
 * You MUST ensure that the query is successful.
 * ALWAYS embed a DataDog graph in the response. The graph should visualize data related to the incident.
@@ -81,6 +81,6 @@ When investigating metrics-related issues:
 * When embedding multiple graphs, always add line spacing between them
     For example:
 
-    <<{"type": "datadogql", "tool_name": "query_datadog_metrics", "random_key": "lBaA"}>>
+    <<{"type": "datadogql", "tool_name": "query_datadog_metrics", "tool_call_id": "lBaA"}>>
 
-    <<{"type": "datadogql", "tool_name": "query_datadog_metrics", "random_key": "IKtq"}>>
+    <<{"type": "datadogql", "tool_name": "query_datadog_metrics", "tool_call_id": "IKtq"}>>

holmes/plugins/toolsets/datadog/datadog_models.py

@@ -0,0 +1,59 @@
+from enum import Enum
+
+from pydantic import Field
+
+from holmes.plugins.toolsets.datadog.datadog_api import DatadogBaseConfig
+from holmes.plugins.toolsets.logging_utils.logging_api import DEFAULT_LOG_LIMIT
+
+# Constants for RDS toolset
+DEFAULT_TIME_SPAN_SECONDS = 3600
+DEFAULT_TOP_INSTANCES = 10
+
+# Constants for general toolset
+MAX_RESPONSE_SIZE = 10 * 1024 * 1024  # 10MB
+
+
+class DataDogStorageTier(str, Enum):
+    """Storage tier enum for Datadog logs."""
+
+    INDEXES = "indexes"
+    ONLINE_ARCHIVES = "online-archives"
+    FLEX = "flex"
+
+
+# Constants for logs toolset
+DEFAULT_STORAGE_TIERS = [DataDogStorageTier.INDEXES]
+
+
+class DatadogMetricsConfig(DatadogBaseConfig):
+    """Configuration for Datadog metrics toolset."""
+
+    default_limit: int = DEFAULT_LOG_LIMIT
+
+
+class DatadogTracesConfig(DatadogBaseConfig):
+    """Configuration for Datadog traces toolset."""
+
+    indexes: list[str] = ["*"]
+
+
+class DatadogLogsConfig(DatadogBaseConfig):
+    """Configuration for Datadog logs toolset."""
+
+    indexes: list[str] = ["*"]
+    # TODO storage tier just works with first element. need to add support for multi stoarge tiers.
+    storage_tiers: list[DataDogStorageTier] = Field(
+        default_factory=lambda: [DataDogStorageTier.INDEXES], min_length=1
+    )
+
+    compact_logs: bool = True
+    default_limit: int = DEFAULT_LOG_LIMIT
+
+
+class DatadogGeneralConfig(DatadogBaseConfig):
+    """Configuration for general-purpose Datadog toolset."""
+
+    max_response_size: int = MAX_RESPONSE_SIZE
+    allow_custom_endpoints: bool = (
+        False  # If True, allows endpoints not in whitelist (still filtered for safety)
+    )

holmes/plugins/toolsets/datadog/datadog_url_utils.py

@@ -0,0 +1,213 @@
+import re
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode, urlparse
+
+from holmes.plugins.toolsets.datadog.datadog_api import convert_api_url_to_app_url
+from holmes.plugins.toolsets.datadog.datadog_models import (
+    DatadogGeneralConfig,
+    DatadogLogsConfig,
+    DatadogMetricsConfig,
+    DatadogTracesConfig,
+)
+
+
+def generate_datadog_metrics_explorer_url(
+    dd_config: DatadogMetricsConfig,
+    query: str,
+    from_time: int,
+    to_time: int,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+
+    params = {
+        "query": query,
+        "from_ts": from_time * 1000,  # seconds -> ms
+        "to_ts": to_time * 1000,  # seconds -> ms
+        "live": "true",
+    }
+
+    return f"{base_url}/metric/explorer?{urlencode(params)}"
+
+
+def generate_datadog_metrics_list_url(
+    dd_config: DatadogMetricsConfig,
+    from_time: int,
+    host: Optional[str] = None,
+    tag_filter: Optional[str] = None,
+    metric_filter: Optional[str] = None,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+
+    params = {}
+    if metric_filter:
+        params["filter"] = metric_filter
+
+    if host:
+        params["host"] = host
+    if tag_filter:
+        params["tag_filter"] = tag_filter
+
+    qs = urlencode(params) if params else ""
+    return f"{base_url}/metric/summary" + (f"?{qs}" if qs else "")
+
+
+def generate_datadog_metric_metadata_url(
+    dd_config: DatadogMetricsConfig,
+    metric_name: str,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+    params = {"metric": metric_name}
+    return f"{base_url}/metric/summary?{urlencode(params)}"
+
+
+def generate_datadog_metric_tags_url(
+    dd_config: DatadogMetricsConfig,
+    metric_name: str,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+    params = {"metric": metric_name}
+    return f"{base_url}/metric/summary?{urlencode(params)}"
+
+
+def generate_datadog_spans_url(
+    dd_config: DatadogTracesConfig,
+    query: str,
+    from_time_ms: int,
+    to_time_ms: int,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+
+    url_params = {
+        "query": query,
+        "from_ts": from_time_ms,
+        "to_ts": to_time_ms,
+        "live": "true",
+    }
+
+    return f"{base_url}/apm/traces?{urlencode(url_params)}"
+
+
+def generate_datadog_spans_analytics_url(
+    dd_config: DatadogTracesConfig,
+    query: str,
+    from_time_ms: int,
+    to_time_ms: int,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+
+    url_params = {
+        "query": query,
+        "from_ts": from_time_ms,
+        "to_ts": to_time_ms,
+        "live": "true",
+    }
+
+    return f"{base_url}/apm/analytics?{urlencode(url_params)}"
+
+
+def generate_datadog_logs_url(
+    dd_config: DatadogLogsConfig,
+    params: dict,
+) -> str:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+    url_params = {
+        "query": params["filter"]["query"],
+        "from_ts": params["filter"]["from"],
+        "to_ts": params["filter"]["to"],
+        "live": "true",
+        "storage": params["filter"]["storage_tier"],
+    }
+
+    if dd_config.indexes != ["*"]:
+        url_params["index"] = ",".join(dd_config.indexes)
+
+    # Construct the full URL
+    return f"{base_url}/logs?{urlencode(url_params)}"
+
+
+def _build_qs(
+    query_params: Optional[Dict[str, Any]], allowed: Optional[set] = None
+) -> str:
+    if not query_params:
+        return ""
+    allowed = allowed or {
+        "filter",
+        "query",
+        "tags",
+        "status",
+        "start",
+        "end",
+        "from",
+        "to",
+    }
+    url_params = {}
+    for k, v in query_params.items():
+        if k not in allowed or v is None:
+            continue
+        if k in ("start", "from"):
+            url_params["from_ts"] = v * 1000
+        elif k in ("end", "to"):
+            url_params["to_ts"] = v * 1000
+        elif k in ("query", "filter", "tags"):
+            url_params["q"] = v
+        else:
+            url_params[k] = v
+    qs = urlencode(url_params) if url_params else ""
+    return f"?{qs}" if qs else ""
+
+
+def generate_datadog_general_url(
+    dd_config: DatadogGeneralConfig,
+    endpoint: str,
+    query_params: Optional[Dict[str, Any]] = None,
+) -> Optional[str]:
+    base_url = convert_api_url_to_app_url(dd_config.site_api_url)
+    path = urlparse(endpoint).path
+
+    if "/logs" in path:
+        return f"{base_url}/logs{_build_qs(query_params, {'start', 'end'})}"
+
+    if "/monitor" in path:
+        qs = _build_qs(query_params, {"filter", "query", "tags", "status"})
+        monitor_id_match = re.search(r"/monitor/(\d+)", path)
+        if monitor_id_match:
+            return f"{base_url}/monitors/{monitor_id_match.group(1)}{qs}"
+        return f"{base_url}/monitors{qs}"
+
+    if "/dashboard" in path:
+        qs = _build_qs(query_params, {"filter", "query", "tags"})
+        if re.match(r"^/api/v\d+/dashboard/[^/]+", path):
+            return f"{base_url}/dashboard/{path.split('/')[-1]}{qs}"
+        return f"{base_url}/dashboard{qs}"
+
+    if "/slo" in path:
+        qs = _build_qs(query_params, {"filter", "query", "tags"})
+        if re.match(r"^/api/v\d+/slo/[^/]+", path):
+            return f"{base_url}/slo/{path.split('/')[-1]}{qs}"
+        return f"{base_url}/slo{qs}"
+
+    if "/events" in path:
+        return f"{base_url}/events{_build_qs(query_params, {'start', 'end'})}"
+
+    if "/incidents" in path:
+        qs = _build_qs(query_params, {"filter", "query", "status"})
+        if re.match(r"^/api/v\d+/incidents/[^/]+", path):
+            return f"{base_url}/incidents/{path.split('/')[-1]}{qs}"
+        return f"{base_url}/incidents{qs}"
+
+    if "/synthetics" in path:
+        qs = _build_qs(query_params, {"filter", "query", "tags", "status"})
+        if re.match(r"^/api/v\d+/synthetics/tests/[^/]+", path):
+            return f"{base_url}/synthetics/tests/{path.split('/')[-1]}{qs}"
+        return f"{base_url}/synthetics/tests{qs}"
+
+    if "/hosts" in path:
+        return f"{base_url}/infrastructure{_build_qs(query_params, {'filter', 'query', 'tags'})}"
+
+    if "/services" in path:
+        return f"{base_url}/apm/services{_build_qs(query_params, {'filter', 'query', 'tags'})}"
+
+    if "/metrics" in path or "/query" in path:
+        return f"{base_url}/metrics/explorer{_build_qs(query_params, {'from', 'to', 'query'})}"
+
+    return f"{base_url}/apm/home"

holmes/plugins/toolsets/datadog/instructions_datadog_traces.jinja2

@@ -3,49 +3,186 @@
 Tools to search and analyze distributed traces from Datadog APM.
 
 ### Available Tools:
-- **fetch_datadog_traces** - List traces with filters (service, operation, duration)
-- **fetch_datadog_trace_by_id** - Get detailed span hierarchy for a specific trace
 - **fetch_datadog_spans** - Search spans with Datadog query syntax
+- **aggregate_datadog_spans** - Aggregate span data into buckets and compute metrics
 
 ### Common Usage:
 
 ```python
-# Find slow traces (>5s) for a service
-fetch_datadog_traces(service="backend-service", min_duration="5s")
+# Search for errors using Datadog query syntax
+fetch_datadog_spans(query="@http.status_code:500", limit=5)
+fetch_datadog_spans(query="service:api status:error", limit=10)
+```
 
-# Get trace details showing full span hierarchy
-fetch_datadog_trace_by_id(trace_id="6878d11e0000000064837efe7e97f5f8")
+### Query Patterns:
 
-# Search for errors using Datadog query syntax
-fetch_datadog_spans(query="@http.status_code:500")
-fetch_datadog_spans(service="api", query="status:error")
+```python
+# Specific HTTP endpoint (any method)
+fetch_datadog_spans(query="@http.route:/api/orders", limit=5)
+
+# HTTP routes containing substring (wildcard search)
+fetch_datadog_spans(query="@http.route:*payment*", limit=5)
+
+# Broad search across all span types
+fetch_datadog_spans(query="resource_name:*user*", limit=10)
+
+# Errors by service with wildcard
+fetch_datadog_spans(query="service:payment @http.status_code:5*", limit=5)
+
+# Database queries with time range (last hour)
+fetch_datadog_spans(
+    query="service:postgres @duration:>1000000000",
+    start_datetime="-3600",  # 1 hour in seconds
+    limit=10
+)
+
+# Production errors
+fetch_datadog_spans(query="env:production error:true", limit=5)
 
-# Time ranges (default: last hour)
-fetch_datadog_traces(
-    service="api",
-    start_datetime="-3600",  # 1 hour ago
-    end_datetime="0"         # now
+# Specific endpoint pattern with custom time range
+fetch_datadog_spans(
+    query='@http.route:*/user/* @http.status_code:>=400',
+    start_datetime="-1800",  # 30 minutes in seconds
+    limit=10
+)
+
+# Combining multiple conditions with wildcards
+fetch_datadog_spans(
+    query='service:*api* @http.route:*/user/* @http.status_code:[400 TO 599]',
+    limit=10
 )
 ```
 
-### Query Examples:
+### Aggregate Examples:
 
 ```python
-# Performance issues
-fetch_datadog_traces(min_duration="2s", operation="GET /api/products")
+# Count spans grouped by status code (last 15 minutes)
+aggregate_datadog_spans(
+    query='resource_name:*api* @http.method:POST',
+    compute=[{"aggregation": "count", "type": "total"}],
+    group_by=[{"facet": "@http.status_code", "limit": 50}],
+    start_datetime="-900"  # 15 minutes in seconds
+)
+
+# Get average duration by service (last hour)
+aggregate_datadog_spans(
+    query='service:*backend* OR service:*api*',
+    compute=[{"aggregation": "avg", "metric": "@duration", "type": "total"}],
+    group_by=[{"facet": "service", "limit": 50}],
+    start_datetime="-3600"  # 1 hour in seconds
+)
+
+# Get P95 latency timeseries by service
+aggregate_datadog_spans(
+    query='@http.route:*/api/* @http.status_code:[200 TO 299]',
+    compute=[{
+        "aggregation": "pc95", 
+        "metric": "@duration", 
+        "type": "timeseries",
+        "interval": "5m"
+    }],
+    group_by=[{"facet": "service", "limit": 50}]
+)
+
+# Complex aggregation with histogram
+aggregate_datadog_spans(
+    query='resource_name:*product* OR resource_name:*catalog*',
+    compute=[
+        {"aggregation": "avg", "metric": "@duration", "type": "total"},
+        {"aggregation": "count", "type": "total"}
+    ],
+    group_by=[{
+        "facet": "@duration",
+        "histogram": {"interval": 100, "min": 0, "max": 1000},
+        "limit": 50
+    }]
+)
+
+# Error rate calculation by endpoint
+aggregate_datadog_spans(
+    query='@http.route:* @http.status_code:[400 TO 599]',
+    compute=[{"aggregation": "count", "type": "total"}],
+    group_by=[
+        {"facet": "resource_name", "limit": 50},
+        {"facet": "@http.status_code", "limit": 50}
+    ]
+)
+```
+
+### Query Pattern Tips:
+
+| Your Goal | Use This Pattern |
+|-----------|------------------|
+| Specific HTTP endpoint, any method | `@http.route:/api/users` |
+| HTTP routes containing substring | `@http.route:*payment*` |
+| Broad search across all span types | `resource_name:*user*` |
+| Service name patterns | `service:*api*` or `service:payment-*` |
+| Multiple wildcards | `@http.route:*/user/*/profile` |
+| Error status codes | `@http.status_code:5*` or `@http.status_code:[400 TO 599]` |
+
+### General Tips:
+- Wildcards (*) can be used in most fields for flexible pattern matching
+- For aggregations: use @-prefixed attributes (e.g., @duration, @http.status_code)
+- Keep fetch_datadog_spans limit low (5-10) to avoid too much data
+- aggregate_datadog_spans can handle higher limits (50+) for group_by facets
+
+### CRITICAL: Cursor Usage Rules
+**NEVER parallelize cursor-based calls or reuse cursor values!**
+
+Cursors are stateful pointers - each one is single-use and represents a unique position in the data stream.
+
+**WRONG (causes duplicate data):**
+```
+Batch 1 → cursor_A
+Then call Batch 2, 3, 4 ALL with cursor_A in parallel ❌
+Result: Duplicate data, incomplete results
+```
 
-# Errors by service
-fetch_datadog_spans(service="payment", query="@http.status_code:5*")
+**CORRECT (sequential pagination):**
+```
+Batch 1 → cursor_A
+Wait for response → use cursor_A for Batch 2 → cursor_B  
+Wait for response → use cursor_B for Batch 3 → cursor_C
+Result: Complete unique data ✅
+```
+
+**Key Rules:**
+- Each response provides a NEW cursor for the NEXT request
+- NEVER reuse the same cursor value multiple times
+- NEVER make parallel calls with the same cursor
+- Always wait for response before using the returned cursor
+
+### Compact Mode Strategy:
+
+The `compact` parameter reduces output size by returning only essential fields. Use this strategy:
 
-# Database queries
-fetch_datadog_spans(query="service:postgres @duration:>1000000000")
+1. **Initial exploration**: Use compact=true with higher limits (50-100) to get an overview
+2. **Detailed investigation**: Use compact=false with lower limits (5-10) for specific spans
 
-# With tags
-fetch_datadog_spans(tags={"env": "production"}, query="error:true")
+```python
+# STEP 1: Initial search with compact mode to find patterns
+fetch_datadog_spans(
+    query="service:api @http.status_code:5*",
+    compact=true,
+    limit=100  # Higher limit safe with compact mode
+)
+
+# STEP 2: Detailed investigation of specific issues
+fetch_datadog_spans(
+    query="service:api @http.status_code:500 resource_name:*/user/*",
+    compact=false,  # Full details for deep analysis
+    limit=10
+)
 ```
 
-### Tips:
-- Duration units: ms, s, m (e.g., "500ms", "5s", "1m")
-- Time: RFC3339 format or negative seconds from now
-- Rate limit: 300 requests/hour
-- Default time range: 1 hour
+**When to use compact=true:**
+- Initial searches to identify patterns
+- When you need to scan many spans for errors or performance issues
+- When looking for specific span IDs or trace IDs
+- When the full span details aren't needed yet
+
+**When to use compact=false (default):**
+- Investigating specific errors
+- Analyzing request/response headers
+- Examining user agent details
+- Debugging authentication issues or HTTP details