holmesgpt 0.13.2__py3-none-any.whl → 0.18.4__py3-none-any.whl

holmes/plugins/toolsets/datadog/datadog_api.py

@@ -1,13 +1,17 @@
 import json
 import logging
-from typing import Any, Optional, Dict, Union
+import re
+import threading
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, Optional, Tuple, Union
+from urllib.parse import urlparse, urlunparse
+
 import requests  # type: ignore
 from pydantic import AnyUrl, BaseModel
 from requests.structures import CaseInsensitiveDict  # type: ignore
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_incrementing
 from tenacity.wait import wait_base
 
-
 START_RETRY_DELAY = (
     5.0  # Initial fallback delay if datadog does not return a reset_time
 )
@@ -16,6 +20,78 @@ MAX_RETRY_COUNT_ON_RATE_LIMIT = 5
 
 RATE_LIMIT_REMAINING_SECONDS_HEADER = "X-RateLimit-Reset"
 
+# Cache for OpenAPI spec
+_openapi_spec_cache: Dict[str, Any] = {}
+
+# Global lock for Datadog API requests to prevent concurrent calls
+_datadog_request_lock = threading.Lock()
+
+# Relative time pattern (m = minutes, mo = months)
+RELATIVE_TIME_PATTERN = re.compile(r"^-?(\d+)([hdwsy]|min|m|mo)$|^now$", re.IGNORECASE)
+
+
+def convert_api_url_to_app_url(api_url: Union[str, AnyUrl]) -> str:
+    """
+    Convert a Datadog API URL to the corresponding web app URL.
+
+    Handles various URL formats:
+    - https://api.datadoghq.com -> https://app.datadoghq.com
+    - https://api.datadoghq.eu -> https://app.datadoghq.eu
+    - https://api.us5.datadoghq.com -> https://app.us5.datadoghq.com
+    - Also handles URLs with paths like https://api.datadoghq.com/api/v1
+
+    Args:
+        api_url: The API URL to convert
+
+    Returns:
+        The web app URL without trailing slash
+    """
+    url_str = str(api_url)
+    parsed = urlparse(url_str)
+
+    # Replace 'api.' subdomain with 'app.' in the hostname
+    # This handles cases like:
+    # - api.datadoghq.com -> app.datadoghq.com
+    # - api.datadoghq.eu -> app.datadoghq.eu
+    # - api.us5.datadoghq.com -> app.us5.datadoghq.com
+    if parsed.hostname and parsed.hostname.startswith("api."):
+        new_hostname = "app." + parsed.hostname[4:]
+        # Reconstruct the netloc with the new hostname
+        if parsed.port:
+            new_netloc = f"{new_hostname}:{parsed.port}"
+        else:
+            new_netloc = new_hostname
+    else:
+        # If it doesn't start with 'api.', keep the hostname as is
+        # This handles edge cases where the URL might not follow the pattern
+        new_netloc = parsed.netloc
+
+    # Remove any /api path segments if present
+    # Some configurations might include /api/v1 or similar in the base URL
+    new_path = parsed.path
+    if new_path.startswith("/api/"):
+        new_path = new_path[4:]  # Remove '/api' prefix
+    elif new_path == "/api":
+        new_path = "/"
+
+    # Reconstruct the URL with the app subdomain
+    app_url = urlunparse(
+        (
+            parsed.scheme,
+            new_netloc,
+            new_path,
+            "",  # params
+            "",  # query
+            "",  # fragment
+        )
+    )
+
+    # Remove trailing slash
+    if app_url.endswith("/"):
+        app_url = app_url[:-1]
+
+    return app_url
+
 
 class DatadogBaseConfig(BaseModel):
     """Base configuration for all Datadog toolsets"""
@@ -166,15 +242,38 @@ def execute_datadog_http_request(
     timeout: int,
     method: str = "POST",
 ) -> Any:
-    # Log the request details
-    logging.info("Datadog API Request:")
-    logging.info(f"  Method: {method}")
-    logging.info(f"  URL: {url}")
-    logging.info(f"  Headers: {json.dumps(sanitize_headers(headers), indent=2)}")
-    logging.info(
-        f"  {'Params' if method == 'GET' else 'Payload'}: {json.dumps(payload_or_params, indent=2)}"
+    # from my limited testing doing 1 just request at a time is faster because the RATE_LIMIT_REMAINING_SECONDS_HEADER is shorter
+    # Serialize all Datadog API requests to avoid rate limits
+    with _datadog_request_lock:
+        return execute_datadog_http_request_with_retries(
+            url, headers, payload_or_params, timeout, method
+        )
+
+
+@retry(
+    retry=retry_if_http_429_error(),
+    wait=wait_for_retry_after_header(
+        fallback=wait_incrementing(
+            start=START_RETRY_DELAY, increment=INCREMENT_RETRY_DELAY
+        )
+    ),
+    stop=stop_after_attempt(MAX_RETRY_COUNT_ON_RATE_LIMIT),
+    before_sleep=lambda retry_state: logging.warning(
+        f"DataDog API rate limited. Retrying... "
+        f"(attempt {retry_state.attempt_number}/{MAX_RETRY_COUNT_ON_RATE_LIMIT})"
+    ),
+    reraise=True,
+)
+def execute_datadog_http_request_with_retries(
+    url: str,
+    headers: dict,
+    payload_or_params: dict,
+    timeout: int,
+    method: str,
+) -> Any:
+    logging.debug(
+        f"Datadog API Request: Method: {method} URL: {url} Headers: {json.dumps(sanitize_headers(headers), indent=2)} {'Params' if method == 'GET' else 'Payload'}: {json.dumps(payload_or_params, indent=2)} Timeout: {timeout}s"
     )
-    logging.info(f"  Timeout: {timeout}s")
 
     if method == "GET":
         response = requests.get(
@@ -186,31 +285,431 @@ def execute_datadog_http_request(
         )
 
     # Log the response details
-    logging.info("Datadog API Response:")
-    logging.info(f"  Status Code: {response.status_code}")
-    logging.info(f"  Response Headers: {dict(sanitize_headers(response.headers))}")
+    logging.debug(
+        f"Datadog API Response: Status Code: {response.status_code} Response Headers: {dict(sanitize_headers(response.headers))}"
+    )
 
     if response.status_code == 200:
         response_data = response.json()
-        # Log response size but not full content (could be large)
-        if isinstance(response_data, dict):
-            logging.info(f"  Response Keys: {list(response_data.keys())}")
-            if "data" in response_data:
-                data_len = (
-                    len(response_data["data"])
-                    if isinstance(response_data["data"], list)
-                    else 1
-                )
-                logging.info(f"  Data Items Count: {data_len}")
-        else:
-            logging.info(f"  Response Type: {type(response_data).__name__}")
         return response_data
 
     else:
-        logging.error(f"  Error Response Body: {response.text}")
+        logging.debug(f"Error Response Body: {response.text}")
         raise DataDogRequestError(
             payload=payload_or_params,
             status_code=response.status_code,
             response_text=response.text,
             response_headers=response.headers,
         )
+
+
+def fetch_openapi_spec(
+    site_api_url: Optional[str] = None, version: str = "both"
+) -> Optional[Dict[str, Any]]:
+    """Fetch and cache the Datadog OpenAPI specification.
+
+    Args:
+        site_api_url: Base URL for Datadog API (not used, kept for compatibility)
+        version: Which version to fetch ('v1', 'v2', or 'both')
+
+    Returns:
+        OpenAPI spec as dictionary (combined if 'both'), or None if fetch fails
+    """
+    global _openapi_spec_cache
+
+    # Use version as cache key
+    cache_key = f"openapi_{version}"
+
+    # Check cache first
+    if cache_key in _openapi_spec_cache:
+        return _openapi_spec_cache[cache_key]
+
+    try:
+        import yaml
+
+        # GitHub raw URLs for Datadog's official OpenAPI specs
+        spec_urls = {
+            "v1": "https://raw.githubusercontent.com/DataDog/datadog-api-client-python/master/.generator/schemas/v1/openapi.yaml",
+            "v2": "https://raw.githubusercontent.com/DataDog/datadog-api-client-python/master/.generator/schemas/v2/openapi.yaml",
+        }
+
+        combined_spec: Dict[str, Any] = {
+            "openapi": "3.0.0",
+            "paths": {},
+            "components": {},
+        }
+
+        versions_to_fetch = []
+        if version == "both":
+            versions_to_fetch = ["v1", "v2"]
+        elif version in spec_urls:
+            versions_to_fetch = [version]
+        else:
+            logging.error(f"Invalid version: {version}")
+            return None
+
+        for ver in versions_to_fetch:
+            try:
+                logging.debug(f"Fetching Datadog OpenAPI spec for {ver}...")
+                response = requests.get(spec_urls[ver], timeout=30)
+                if response.status_code == 200:
+                    # Parse YAML to dict
+                    spec = yaml.safe_load(response.text)
+
+                    if version == "both":
+                        # Merge specs
+                        if "paths" in spec:
+                            # Prefix v1 paths with /api/v1 and v2 with /api/v2
+                            for path, methods in spec.get("paths", {}).items():
+                                prefixed_path = (
+                                    f"/api/{ver}{path}"
+                                    if not path.startswith("/api/")
+                                    else path
+                                )
+                                paths_dict = combined_spec.get("paths", {})
+                                if isinstance(paths_dict, dict):
+                                    paths_dict[prefixed_path] = methods
+
+                        # Merge components
+                        if "components" in spec:
+                            for comp_type, components in spec.get(
+                                "components", {}
+                            ).items():
+                                components_dict = combined_spec.get("components", {})
+                                if isinstance(components_dict, dict):
+                                    if comp_type not in components_dict:
+                                        components_dict[comp_type] = {}
+                                    components_dict[comp_type].update(components)
+                    else:
+                        combined_spec = spec
+
+                    logging.info(f"Successfully fetched OpenAPI spec for {ver}")
+                else:
+                    logging.warning(
+                        f"Failed to fetch spec for {ver}: HTTP {response.status_code}"
+                    )
+            except Exception as e:
+                logging.error(f"Failed to fetch spec for {ver}: {e}")
+                if version != "both":
+                    return None
+
+        if combined_spec["paths"]:
+            _openapi_spec_cache[cache_key] = combined_spec
+            logging.info(
+                f"Cached OpenAPI spec with {len(combined_spec['paths'])} endpoints"
+            )
+            return combined_spec
+        else:
+            logging.warning("No endpoints found in OpenAPI spec")
+            return None
+
+    except Exception as e:
+        logging.error(f"Error fetching OpenAPI spec: {e}")
+        return None
+
+
+def get_endpoint_requirements(
+    spec: Dict[str, Any], endpoint: str, method: str
+) -> Optional[Dict[str, Any]]:
+    """Extract parameter requirements for a specific endpoint from OpenAPI spec.
+
+    Args:
+        spec: OpenAPI specification
+        endpoint: API endpoint path
+        method: HTTP method
+
+    Returns:
+        Dictionary with parameter requirements, or None if not found
+    """
+    if not spec or "paths" not in spec:
+        return None
+
+    # Normalize endpoint path
+    endpoint = endpoint.strip("/")
+    if not endpoint.startswith("/"):
+        endpoint = "/" + endpoint
+
+    # Find the endpoint in the spec
+    paths = spec.get("paths", {})
+    if endpoint not in paths:
+        # Try to find a matching pattern (e.g., /api/v2/logs/events/search)
+        for path_pattern in paths.keys():
+            if (
+                path_pattern == endpoint
+                or path_pattern.replace("{", "").replace("}", "") in endpoint
+            ):
+                endpoint = path_pattern
+                break
+        else:
+            return None
+
+    # Get method requirements
+    endpoint_spec = paths.get(endpoint, {})
+    method_spec = endpoint_spec.get(method.lower(), {})
+
+    if not method_spec:
+        return None
+
+    requirements = {
+        "description": method_spec.get("description", ""),
+        "parameters": [],
+        "requestBody": None,
+    }
+
+    # Extract parameters
+    for param in method_spec.get("parameters", []):
+        param_info = {
+            "name": param.get("name"),
+            "in": param.get("in"),  # query, path, header
+            "required": param.get("required", False),
+            "description": param.get("description", ""),
+            "schema": param.get("schema", {}),
+        }
+        requirements["parameters"].append(param_info)
+
+    # Extract request body schema
+    if "requestBody" in method_spec:
+        body = method_spec["requestBody"]
+        content = body.get("content", {})
+        json_content = content.get("application/json", {})
+        requirements["requestBody"] = {
+            "required": body.get("required", False),
+            "schema": json_content.get("schema", {}),
+            "description": body.get("description", ""),
+        }
+
+    return requirements
+
+
+def convert_relative_time(time_str: str) -> Tuple[str, str]:
+    """Convert relative time strings to RFC3339 format.
+
+    Args:
+        time_str: Time string (e.g., '-24h', 'now', '-7d', '2024-01-01T00:00:00Z')
+
+    Returns:
+        Tuple of (converted_time, format_type) where format_type is 'relative', 'rfc3339', or 'unix'
+    """
+    # Check if already in RFC3339 format
+    try:
+        # Try parsing as RFC3339
+        if "T" in time_str and (
+            time_str.endswith("Z") or "+" in time_str or "-" in time_str[-6:]
+        ):
+            datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+            return time_str, "rfc3339"
+    except (ValueError, AttributeError):
+        pass
+
+    # Check if Unix timestamp
+    try:
+        timestamp = float(time_str)
+        if 1000000000 < timestamp < 2000000000:  # Reasonable Unix timestamp range
+            return time_str, "unix"
+    except (ValueError, TypeError):
+        pass
+
+    # Check for relative time
+    match = RELATIVE_TIME_PATTERN.match(time_str.strip())
+    if not match:
+        # Return as-is if not recognized
+        return time_str, "unknown"
+
+    now = datetime.now(timezone.utc)
+
+    if time_str.lower() == "now":
+        return now.isoformat().replace("+00:00", "Z"), "relative"
+
+    # Parse relative time
+    groups = match.groups()
+    if groups[0] is None:
+        return time_str, "unknown"
+
+    amount = int(groups[0])
+    unit = groups[1].lower()
+
+    # Convert to timedelta
+    if unit == "s":
+        delta = timedelta(seconds=amount)
+    elif unit == "min":
+        delta = timedelta(minutes=amount)
+    elif unit == "m":
+        delta = timedelta(minutes=amount)  # m = minutes
+    elif unit == "h":
+        delta = timedelta(hours=amount)
+    elif unit == "d":
+        delta = timedelta(days=amount)
+    elif unit == "w":
+        delta = timedelta(weeks=amount)
+    elif unit == "mo":
+        delta = timedelta(days=amount * 30)  # mo = months (approximate)
+    elif unit == "y":
+        delta = timedelta(days=amount * 365)  # Approximate
+    else:
+        return time_str, "unknown"
+
+    # Apply delta (subtract if negative relative time)
+    if time_str.startswith("-"):
+        result_time = now - delta
+    else:
+        result_time = now + delta
+
+    return result_time.isoformat().replace("+00:00", "Z"), "relative"
+
+
+def preprocess_time_fields(payload: Dict[str, Any], endpoint: str) -> Dict[str, Any]:
+    """Preprocess time fields in payload, converting relative times to appropriate format.
+
+    Args:
+        payload: Request payload
+        endpoint: API endpoint
+
+    Returns:
+        Modified payload with converted time fields
+    """
+    # Deep copy to avoid modifying original
+    import copy
+
+    processed = copy.deepcopy(payload)
+
+    # Common time field paths to check
+    time_fields = [
+        ["filter", "from"],
+        ["filter", "to"],
+        ["from"],
+        ["to"],
+        ["start"],
+        ["end"],
+        ["start_time"],
+        ["end_time"],
+    ]
+
+    def get_nested(d, path):
+        """Get nested dictionary value."""
+        for key in path:
+            if isinstance(d, dict) and key in d:
+                d = d[key]
+            else:
+                return None
+        return d
+
+    def set_nested(d, path, value):
+        """Set nested dictionary value."""
+        for key in path[:-1]:
+            if key not in d:
+                d[key] = {}
+            d = d[key]
+        d[path[-1]] = value
+
+    conversions = []
+
+    for field_path in time_fields:
+        value = get_nested(processed, field_path)
+        if value and isinstance(value, str):
+            converted, format_type = convert_relative_time(value)
+            if format_type == "relative":
+                set_nested(processed, field_path, converted)
+                conversions.append(
+                    f"{'.'.join(field_path)}: '{value}' -> '{converted}'"
+                )
+
+    if conversions:
+        logging.info(f"Converted relative time fields: {', '.join(conversions)}")
+
+    return processed
+
+
+def enhance_error_message(
+    error: DataDogRequestError, endpoint: str, method: str, site_api_url: str
+) -> str:
+    """Enhance error message with OpenAPI spec details and format examples.
+
+    Args:
+        error: Original DataDog request error
+        endpoint: API endpoint
+        method: HTTP method
+        site_api_url: Base API URL
+
+    Returns:
+        Enhanced error message
+    """
+    base_msg = f"HTTP error: {error.status_code} - {error.response_text}"
+
+    # For 400 errors, try to provide more context
+    if error.status_code == 400:
+        enhanced_parts = [base_msg]
+
+        # Try to parse error details
+        try:
+            error_body = json.loads(error.response_text)
+            if "errors" in error_body:
+                enhanced_parts.append(f"\nErrors: {error_body['errors']}")
+
+                # Check for specific field validation errors
+                for err in error_body.get("errors", []):
+                    if "input_validation_error" in str(err):
+                        enhanced_parts.append("\n⚠️ Input validation error detected.")
+
+                        # Add time format help
+                        if any(
+                            field in str(err).lower()
+                            for field in ["from", "to", "time", "date"]
+                        ):
+                            enhanced_parts.append(
+                                "\nTime format requirements:\n"
+                                "  - v1 API: Unix timestamps (e.g., 1704067200)\n"
+                                "  - v2 API: RFC3339 format (e.g., '2024-01-01T00:00:00Z')\n"
+                                "  - NOT supported: Relative times like '-24h', 'now', '-7d'"
+                            )
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+        # Try to fetch OpenAPI spec for more details
+        spec = fetch_openapi_spec(version="both")
+        if spec:
+            requirements = get_endpoint_requirements(spec, endpoint, method)
+            if requirements:
+                enhanced_parts.append(f"\nEndpoint: {method} {endpoint}")
+                if requirements["description"]:
+                    enhanced_parts.append(f"Description: {requirements['description']}")
+
+                # Add parameter requirements
+                if requirements["parameters"]:
+                    enhanced_parts.append("\nRequired parameters:")
+                    for param in requirements["parameters"]:
+                        if param["required"]:
+                            enhanced_parts.append(
+                                f"  - {param['name']} ({param['in']}): {param['description']}"
+                            )
+
+                # Add request body schema hints
+                if (
+                    requirements["requestBody"]
+                    and requirements["requestBody"]["required"]
+                ):
+                    enhanced_parts.append("\nRequest body is required")
+                    if requirements["requestBody"]["description"]:
+                        enhanced_parts.append(
+                            f"Body: {requirements['requestBody']['description']}"
+                        )
+
+        # Add example for common endpoints
+        if "/logs/events/search" in endpoint:
+            enhanced_parts.append(
+                "\nExample request body for logs search:\n"
+                "```json\n"
+                "{\n"
+                '  "filter": {\n'
+                '    "from": "2024-01-01T00:00:00Z",\n'
+                '    "to": "2024-01-02T00:00:00Z",\n'
+                '    "query": "*"\n'
+                "  },\n"
+                '  "sort": "-timestamp",\n'
+                '  "page": {"limit": 50}\n'
+                "}\n"
+                "```"
+            )
+
+        return "\n".join(enhanced_parts)
+
+    return base_msg

holmes/plugins/toolsets/datadog/datadog_logs_instructions.jinja2

@@ -14,26 +14,36 @@ Before running logs queries:
 
 ### CRITICAL: Pod Name Resolution Workflow
 
-**When user provides an exact pod name** (e.g., `my-workload-5f9d8b7c4d-x2km9`):
-- FIRST query Datadog directly with that pod name using appropriate tags
+**IMPORTANT WILDCARD USAGE:**
+- **ALWAYS use wildcards** when searching for pods unless you have the COMPLETE pod name with all suffixes
+- Kubernetes pod names include deployment hash + replica ID (e.g., `nginx-ingress-7b9899-x2km9`, `frontend-5f4d3b2a1-abc123`)
+- When user says "nginx pod" or "frontend pod", search for `nginx-*` or `frontend-*` NOT just `nginx` or `frontend`
+- Datadog supports wildcards: `*` matches any characters (e.g., `nginx-*`, `*ingress*`, `*-x2km9`)
+- For partial matches, use wildcards on both sides: `*keyword*` to find logs from any pod containing "keyword"
+
+**When user provides what looks like a complete pod name** (e.g., `my-workload-5f9d8b7c4d-x2km9`):
+- Query Datadog directly with that exact pod name
 - Do NOT try to verify if the pod exists in Kubernetes first
 - This allows querying historical pods that have been deleted/replaced
 
-**When user provides a generic workload name** (e.g., "my-workload", "nginx", "telemetry-processor"):
-- First use `kubectl_find_resource` to find actual pod names
-- Example: `kubectl_find_resource` with "my-workload" → finds pods like "my-workload-8f8cdfxyz-c7zdr"
-- Then use those specific pod names in Datadog queries
-- Alternative: Use deployment-level tags when appropriate
+**When user provides a simple/generic name** (e.g., "nginx", "redis", "payment-service", "auth"):
+- **DEFAULT ACTION: Use wildcards** - Query with `pod-name-*` pattern
+- For historical queries (yesterday, last week): ALWAYS use wildcards directly in Datadog
+- For current issues: Optionally use `kubectl_find_resource` to find exact pod names, but wildcards often work better
+- Examples:
+  - User says "nginx pod" → Query Datadog with `nginx-*`
+  - User says "redis instance" → Query Datadog with `redis-*`
+  - User says "payment service" → Query Datadog with `payment-*`
 
-**Why this matters:**
+**Why wildcards are critical:**
 - Pod names in Datadog are the actual Kubernetes pod names (with random suffixes)
-- Historical pods that no longer exist in the cluster can still have logs in Datadog
-- Deployment/service names alone are NOT pod names (they need the suffix)
+- Users typically refer to pods by their deployment/service name without suffixes
+- Without wildcards, queries for "nginx" will find NOTHING when actual pods are named "nginx-7b9899-x2km9"
+- Historical pods that no longer exist can only be found via Datadog with proper wildcard usage
 
 ### Time Parameters
 - Use RFC3339 format: `2023-03-01T10:30:00Z`
 - Or relative seconds: `-3600` for 1 hour ago
-- Defaults to 1 hour window if not specified
 
 ### Common Investigation Patterns
 
@@ -41,3 +51,37 @@ Before running logs queries:
 1. User asks: "Show logs for my-workload"
 2. Use `kubectl_find_resource` → find pod "my-workload-abc123-xyz"
 3. Query Datadog for pod "my-workload-abc123-xyz" logs
+
+
+### Search Query Guidelines
+
+1. Avoid using @timestamp Attribute in the search queries (e.g  for example  @timestamp:[2025-12-10T01:00:00.000Z TO 2025-12-10T04:00:00.000Z)
+   Rely on the fetch_datadog_logs function start_datetime and end_datetime parameters for that.
+2. Datadog default TAGS for kubernetes are *kube_namespace* and *pod_name*, if a user specificy custom TAGS used in his environment please use them in your search queries.
+3. If you see a useful TAG in your Old fetch_datadog_logs query use it for further queries.
+
+### CRITICAL: Cursor Usage Rules
+**NEVER parallelize cursor-based calls or reuse cursor values!**
+
+Cursors are stateful pointers - each one is single-use and represents a unique position in the data stream.
+
+**WRONG (causes duplicate data):**
+```
+Batch 1 → cursor_A
+Then call Batch 2, 3, 4 ALL with cursor_A in parallel ❌
+Result: Duplicate data, incomplete results
+```
+
+**CORRECT (sequential pagination):**
+```
+Batch 1 → cursor_A
+Wait for response → use cursor_A for Batch 2 → cursor_B  
+Wait for response → use cursor_B for Batch 3 → cursor_C
+Result: Complete unique data ✅
+```
+
+**Key Rules:**
+- Each response provides a NEW cursor for the NEXT request
+- NEVER reuse the same cursor value multiple times
+- NEVER make parallel calls with the same cursor
+- Always wait for response before using the returned cursor