holmesgpt 0.13.2__py3-none-any.whl → 0.18.4__py3-none-any.whl

holmes/utils/default_toolset_installation_guide.jinja2

@@ -1,21 +1,4 @@
-{% if enabled %}
-{% if is_default %}
-This integration is enabled by default.
-
-If you would like to disable this toolset (not recommended), you need to update the `generated_values.yaml` configuration.
-{% else %}
-To disable this integration, you need to update the `generated_values.yaml` configuration.
-{% endif %}
-
-```yaml
-holmes:
-  toolsets:
-    {{toolset_name}}:
-        enabled: false
-```
-
-{% else %}
-To enable this integration, update the Helm values for Robusta (`generated_values.yaml`):
+To enable/disable this integration, update the Helm values for Robusta (`generated_values.yaml`):
 
 ```yaml
 holmes:
@@ -34,11 +17,7 @@ holmes:
         {{ example_config | indent(8) }}
       {% endif %}
 ```
-
-{% endif %}
-
 And deploy the updated configuration using Helm:
-
 ```bash
 helm upgrade robusta robusta/robusta --values=generated_values.yaml --set clusterName=<YOUR_CLUSTER_NAME>
 ```

holmes/utils/env.py

@@ -6,6 +6,13 @@ from typing import Any, Optional
 from pydantic import SecretStr
 
 
+def environ_get_safe_int(env_var: str, default: str = "0") -> int:
+    try:
+        return max(int(os.environ.get(env_var, default)), 0)
+    except ValueError:
+        return int(default)
+
+
 def get_env_replacement(value: str) -> Optional[str]:
     env_patterns = re.findall(r"{{\s*env\.([^}]*)\s*}}", value)
 

holmes/utils/file_utils.py

@@ -1,6 +1,7 @@
 import json
-import os
 import logging
+import os
+
 import yaml  # type: ignore
 
 

holmes/utils/global_instructions.py

@@ -1,20 +1,69 @@
-from typing import List, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from pydantic import BaseModel
 
+from holmes.plugins.runbooks import RunbookCatalog
+
+if TYPE_CHECKING:
+    from holmes.core.resource_instruction import ResourceInstructions
+
 
 class Instructions(BaseModel):
     instructions: List[str] = []
 
 
-def add_global_instructions_to_user_prompt(
-    user_prompt: str, global_instructions: Optional[Instructions]
+def _format_instructions_block(
+    items: List[str], header: str = "My instructions to check:"
 ) -> str:
-    if (
-        global_instructions
-        and global_instructions.instructions
-        and len(global_instructions.instructions[0]) > 0
-    ):
-        instructions = "\n\n".join(global_instructions.instructions)
-        user_prompt += f"\n\nGlobal Instructions (use if relevant): {instructions}\n"
-    return user_prompt
+    lines = [f"* {s}" for s in items if isinstance(s, str) and s.strip()]
+    if not lines:
+        return ""
+    bullets = "\n".join(lines) + "\n"
+    return f"{header}\n{bullets}"
+
+
+def _format_resource_instructions(
+    resource_instructions: Optional["ResourceInstructions"],
+) -> List[str]:  # type: ignore
+    items = []
+    if resource_instructions is not None:
+        if getattr(resource_instructions, "instructions", None):
+            items.extend(resource_instructions.instructions)
+        if getattr(resource_instructions, "documents", None):
+            for document in resource_instructions.documents:
+                items.append(f"fetch information from this URL: {document.url}")
+    return items
+
+
+def generate_runbooks_args(
+    runbook_catalog: Optional[RunbookCatalog],
+    global_instructions: Optional[Instructions] = None,
+    issue_instructions: Optional[List[str]] = None,
+    resource_instructions: Optional["ResourceInstructions"] = None,  # type: ignore
+) -> Dict[str, str]:
+    catalog_str = runbook_catalog.to_prompt_string() if runbook_catalog else ""
+
+    combined_instructions = []
+    if issue_instructions:
+        combined_instructions.extend(issue_instructions)
+    combined_instructions.extend(_format_resource_instructions(resource_instructions))
+    issue_block = (
+        _format_instructions_block(combined_instructions)
+        if combined_instructions
+        else ""
+    )
+
+    gi_list = getattr(global_instructions, "instructions", None) or []
+    global_block = (
+        _format_instructions_block(
+            [s for s in gi_list if isinstance(s, str)], header=""
+        )
+        if gi_list
+        else ""
+    )
+
+    return {
+        "runbook_catalog": catalog_str,
+        "custom_instructions": issue_block,
+        "global_instructions": global_block,
+    }

holmes/utils/holmes_status.py

@@ -1,8 +1,10 @@
-from holmes.core.supabase_dal import SupabaseDal
-from holmes.config import Config
-from holmes import get_version  # type: ignore
+import json
 import logging
 
+from holmes import get_version  # type: ignore
+from holmes.config import Config
+from holmes.core.supabase_dal import SupabaseDal
+
 
 def update_holmes_status_in_db(dal: SupabaseDal, config: Config):
     logging.info("Updating status of holmes")
@@ -16,7 +18,7 @@ def update_holmes_status_in_db(dal: SupabaseDal, config: Config):
     dal.upsert_holmes_status(
         {
             "cluster_id": config.cluster_name,
-            "model": config.get_models_list(),
+            "model": json.dumps(config.get_models_list()),
             "version": get_version(),
         }
     )

holmes/utils/holmes_sync_toolsets.py

@@ -66,8 +66,6 @@ def render_default_installation_instructions_for_toolset(toolset: Toolset) -> st
     context: dict[str, Any] = {
         "env_vars": env_vars if env_vars else [],
         "toolset_name": toolset.name,
-        "is_default": toolset.is_default,
-        "enabled": toolset.enabled,
     }
 
     example_config = toolset.get_example_config()

holmes/utils/krr_utils.py

@@ -0,0 +1,188 @@
+"""Utilities for KRR (Kubernetes Resource Recommendations) data processing."""
+
+import logging
+from typing import Any, Dict
+
+
+def parse_cpu(cpu_value: Any) -> float:
+    """Parse Kubernetes CPU value to float (in cores).
+
+    Handles:
+    - Numeric values (0.1, 1, etc.) - already in cores
+    - String values with 'm' suffix (100m = 0.1 cores)
+    - String numeric values ("0.5")
+
+    Args:
+        cpu_value: CPU value to parse (can be int, float, str, or None)
+
+    Returns:
+        CPU value in cores as float, or 0.0 if invalid
+    """
+    if cpu_value is None or cpu_value == "" or cpu_value == "?":
+        return 0.0
+    try:
+        if isinstance(cpu_value, (int, float)):
+            return float(cpu_value)
+
+        cpu_str = str(cpu_value).strip()
+        if cpu_str.endswith("m"):
+            return float(cpu_str[:-1]) / 1000.0
+        return float(cpu_str)
+    except (ValueError, AttributeError, TypeError):
+        return 0.0
+
+
+def parse_memory(memory_value: Any) -> float:
+    """Parse Kubernetes memory value to float (in bytes).
+
+    Handles:
+    - Numeric values (already in bytes)
+    - String values with units (100Mi, 1Gi, etc.)
+    - String numeric values ("1048576")
+
+    Args:
+        memory_value: Memory value to parse (can be int, float, str, or None)
+
+    Returns:
+        Memory value in bytes as float, or 0.0 if invalid
+    """
+    if memory_value is None or memory_value == "" or memory_value == "?":
+        return 0.0
+    try:
+        if isinstance(memory_value, (int, float)):
+            return float(memory_value)
+
+        memory_str = str(memory_value).strip()
+        units = {
+            "Ki": 1024,
+            "Mi": 1024**2,
+            "Gi": 1024**3,
+            "Ti": 1024**4,
+            "K": 1000,
+            "M": 1000**2,
+            "G": 1000**3,
+            "T": 1000**4,
+        }
+        for unit, multiplier in units.items():
+            if memory_str.endswith(unit):
+                return float(memory_str[: -len(unit)]) * multiplier
+        return float(memory_str)
+    except (ValueError, AttributeError, TypeError):
+        return 0.0
+
+
+# Helper to get numeric value from allocated/recommended, handling "?" strings
+def get_value(data: Dict, field: str, subfield: str) -> Any:
+    if not data:
+        return 0.0
+    val = data.get(field, {}).get(subfield)
+    if val is None or val == "?":
+        return 0.0
+    return val
+
+
+def calculate_krr_savings(result: Dict, sort_by: str) -> float:
+    """Calculate potential savings from KRR recommendation data.
+
+    The KRR data structure has a 'content' field that contains a list of resource
+    recommendations. Each item in the list represents either CPU or memory, with:
+    - resource: "cpu" or "memory"
+    - allocated: {request: value, limit: value} - current allocation
+    - recommended: {request: value, limit: value} - recommended allocation
+
+    Args:
+        result: KRR scan result dictionary with 'content' field
+        sort_by: Sorting criteria, one of:
+            - "cpu_total": Total CPU savings (requests + limits)
+            - "memory_total": Total memory savings (requests + limits)
+            - "cpu_requests": CPU requests savings only
+            - "memory_requests": Memory requests savings only
+            - "cpu_limits": CPU limits savings only
+            - "memory_limits": Memory limits savings only
+
+    Returns:
+        Calculated savings as a float (>= 0.0). Returns 0.0 for invalid data
+        or when recommended values are higher than allocated.
+    """
+    try:
+        content_list = result.get("content", [])
+        if not content_list or not isinstance(content_list, list):
+            return 0.0
+
+        cpu_data = None
+        memory_data = None
+        for item in content_list:
+            if item.get("resource") == "cpu":
+                cpu_data = item
+            elif item.get("resource") == "memory":
+                memory_data = item
+
+        if not cpu_data and not memory_data:
+            return 0.0
+
+        savings = 0.0
+
+        if sort_by == "cpu_total" and cpu_data:
+            cpu_req_allocated = parse_cpu(get_value(cpu_data, "allocated", "request"))
+            cpu_req_recommended = parse_cpu(
+                get_value(cpu_data, "recommended", "request")
+            )
+            cpu_lim_allocated = parse_cpu(get_value(cpu_data, "allocated", "limit"))
+            cpu_lim_recommended = parse_cpu(get_value(cpu_data, "recommended", "limit"))
+
+            savings = (cpu_req_allocated - cpu_req_recommended) + (
+                cpu_lim_allocated - cpu_lim_recommended
+            )
+
+        elif sort_by == "memory_total" and memory_data:
+            mem_req_allocated = parse_memory(
+                get_value(memory_data, "allocated", "request")
+            )
+            mem_req_recommended = parse_memory(
+                get_value(memory_data, "recommended", "request")
+            )
+            mem_lim_allocated = parse_memory(
+                get_value(memory_data, "allocated", "limit")
+            )
+            mem_lim_recommended = parse_memory(
+                get_value(memory_data, "recommended", "limit")
+            )
+
+            savings = (mem_req_allocated - mem_req_recommended) + (
+                mem_lim_allocated - mem_lim_recommended
+            )
+
+        elif sort_by == "cpu_requests" and cpu_data:
+            cpu_req_allocated = parse_cpu(get_value(cpu_data, "allocated", "request"))
+            cpu_req_recommended = parse_cpu(
+                get_value(cpu_data, "recommended", "request")
+            )
+            savings = cpu_req_allocated - cpu_req_recommended
+
+        elif sort_by == "memory_requests" and memory_data:
+            mem_req_allocated = parse_memory(
+                get_value(memory_data, "allocated", "request")
+            )
+            mem_req_recommended = parse_memory(
+                get_value(memory_data, "recommended", "request")
+            )
+            savings = mem_req_allocated - mem_req_recommended
+
+        elif sort_by == "cpu_limits" and cpu_data:
+            cpu_lim_allocated = parse_cpu(get_value(cpu_data, "allocated", "limit"))
+            cpu_lim_recommended = parse_cpu(get_value(cpu_data, "recommended", "limit"))
+            savings = cpu_lim_allocated - cpu_lim_recommended
+
+        elif sort_by == "memory_limits" and memory_data:
+            mem_lim_allocated = parse_memory(
+                get_value(memory_data, "allocated", "limit")
+            )
+            mem_lim_recommended = parse_memory(
+                get_value(memory_data, "recommended", "limit")
+            )
+            savings = mem_lim_allocated - mem_lim_recommended
+
+        return savings
+    except Exception as e:
+        logging.debug(f"Error calculating savings for result: {e}")
+        return 0.0

holmes/utils/log.py

@@ -0,0 +1,15 @@
+"""Logging utilities for Holmes."""
+
+import logging
+from typing import Any
+
+
+class EndpointFilter(logging.Filter):
+    """Filter out log records for specific endpoint paths."""
+
+    def __init__(self, path: str, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self._path = path
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.getMessage().find(self._path) == -1

holmes/utils/markdown_utils.py

@@ -1,9 +1,8 @@
 # based on https://github.com/kostyachum/python-markdown-plain-text/blob/main/markdown_plain_text/extention.py
 # MIT licensed
-from markdown import Extension, Markdown  # type: ignore
+from xml.etree.ElementTree import Comment, ElementTree, ProcessingInstruction
 
-from xml.etree.ElementTree import ProcessingInstruction
-from xml.etree.ElementTree import Comment, ElementTree
+from markdown import Extension, Markdown  # type: ignore
 
 
 def _serialize_plain_text(write, elem):

holmes/utils/memory_limit.py

@@ -0,0 +1,58 @@
+"""
+Memory limit utilities for tool subprocess execution.
+"""
+
+import logging
+
+from holmes.common.env_vars import TOOL_MEMORY_LIMIT_MB
+
+logger = logging.getLogger(__name__)
+
+
+def get_ulimit_prefix() -> str:
+    """
+    Get the ulimit command prefix for memory protection.
+
+    Returns a shell command prefix that sets virtual memory limit.
+    The '|| true' ensures we continue even if ulimit is not supported.
+    """
+    memory_limit_kb = TOOL_MEMORY_LIMIT_MB * 1024
+    return f"ulimit -v {memory_limit_kb} || true; "
+
+
+def check_oom_and_append_hint(output: str, return_code: int) -> str:
+    """
+    Check if a command was OOM killed and append a helpful hint.
+
+    Args:
+        output: The command output
+        return_code: The command's return code
+
+    Returns:
+        Output with OOM hint appended if OOM was detected
+    """
+    # Common OOM indicators:
+    # - Return code 137 (128 + 9 = SIGKILL, commonly OOM)
+    # - Return code -9 (SIGKILL on some systems)
+    # - "Killed" in output (Linux OOM killer message)
+    # - "MemoryError" (Python)
+    # - "Cannot allocate memory" (various tools)
+    is_oom = (
+        return_code in (137, -9)
+        or "Killed" in output
+        or "MemoryError" in output
+        or "Cannot allocate memory" in output
+        or "bad_alloc" in output
+    )
+
+    if is_oom:
+        hint = (
+            f"\n\n[OOM] Command was killed due to memory limits (current limit: {TOOL_MEMORY_LIMIT_MB} MB). "
+            f"Try querying the data differently to reduce memory usage - add filters to narrow the results, "
+            f"use smaller time ranges, or try alternative tools that may be more memory-efficient. "
+            f"If you cannot succeed with a modified query, you may recommend the user increase the limit "
+            f"by setting the TOOL_MEMORY_LIMIT_MB environment variable (Tool memory limit, MB)."
+        )
+        return output + hint
+
+    return output

holmes/utils/sentry_helper.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+import sentry_sdk
+
+from holmes.core.models import ToolCallResult, TruncationMetadata
+
+
+def capture_tool_truncations(truncations: list[TruncationMetadata]):
+    for truncation in truncations:
+        _capture_tool_truncation(truncation)
+
+
+def _capture_tool_truncation(truncation: TruncationMetadata):
+    sentry_sdk.capture_message(
+        f"Tool {truncation.tool_name} was truncated",
+        level="warning",
+        tags={
+            "tool_name": truncation.tool_name,
+            "tool_original_token_count": truncation.original_token_count,
+            "tool_new_token_count": truncation.end_index,
+        },
+    )
+
+
+def capture_toolcall_contains_too_many_tokens(
+    tool_call_result: ToolCallResult, token_count: int, max_allowed_token_count: int
+):
+    sentry_sdk.capture_message(
+        f"Tool call {tool_call_result.tool_name} contains too many tokens",
+        level="warning",
+        tags={
+            "tool_name": tool_call_result.tool_name,
+            "tool_original_token_count": token_count,
+            "tool_max_allowed_token_count": max_allowed_token_count,
+            "tool_description": tool_call_result.description,
+        },
+    )
+
+
+def capture_structured_output_incorrect_tool_call():
+    sentry_sdk.capture_message(
+        "Structured output incorrect tool call",
+        level="warning",
+    )
+
+
+def capture_sections_none(content: Optional[str]):
+    # Limit display length to avoid sending huge payloads to Sentry
+    _MAX_DISPLAY_LENGTH = 1500
+    display_content = ""
+    if content:
+        if len(content) > _MAX_DISPLAY_LENGTH * 2:
+            # Show first and last portions of content
+            display_content = f"{content[:_MAX_DISPLAY_LENGTH]}...\n\n...{content[-_MAX_DISPLAY_LENGTH:]}"
+        else:
+            display_content = content
+
+    with sentry_sdk.push_scope() as scope:
+        scope.set_extra("content", display_content)
+        scope.set_extra("content_length", len(content) if content else 0)
+        sentry_sdk.capture_message(
+            "Holmes answer couldn't be parsed into sections",
+            level="warning",
+        )

holmes/utils/stream.py

@@ -1,10 +1,17 @@
 import json
+import logging
 from enum import Enum
-from typing import Generator, Optional, List
+from functools import partial
+from typing import Generator, List, Optional, Union
+
 import litellm
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.types.utils import ModelResponse, TextCompletionResponse
 from pydantic import BaseModel, Field
+
 from holmes.core.investigation_structured_output import process_response_into_sections
-from functools import partial
+from holmes.core.llm import TokenCountMetadata, get_llm_usage
+from holmes.utils import sentry_helper
 
 
 class StreamEvents(str, Enum):
@@ -13,6 +20,9 @@ class StreamEvents(str, Enum):
     TOOL_RESULT = "tool_calling_result"
     ERROR = "error"
     AI_MESSAGE = "ai_message"
+    APPROVAL_REQUIRED = "approval_required"
+    TOKEN_COUNT = "token_count"
+    CONVERSATION_HISTORY_COMPACTED = "conversation_history_compacted"
 
 
 class StreamMessage(BaseModel):
@@ -55,12 +65,18 @@ def stream_investigate_formatter(
                     message.data.get("content")
                 )
 
+                if sections is None:
+                    sentry_helper.capture_sections_none(
+                        content=message.data.get("content"),
+                    )
+
                 yield create_sse_message(
                     StreamEvents.ANSWER_END.value,
                     {
                         "sections": sections or {},
                         "analysis": text_response,
                         "instructions": runbooks or [],
+                        "metadata": message.data.get("metadata") or {},
                     },
                 )
             else:
@@ -76,15 +92,60 @@ def stream_chat_formatter(
     try:
         for message in call_stream:
             if message.event == StreamEvents.ANSWER_END:
+                response_data = {
+                    "analysis": message.data.get("content"),
+                    "conversation_history": message.data.get("messages"),
+                    "follow_up_actions": followups,
+                    "metadata": message.data.get("metadata") or {},
+                }
+
+                yield create_sse_message(StreamEvents.ANSWER_END.value, response_data)
+            elif message.event == StreamEvents.APPROVAL_REQUIRED:
+                response_data = {
+                    "analysis": message.data.get("content"),
+                    "conversation_history": message.data.get("messages"),
+                    "follow_up_actions": followups,
+                }
+
+                response_data["requires_approval"] = True
+                response_data["pending_approvals"] = message.data.get(
+                    "pending_approvals", []
+                )
+
                 yield create_sse_message(
-                    StreamEvents.ANSWER_END.value,
-                    {
-                        "analysis": message.data.get("content"),
-                        "conversation_history": message.data.get("messages"),
-                        "follow_up_actions": followups,
-                    },
+                    StreamEvents.APPROVAL_REQUIRED.value, response_data
                 )
             else:
                 yield create_sse_message(message.event.value, message.data)
     except litellm.exceptions.RateLimitError as e:
         yield create_rate_limit_error_message(str(e))
+    except Exception as e:
+        logging.error(e)
+        if "Model is getting throttled" in str(e):  # happens for bedrock
+            yield create_rate_limit_error_message(str(e))
+        else:
+            yield create_sse_error_message(description=str(e), error_code=1, msg=str(e))
+
+
+def add_token_count_to_metadata(
+    tokens: TokenCountMetadata,
+    metadata: dict,
+    max_context_size: int,
+    maximum_output_token: int,
+    full_llm_response: Union[
+        ModelResponse, CustomStreamWrapper, TextCompletionResponse
+    ],
+):
+    metadata["usage"] = get_llm_usage(full_llm_response)
+    metadata["tokens"] = tokens.model_dump()
+    metadata["max_tokens"] = max_context_size
+    metadata["max_output_tokens"] = maximum_output_token
+
+
+def build_stream_event_token_count(metadata: dict) -> StreamMessage:
+    return StreamMessage(
+        event=StreamEvents.TOKEN_COUNT,
+        data={
+            "metadata": metadata,
+        },
+    )

holmes/utils/tags.py

@@ -1,9 +1,10 @@
+import json
 import logging
-from typing import Optional
-from typing_extensions import Dict, List
 import re
-import json
 from copy import deepcopy
+from typing import Optional
+
+from typing_extensions import Dict, List
 
 
 def stringify_tag(tag: Dict[str, str]) -> Optional[str]: