holmesgpt 0.13.2__py3-none-any.whl → 0.16.2a0__py3-none-any.whl

holmes/utils/global_instructions.py

@@ -1,20 +1,85 @@
-from typing import List, Optional
-
+from typing import Optional, List, TYPE_CHECKING
 from pydantic import BaseModel
+from holmes.plugins.prompts import load_and_render_prompt
+from holmes.plugins.runbooks import RunbookCatalog
+
+if TYPE_CHECKING:
+    from holmes.core.resource_instruction import ResourceInstructions
 
 
 class Instructions(BaseModel):
     instructions: List[str] = []
 
 
-def add_global_instructions_to_user_prompt(
-    user_prompt: str, global_instructions: Optional[Instructions]
+def _format_instructions_block(
+    items: List[str], header: str = "My instructions to check:"
+) -> str:
+    lines = [f"* {s}" for s in items if isinstance(s, str) and s.strip()]
+    if not lines:
+        return ""
+    bullets = "\n".join(lines) + "\n"
+    return f"{header}\n{bullets}"
+
+
+def _format_resource_instructions(
+    resource_instructions: Optional["ResourceInstructions"],
+) -> List[str]:  # type: ignore
+    items = []
+    if resource_instructions is not None:
+        if getattr(resource_instructions, "instructions", None):
+            items.extend(resource_instructions.instructions)
+        if getattr(resource_instructions, "documents", None):
+            for document in resource_instructions.documents:
+                items.append(f"fetch information from this URL: {document.url}")
+    return items
+
+
+def add_runbooks_to_user_prompt(
+    user_prompt: str,
+    runbook_catalog: Optional[RunbookCatalog],
+    global_instructions: Optional[Instructions] = None,
+    issue_instructions: Optional[List[str]] = None,
+    resource_instructions: Optional["ResourceInstructions"] = None,  # type: ignore
 ) -> str:
     if (
-        global_instructions
-        and global_instructions.instructions
-        and len(global_instructions.instructions[0]) > 0
+        not runbook_catalog
+        and not issue_instructions
+        and not resource_instructions
+        and not global_instructions
     ):
-        instructions = "\n\n".join(global_instructions.instructions)
-        user_prompt += f"\n\nGlobal Instructions (use if relevant): {instructions}\n"
-    return user_prompt
+        return user_prompt
+
+    catalog_str = runbook_catalog.to_prompt_string() if runbook_catalog else ""
+
+    # Combine and format all instructions
+    combined_instructions = []
+    if issue_instructions:
+        combined_instructions.extend(issue_instructions)
+    combined_instructions.extend(_format_resource_instructions(resource_instructions))
+    issue_block = (
+        _format_instructions_block(combined_instructions)
+        if combined_instructions
+        else ""
+    )
+
+    gi_list = getattr(global_instructions, "instructions", None) or []
+    global_block = (
+        _format_instructions_block(
+            [s for s in gi_list if isinstance(s, str)], header=""
+        )
+        if gi_list
+        else ""
+    )
+
+    rendered = load_and_render_prompt(
+        "builtin://_runbook_instructions.jinja2",
+        context={
+            "runbook_catalog": catalog_str,
+            "custom_instructions": issue_block,
+            "global_instructions": global_block,
+        },
+    )
+
+    if user_prompt and not user_prompt.endswith("\n"):
+        user_prompt += "\n"
+    return f"{user_prompt}\n{rendered}"

holmes/utils/holmes_status.py

@@ -1,3 +1,4 @@
+import json
 from holmes.core.supabase_dal import SupabaseDal
 from holmes.config import Config
 from holmes import get_version  # type: ignore
@@ -16,7 +17,7 @@ def update_holmes_status_in_db(dal: SupabaseDal, config: Config):
     dal.upsert_holmes_status(
         {
             "cluster_id": config.cluster_name,
-            "model": config.get_models_list(),
+            "model": json.dumps(config.get_models_list()),
             "version": get_version(),
         }
     )

holmes/utils/holmes_sync_toolsets.py

@@ -66,8 +66,6 @@ def render_default_installation_instructions_for_toolset(toolset: Toolset) -> st
     context: dict[str, Any] = {
         "env_vars": env_vars if env_vars else [],
         "toolset_name": toolset.name,
-        "is_default": toolset.is_default,
-        "enabled": toolset.enabled,
     }
 
     example_config = toolset.get_example_config()

holmes/utils/krr_utils.py

@@ -0,0 +1,188 @@
+"""Utilities for KRR (Kubernetes Resource Recommendations) data processing."""
+
+import logging
+from typing import Any, Dict
+
+
+def parse_cpu(cpu_value: Any) -> float:
+    """Parse Kubernetes CPU value to float (in cores).
+
+    Handles:
+    - Numeric values (0.1, 1, etc.) - already in cores
+    - String values with 'm' suffix (100m = 0.1 cores)
+    - String numeric values ("0.5")
+
+    Args:
+        cpu_value: CPU value to parse (can be int, float, str, or None)
+
+    Returns:
+        CPU value in cores as float, or 0.0 if invalid
+    """
+    if cpu_value is None or cpu_value == "" or cpu_value == "?":
+        return 0.0
+    try:
+        if isinstance(cpu_value, (int, float)):
+            return float(cpu_value)
+
+        cpu_str = str(cpu_value).strip()
+        if cpu_str.endswith("m"):
+            return float(cpu_str[:-1]) / 1000.0
+        return float(cpu_str)
+    except (ValueError, AttributeError, TypeError):
+        return 0.0
+
+
+def parse_memory(memory_value: Any) -> float:
+    """Parse Kubernetes memory value to float (in bytes).
+
+    Handles:
+    - Numeric values (already in bytes)
+    - String values with units (100Mi, 1Gi, etc.)
+    - String numeric values ("1048576")
+
+    Args:
+        memory_value: Memory value to parse (can be int, float, str, or None)
+
+    Returns:
+        Memory value in bytes as float, or 0.0 if invalid
+    """
+    if memory_value is None or memory_value == "" or memory_value == "?":
+        return 0.0
+    try:
+        if isinstance(memory_value, (int, float)):
+            return float(memory_value)
+
+        memory_str = str(memory_value).strip()
+        units = {
+            "Ki": 1024,
+            "Mi": 1024**2,
+            "Gi": 1024**3,
+            "Ti": 1024**4,
+            "K": 1000,
+            "M": 1000**2,
+            "G": 1000**3,
+            "T": 1000**4,
+        }
+        for unit, multiplier in units.items():
+            if memory_str.endswith(unit):
+                return float(memory_str[: -len(unit)]) * multiplier
+        return float(memory_str)
+    except (ValueError, AttributeError, TypeError):
+        return 0.0
+
+
+# Helper to get numeric value from allocated/recommended, handling "?" strings
+def get_value(data: Dict, field: str, subfield: str) -> Any:
+    if not data:
+        return 0.0
+    val = data.get(field, {}).get(subfield)
+    if val is None or val == "?":
+        return 0.0
+    return val
+
+
+def calculate_krr_savings(result: Dict, sort_by: str) -> float:
+    """Calculate potential savings from KRR recommendation data.
+
+    The KRR data structure has a 'content' field that contains a list of resource
+    recommendations. Each item in the list represents either CPU or memory, with:
+    - resource: "cpu" or "memory"
+    - allocated: {request: value, limit: value} - current allocation
+    - recommended: {request: value, limit: value} - recommended allocation
+
+    Args:
+        result: KRR scan result dictionary with 'content' field
+        sort_by: Sorting criteria, one of:
+            - "cpu_total": Total CPU savings (requests + limits)
+            - "memory_total": Total memory savings (requests + limits)
+            - "cpu_requests": CPU requests savings only
+            - "memory_requests": Memory requests savings only
+            - "cpu_limits": CPU limits savings only
+            - "memory_limits": Memory limits savings only
+
+    Returns:
+        Calculated savings as a float (>= 0.0). Returns 0.0 for invalid data
+        or when recommended values are higher than allocated.
+    """
+    try:
+        content_list = result.get("content", [])
+        if not content_list or not isinstance(content_list, list):
+            return 0.0
+
+        cpu_data = None
+        memory_data = None
+        for item in content_list:
+            if item.get("resource") == "cpu":
+                cpu_data = item
+            elif item.get("resource") == "memory":
+                memory_data = item
+
+        if not cpu_data and not memory_data:
+            return 0.0
+
+        savings = 0.0
+
+        if sort_by == "cpu_total" and cpu_data:
+            cpu_req_allocated = parse_cpu(get_value(cpu_data, "allocated", "request"))
+            cpu_req_recommended = parse_cpu(
+                get_value(cpu_data, "recommended", "request")
+            )
+            cpu_lim_allocated = parse_cpu(get_value(cpu_data, "allocated", "limit"))
+            cpu_lim_recommended = parse_cpu(get_value(cpu_data, "recommended", "limit"))
+
+            savings = (cpu_req_allocated - cpu_req_recommended) + (
+                cpu_lim_allocated - cpu_lim_recommended
+            )
+
+        elif sort_by == "memory_total" and memory_data:
+            mem_req_allocated = parse_memory(
+                get_value(memory_data, "allocated", "request")
+            )
+            mem_req_recommended = parse_memory(
+                get_value(memory_data, "recommended", "request")
+            )
+            mem_lim_allocated = parse_memory(
+                get_value(memory_data, "allocated", "limit")
+            )
+            mem_lim_recommended = parse_memory(
+                get_value(memory_data, "recommended", "limit")
+            )
+
+            savings = (mem_req_allocated - mem_req_recommended) + (
+                mem_lim_allocated - mem_lim_recommended
+            )
+
+        elif sort_by == "cpu_requests" and cpu_data:
+            cpu_req_allocated = parse_cpu(get_value(cpu_data, "allocated", "request"))
+            cpu_req_recommended = parse_cpu(
+                get_value(cpu_data, "recommended", "request")
+            )
+            savings = cpu_req_allocated - cpu_req_recommended
+
+        elif sort_by == "memory_requests" and memory_data:
+            mem_req_allocated = parse_memory(
+                get_value(memory_data, "allocated", "request")
+            )
+            mem_req_recommended = parse_memory(
+                get_value(memory_data, "recommended", "request")
+            )
+            savings = mem_req_allocated - mem_req_recommended
+
+        elif sort_by == "cpu_limits" and cpu_data:
+            cpu_lim_allocated = parse_cpu(get_value(cpu_data, "allocated", "limit"))
+            cpu_lim_recommended = parse_cpu(get_value(cpu_data, "recommended", "limit"))
+            savings = cpu_lim_allocated - cpu_lim_recommended
+
+        elif sort_by == "memory_limits" and memory_data:
+            mem_lim_allocated = parse_memory(
+                get_value(memory_data, "allocated", "limit")
+            )
+            mem_lim_recommended = parse_memory(
+                get_value(memory_data, "recommended", "limit")
+            )
+            savings = mem_lim_allocated - mem_lim_recommended
+
+        return savings
+    except Exception as e:
+        logging.debug(f"Error calculating savings for result: {e}")
+        return 0.0

holmes/utils/sentry_helper.py

@@ -0,0 +1,41 @@
+import sentry_sdk
+from holmes.core.models import ToolCallResult, TruncationMetadata
+
+
+def capture_tool_truncations(truncations: list[TruncationMetadata]):
+    for truncation in truncations:
+        _capture_tool_truncation(truncation)
+
+
+def _capture_tool_truncation(truncation: TruncationMetadata):
+    sentry_sdk.capture_message(
+        f"Tool {truncation.tool_name} was truncated",
+        level="warning",
+        tags={
+            "tool_name": truncation.tool_name,
+            "tool_original_token_count": truncation.original_token_count,
+            "tool_new_token_count": truncation.end_index,
+        },
+    )
+
+
+def capture_toolcall_contains_too_many_tokens(
+    tool_call_result: ToolCallResult, token_count: int, max_allowed_token_count: int
+):
+    sentry_sdk.capture_message(
+        f"Tool call {tool_call_result.tool_name} contains too many tokens",
+        level="warning",
+        tags={
+            "tool_name": tool_call_result.tool_name,
+            "tool_original_token_count": token_count,
+            "tool_max_allowed_token_count": max_allowed_token_count,
+            "tool_description": tool_call_result.description,
+        },
+    )
+
+
+def capture_structured_output_incorrect_tool_call():
+    sentry_sdk.capture_message(
+        "Structured output incorrect tool call",
+        level="warning",
+    )

holmes/utils/stream.py

@@ -1,10 +1,15 @@
 import json
 from enum import Enum
-from typing import Generator, Optional, List
+from typing import Generator, Optional, List, Union
 import litellm
 from pydantic import BaseModel, Field
 from holmes.core.investigation_structured_output import process_response_into_sections
 from functools import partial
+import logging
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.types.utils import ModelResponse, TextCompletionResponse
+
+from holmes.core.llm import TokenCountMetadata, get_llm_usage
 
 
 class StreamEvents(str, Enum):
@@ -13,6 +18,9 @@ class StreamEvents(str, Enum):
     TOOL_RESULT = "tool_calling_result"
     ERROR = "error"
     AI_MESSAGE = "ai_message"
+    APPROVAL_REQUIRED = "approval_required"
+    TOKEN_COUNT = "token_count"
+    CONVERSATION_HISTORY_COMPACTED = "conversation_history_compacted"
 
 
 class StreamMessage(BaseModel):
@@ -61,6 +69,7 @@ def stream_investigate_formatter(
                         "sections": sections or {},
                         "analysis": text_response,
                         "instructions": runbooks or [],
+                        "metadata": message.data.get("metadata") or {},
                     },
                 )
             else:
@@ -76,15 +85,60 @@ def stream_chat_formatter(
     try:
         for message in call_stream:
             if message.event == StreamEvents.ANSWER_END:
+                response_data = {
+                    "analysis": message.data.get("content"),
+                    "conversation_history": message.data.get("messages"),
+                    "follow_up_actions": followups,
+                    "metadata": message.data.get("metadata") or {},
+                }
+
+                yield create_sse_message(StreamEvents.ANSWER_END.value, response_data)
+            elif message.event == StreamEvents.APPROVAL_REQUIRED:
+                response_data = {
+                    "analysis": message.data.get("content"),
+                    "conversation_history": message.data.get("messages"),
+                    "follow_up_actions": followups,
+                }
+
+                response_data["requires_approval"] = True
+                response_data["pending_approvals"] = message.data.get(
+                    "pending_approvals", []
+                )
+
                 yield create_sse_message(
-                    StreamEvents.ANSWER_END.value,
-                    {
-                        "analysis": message.data.get("content"),
-                        "conversation_history": message.data.get("messages"),
-                        "follow_up_actions": followups,
-                    },
+                    StreamEvents.APPROVAL_REQUIRED.value, response_data
                 )
             else:
                 yield create_sse_message(message.event.value, message.data)
     except litellm.exceptions.RateLimitError as e:
         yield create_rate_limit_error_message(str(e))
+    except Exception as e:
+        logging.error(e)
+        if "Model is getting throttled" in str(e):  # happens for bedrock
+            yield create_rate_limit_error_message(str(e))
+        else:
+            yield create_sse_error_message(description=str(e), error_code=1, msg=str(e))
+
+
+def add_token_count_to_metadata(
+    tokens: TokenCountMetadata,
+    metadata: dict,
+    max_context_size: int,
+    maximum_output_token: int,
+    full_llm_response: Union[
+        ModelResponse, CustomStreamWrapper, TextCompletionResponse
+    ],
+):
+    metadata["usage"] = get_llm_usage(full_llm_response)
+    metadata["tokens"] = tokens.model_dump()
+    metadata["max_tokens"] = max_context_size
+    metadata["max_output_tokens"] = maximum_output_token
+
+
+def build_stream_event_token_count(metadata: dict) -> StreamMessage:
+    return StreamMessage(
+        event=StreamEvents.TOKEN_COUNT,
+        data={
+            "metadata": metadata,
+        },
+    )

holmes/version.py

@@ -57,11 +57,41 @@ def get_version() -> str:
         return __version__
 
     # we are running from an unreleased dev version
+    archival_file_path = os.path.join(this_path, ".git_archival.json")
+    if os.path.exists(archival_file_path):
+        try:
+            with open(archival_file_path, "r") as f:
+                archival_data = json.load(f)
+                refs = archival_data.get("refs", "")
+                hash_short = archival_data.get("hash-short", "")
+
+                # Check if Git substitution didn't happen (placeholders are still present)
+                if "$Format:" in refs or "$Format:" in hash_short:
+                    # Placeholders not substituted, skip to next method
+                    pass
+                else:
+                    # Valid archival data found
+                    return f"dev-{refs}-{hash_short}"
+        except Exception:
+            pass
+
+    # Now try git commands for development environments
     try:
+        env = os.environ.copy()
+        # Set ceiling to prevent walking up beyond the project root
+        # We want to allow access to holmes/.git but not beyond holmes
+        project_root = os.path.dirname(this_path)  # holmes
+        env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(
+            project_root
+        )  # holmes's parent
+
         # Get the latest git tag
         tag = (
             subprocess.check_output(
-                ["git", "describe", "--tags"], stderr=subprocess.STDOUT, cwd=this_path
+                ["git", "describe", "--tags"],
+                stderr=subprocess.STDOUT,
+                cwd=this_path,
+                env=env,
             )
             .decode()
             .strip()
@@ -73,6 +103,7 @@ def get_version() -> str:
                 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
                 stderr=subprocess.STDOUT,
                 cwd=this_path,
+                env=env,
             )
             .decode()
             .strip()
@@ -84,6 +115,7 @@ def get_version() -> str:
                 ["git", "status", "--porcelain"],
                 stderr=subprocess.STDOUT,
                 cwd=this_path,
+                env=env,
             )
             .decode()
             .strip()
@@ -95,19 +127,7 @@ def get_version() -> str:
     except Exception:
         pass
 
-    # we are running without git history, but we still might have git archival data (e.g. if we were pip installed)
-    archival_file_path = os.path.join(this_path, ".git_archival.json")
-    if os.path.exists(archival_file_path):
-        try:
-            with open(archival_file_path, "r") as f:
-                archival_data = json.load(f)
-                return f"dev-{archival_data['refs']}-{archival_data['hash-short']}"
-        except Exception:
-            pass
-
-        return "dev-version"
-
-    return "unknown-version"
+    return "dev-unknown"
 
 
 @cache