holmesgpt 0.14.2__py3-none-any.whl → 0.14.3a0__py3-none-any.whl

holmes/__init__.py

@@ -1,5 +1,5 @@
 # This is patched by github actions during release
-__version__ = "0.14.2"
+__version__ = "0.14.3-alpha"
 
 # Re-export version functions from version module for backward compatibility
 from .version import (

holmes/common/env_vars.py

@@ -81,3 +81,9 @@ MAX_GRAPH_POINTS = float(os.environ.get("MAX_GRAPH_POINTS", 100))
 TOOL_MAX_ALLOCATED_CONTEXT_WINDOW_PCT = float(
     os.environ.get("TOOL_MAX_ALLOCATED_CONTEXT_WINDOW_PCT", 15)
 )
+
+MAX_EVIDENCE_DATA_CHARACTERS_BEFORE_TRUNCATION = int(
+    os.environ.get("MAX_EVIDENCE_DATA_CHARACTERS_BEFORE_TRUNCATION", 3000)
+)
+
+DISABLE_PROMETHEUS_TOOLSET = load_bool("DISABLE_PROMETHEUS_TOOLSET", False)

holmes/config.py

@@ -9,11 +9,8 @@ import sentry_sdk
 import yaml  # type: ignore
 from pydantic import BaseModel, ConfigDict, FilePath, PrivateAttr, SecretStr
 
-
+from holmes.common.env_vars import ROBUSTA_CONFIG_PATH
 from holmes.core.llm import DefaultLLM, LLMModelRegistry
-from holmes.common.env_vars import (
-    ROBUSTA_CONFIG_PATH,
-)
 from holmes.core.tools_utils.tool_executor import ToolExecutor
 from holmes.core.toolset_manager import ToolsetManager
 from holmes.plugins.runbooks import (
@@ -33,8 +30,8 @@ if TYPE_CHECKING:
     from holmes.plugins.sources.pagerduty import PagerDutySource
     from holmes.plugins.sources.prometheus.plugin import AlertManagerSource
 
-from holmes.core.supabase_dal import SupabaseDal
 from holmes.core.config import config_path_dir
+from holmes.core.supabase_dal import SupabaseDal
 from holmes.utils.definitions import RobustaConfig
 from holmes.utils.pydantic_utils import RobustaBaseConfig, load_model_from_file
 
@@ -129,7 +126,7 @@ class Config(RobustaBaseConfig):
         return self._llm_model_registry
 
     def log_useful_info(self):
-        if self.llm_model_registry and self.llm_model_registry.models:
+        if self.llm_model_registry.models:
             logging.info(
                 f"Loaded models: {list(self.llm_model_registry.models.keys())}"
             )

holmes/core/conversations.py

@@ -262,7 +262,10 @@ def build_issue_chat_messages(
 
 
 def add_or_update_system_prompt(
-    conversation_history: List[Dict[str, str]], ai: ToolCallingLLM, config: Config
+    conversation_history: List[Dict[str, str]],
+    ai: ToolCallingLLM,
+    config: Config,
+    additional_system_prompt: Optional[str] = None,
 ):
     """Either add the system prompt or replace an existing system prompt.
     As a 'defensive' measure, this code will only replace an existing system prompt if it is the
@@ -278,6 +281,9 @@ def add_or_update_system_prompt(
 
     system_prompt = load_and_render_prompt(template_path, context)
 
+    if additional_system_prompt:
+        system_prompt = system_prompt + "\n" + additional_system_prompt
+
     if not conversation_history or len(conversation_history) == 0:
         conversation_history.append({"role": "system", "content": system_prompt})
     elif conversation_history[0]["role"] == "system":
@@ -303,6 +309,7 @@ def build_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    additional_system_prompt: Optional[str] = None,
 ) -> List[dict]:
     """
     This function generates a list of messages for general chat conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -358,7 +365,10 @@ def build_chat_messages(
         conversation_history = conversation_history.copy()
 
     conversation_history = add_or_update_system_prompt(
-        conversation_history=conversation_history, ai=ai, config=config
+        conversation_history=conversation_history,
+        ai=ai,
+        config=config,
+        additional_system_prompt=additional_system_prompt,
     )
 
     ask = add_global_instructions_to_user_prompt(ask, global_instructions)

holmes/core/feedback.py

@@ -0,0 +1,191 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Callable, Optional
+
+from .llm import LLM
+
+DEFAULT_PRIVACY_NOTICE_BANNER = "Your feedback will be used to improve Holmesgpt's performance. Please avoid sharing sensitive personal information. By continuing, you consent to this data usage."
+PRIVACY_NOTICE_BANNER = os.environ.get(
+    "PRIVACY_NOTICE_BANNER", DEFAULT_PRIVACY_NOTICE_BANNER
+)
+
+
+class FeedbackInfoBase(ABC):
+    """Abstract base class for all feedback-related classes that must implement to_dict()."""
+
+    @abstractmethod
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation. Must be implemented by all subclasses."""
+        pass
+
+
+class FeedbackLLM(FeedbackInfoBase):
+    """Class to represent a LLM in the feedback."""
+
+    def __init__(self, model: str, max_context_size: int):
+        self.model = model
+        self.max_context_size = max_context_size
+
+    def update_from_llm(self, llm: LLM):
+        self.model = llm.model
+        self.max_context_size = llm.get_context_window_size()
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return self.__dict__
+
+
+# TODO: extend the FeedbackLLMResponse to include each tool call results details used for evaluate the overall response.
+# Currenlty tool call details in plan:
+# - toolcall parameter and success/failure, toolcall truncation size
+# - Holmes plan (todo list)
+# - Holmes intermediate output
+class FeedbackLLMResponse(FeedbackInfoBase):
+    """Class to represent a LLM response in the feedback"""
+
+    def __init__(self, user_ask: str, response: str):
+        self.user_ask = user_ask
+        self.response = response
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return self.__dict__
+
+
+class FeedbackMetadata(FeedbackInfoBase):
+    """Class to store feedback metadata."""
+
+    def __init__(self):
+        # In iteration mode, there can be multiple ask and response pairs.
+        self.llm_responses = []
+        self.llm = FeedbackLLM("", 0)
+
+    def add_llm_response(self, user_ask: str, response: str) -> None:
+        """Add a LLM response to the metadata."""
+        llm_response = FeedbackLLMResponse(user_ask, response)
+        self.llm_responses.append(llm_response)
+
+    def update_llm(self, llm: LLM) -> None:
+        """Update the LLM information in the metadata."""
+        self.llm.update_from_llm(llm)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "llm_responses": [resp.to_dict() for resp in self.llm_responses],
+            "llm": self.llm.to_dict(),
+        }
+
+
+class UserFeedback(FeedbackInfoBase):
+    """Class to store user rate and comment to the AI response."""
+
+    def __init__(self, is_positive: bool, comment: Optional[str]):
+        self.is_positive = is_positive
+        self.comment = comment
+
+    @property
+    def rating_text(self) -> str:
+        """Return human-readable rating text."""
+        return "useful" if self.is_positive else "not useful"
+
+    @property
+    def rating_emoji(self) -> str:
+        """Return emoji representation of the rating."""
+        return "👍" if self.is_positive else "👎"
+
+    def __str__(self) -> str:
+        """Return string representation of the feedback."""
+        if self.comment:
+            return f"Rating: {self.rating_text}. Comment: {self.comment}"
+        else:
+            return f"Rating: {self.rating_text}. No additional comment."
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "is_positive": self.is_positive,
+            "comment": self.comment,
+        }
+
+
+class Feedback(FeedbackInfoBase):
+    """Class to store overall feedback data used to evaluate the AI response."""
+
+    def __init__(self):
+        self.metadata = FeedbackMetadata()
+        self.user_feedback: Optional[UserFeedback] = None
+
+    def set_user_feedback(self, user_feedback: UserFeedback) -> None:
+        """Set the user feedback."""
+        self.user_feedback = user_feedback
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "metadata": self.metadata.to_dict(),
+            "user_feedback": self.user_feedback.to_dict()
+            if self.user_feedback
+            else None,
+        }
+
+
+FeedbackCallback = Callable[[Feedback], None]
+
+
+def feedback_callback_example(feedback: Feedback) -> None:
+    """
+    Example implementation of a feedback callback function.
+
+    This function demonstrates how to process feedback data using to_dict() methods
+    and could be used for:
+    - Logging feedback to files or databases
+    - Sending feedback to analytics services
+    - Training data collection
+    - User satisfaction monitoring
+
+    Args:
+        feedback: Feedback object containing user feedback and metadata
+    """
+    print("\n=== Feedback Received ===")
+
+    # Convert entire feedback to dict first - this is the main data structure
+    feedback_dict = feedback.to_dict()
+    print(f"Complete feedback dictionary keys: {list(feedback_dict.keys())}")
+
+    # How to check user feedback using to_dict()
+    print("\n1. Checking User Feedback:")
+    user_feedback_dict = (
+        feedback.user_feedback.to_dict() if feedback.user_feedback else None
+    )
+    if user_feedback_dict:
+        print(f"   User feedback dict: {user_feedback_dict}")
+        print(f"   Is positive: {user_feedback_dict['is_positive']}")
+        print(f"   Comment: {user_feedback_dict['comment'] or 'None'}")
+        # You can also access properties through the object:
+        print(f"   Rating emoji: {feedback.user_feedback.rating_emoji}")  # type: ignore
+        print(f"   Rating text: {feedback.user_feedback.rating_text}")  # type: ignore
+    else:
+        print("   No user feedback provided (user_feedback is None)")
+
+    # How to check LLM information using to_dict()
+    print("\n2. Checking LLM Information:")
+    metadata_dict = feedback.metadata.to_dict()
+    llm_dict = metadata_dict["llm"]
+    print(f"   LLM dict: {llm_dict}")
+    print(f"   Model: {llm_dict['model']}")
+    print(f"   Max context size: {llm_dict['max_context_size']}")
+
+    # How to check ask and response pairs using to_dict()
+    print("\n3. Checking Ask and Response History:")
+    llm_responses_dict = metadata_dict["llm_responses"]
+    print(f"   Number of exchanges: {len(llm_responses_dict)}")
+
+    for i, response_dict in enumerate(llm_responses_dict, 1):
+        print(f"   Exchange {i} dict: {list(response_dict.keys())}")
+        user_ask = response_dict["user_ask"]
+        ai_response = response_dict["response"]
+        print(f"     User ask: {user_ask}")
+        print(f"     AI response: {ai_response}")
+
+    print("=== End Feedback ===\n")

holmes/core/llm.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from abc import abstractmethod
+from math import floor
 from typing import Any, Dict, List, Optional, Type, Union, TYPE_CHECKING
 
 from litellm.types.utils import ModelResponse, TextCompletionResponse
@@ -292,6 +293,8 @@ class DefaultLLM(LLM):
             raise Exception(f"Unexpected type returned by the LLM {type(result)}")
 
     def get_maximum_output_token(self) -> int:
+        max_output_tokens = floor(min(64000, self.get_context_window_size() / 5))
+
         if OVERRIDE_MAX_OUTPUT_TOKEN:
             logging.debug(
                 f"Using OVERRIDE_MAX_OUTPUT_TOKEN {OVERRIDE_MAX_OUTPUT_TOKEN}"
@@ -301,17 +304,22 @@ class DefaultLLM(LLM):
         # Try each name variant
         for name in self._get_model_name_variants_for_lookup():
             try:
-                return litellm.model_cost[name]["max_output_tokens"]
+                litellm_max_output_tokens = litellm.model_cost[name][
+                    "max_output_tokens"
+                ]
+                if litellm_max_output_tokens < max_output_tokens:
+                    max_output_tokens = litellm_max_output_tokens
+                return max_output_tokens
             except Exception:
                 continue
 
         # Log which lookups we tried
         logging.warning(
             f"Couldn't find model {self.model} in litellm's model list (tried: {', '.join(self._get_model_name_variants_for_lookup())}), "
-            f"using default 4096 tokens for max_output_tokens. "
+            f"using {max_output_tokens} tokens for max_output_tokens. "
             f"To override, set OVERRIDE_MAX_OUTPUT_TOKEN environment variable to the correct value for your model."
         )
-        return 4096
+        return max_output_tokens
 
     def _add_cache_control_to_last_message(
         self, messages: List[Dict[str, Any]]
@@ -349,7 +357,7 @@ class DefaultLLM(LLM):
         if content is None:
             return
 
-        if isinstance(content, str):
+        if isinstance(content, str) and content:
             # Convert string to structured format with cache_control
             target_msg["content"] = [
                 {
@@ -520,13 +528,14 @@ class LLMModelRegistry:
     def _create_robusta_model_entry(
         self, model_name: str, args: Optional[dict[str, Any]] = None
     ) -> dict[str, Any]:
-        return self._create_model_entry(
+        entry = self._create_model_entry(
             model="gpt-4o",  # Robusta AI model is using openai like API.
             model_name=model_name,
             base_url=f"{ROBUSTA_API_ENDPOINT}/llm/{model_name}",
             is_robusta_model=True,
-            args=args or {},
         )
+        entry["custom_args"] = args or {}  # type: ignore[assignment]
+        return entry
 
     def _create_model_entry(
         self,
@@ -534,18 +543,13 @@ class LLMModelRegistry:
         model_name: str,
         base_url: Optional[str] = None,
         is_robusta_model: Optional[bool] = None,
-        args: Optional[dict[str, Any]] = None,
     ) -> dict[str, Any]:
-        entry = {
+        return {
             "name": model_name,
             "base_url": base_url,
             "is_robusta_model": is_robusta_model,
             "model": model,
         }
-        if args:
-            entry["custom_args"] = args  # type: ignore[assignment]
-
-        return entry
 
 
 def get_llm_usage(

holmes/core/models.py

@@ -1,9 +1,87 @@
+import json
 from holmes.core.investigation_structured_output import InputSectionsDataType
-from holmes.core.tool_calling_llm import ToolCallResult
 from typing import Optional, List, Dict, Any, Union
 from pydantic import BaseModel, model_validator, Field
 from enum import Enum
 
+from holmes.core.tools import StructuredToolResult, StructuredToolResultStatus
+
+
+class TruncationMetadata(BaseModel):
+    tool_call_id: str
+    start_index: int
+    end_index: int
+    tool_name: str
+    original_token_count: int
+
+
+class TruncationResult(BaseModel):
+    truncated_messages: list[dict]
+    truncations: list[TruncationMetadata]
+
+
+class ToolCallResult(BaseModel):
+    tool_call_id: str
+    tool_name: str
+    description: str
+    result: StructuredToolResult
+    size: Optional[int] = None
+
+    def as_tool_call_message(self):
+        return {
+            "tool_call_id": self.tool_call_id,
+            "role": "tool",
+            "name": self.tool_name,
+            "content": format_tool_result_data(self.result),
+        }
+
+    def as_tool_result_response(self):
+        result_dump = self.result.model_dump()
+        result_dump["data"] = self.result.get_stringified_data()
+
+        return {
+            "tool_call_id": self.tool_call_id,
+            "tool_name": self.tool_name,
+            "description": self.description,
+            "role": "tool",
+            "result": result_dump,
+        }
+
+    def as_streaming_tool_result_response(self):
+        result_dump = self.result.model_dump()
+        result_dump["data"] = self.result.get_stringified_data()
+
+        return {
+            "tool_call_id": self.tool_call_id,
+            "role": "tool",
+            "description": self.description,
+            "name": self.tool_name,
+            "result": result_dump,
+        }
+
+
+def format_tool_result_data(tool_result: StructuredToolResult) -> str:
+    tool_response = tool_result.data
+    if isinstance(tool_result.data, str):
+        tool_response = tool_result.data
+    else:
+        try:
+            if isinstance(tool_result.data, BaseModel):
+                tool_response = tool_result.data.model_dump_json(indent=2)
+            else:
+                tool_response = json.dumps(tool_result.data, indent=2)
+        except Exception:
+            tool_response = str(tool_result.data)
+    if tool_result.status == StructuredToolResultStatus.ERROR:
+        tool_response = f"{tool_result.error or 'Tool execution failed'}:\n\n{tool_result.data or ''}".strip()
+
+    if tool_result.params:
+        tool_response = (
+            f"Params used for the tool call: {json.dumps(tool_result.params)}. The tool call output follows on the next line.\n"
+            + tool_response
+        )
+    return tool_response
+
 
 class InvestigationResult(BaseModel):
     analysis: Optional[str] = None
@@ -87,10 +165,31 @@ class ConversationRequest(BaseModel):
     include_tool_call_results: bool = False
 
 
+class PendingToolApproval(BaseModel):
+    """Represents a tool call that requires user approval."""
+
+    tool_call_id: str
+    tool_name: str
+    description: str
+    params: Dict[str, Any]
+
+
+class ToolApprovalDecision(BaseModel):
+    """Represents a user's decision on a tool approval."""
+
+    tool_call_id: str
+    approved: bool
+
+
 class ChatRequestBaseModel(BaseModel):
     conversation_history: Optional[list[dict]] = None
     model: Optional[str] = None
     stream: bool = Field(default=False)
+    enable_tool_approval: Optional[bool] = (
+        False  # Optional boolean for backwards compatibility
+    )
+    tool_decisions: Optional[List[ToolApprovalDecision]] = None
+    additional_system_prompt: Optional[str] = None
 
     # In our setup with litellm, the first message in conversation_history
     # should follow the structure [{"role": "system", "content": ...}],
@@ -146,6 +245,7 @@ class ChatResponse(BaseModel):
     conversation_history: list[dict]
     tool_calls: Optional[List[ToolCallResult]] = []
     follow_up_actions: Optional[List[FollowUpAction]] = []
+    pending_approvals: Optional[List[PendingToolApproval]] = None
     metadata: Optional[Dict[Any, Any]] = None
 
 

holmes/core/supabase_dal.py

@@ -1,5 +1,6 @@
 import base64
 import binascii
+import gzip
 import json
 import logging
 import os
@@ -7,7 +8,6 @@ import threading
 from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple
 from uuid import uuid4
-import gzip
 
 import yaml  # type: ignore
 from cachetools import TTLCache  # type: ignore
@@ -30,6 +30,9 @@ from holmes.core.resource_instruction import (
     ResourceInstructionDocument,
     ResourceInstructions,
 )
+from holmes.core.truncation.dal_truncation_utils import (
+    truncate_evidences_entities_if_necessary,
+)
 from holmes.utils.definitions import RobustaConfig
 from holmes.utils.env import get_env_replacement
 from holmes.utils.global_instructions import Instructions
@@ -46,6 +49,9 @@ HOLMES_TOOLSET = "HolmesToolsStatus"
 SCANS_META_TABLE = "ScansMeta"
 SCANS_RESULTS_TABLE = "ScansResults"
 
+ENRICHMENT_BLACKLIST = ["text_file", "graph", "ai_analysis", "holmes"]
+ENRICHMENT_BLACKLIST_SET = set(ENRICHMENT_BLACKLIST)
+
 
 class RobustaToken(BaseModel):
     store_url: str
@@ -60,7 +66,7 @@ class SupabaseDal:
         self.enabled = self.__init_config()
         self.cluster = cluster
         if not self.enabled:
-            logging.info(
+            logging.debug(
                 "Not connecting to Robusta platform - robusta token not provided - using ROBUSTA_AI will not be possible"
             )
             return
@@ -118,7 +124,7 @@ class SupabaseDal:
                 )
 
         if not os.path.exists(config_file_path):
-            logging.info(f"No robusta config in {config_file_path}")
+            logging.debug(f"No robusta config in {config_file_path}")
             return None
 
         logging.info(f"loading config {config_file_path}")
@@ -262,11 +268,14 @@ class SupabaseDal:
                 .select("*")
                 .eq("account_id", self.account_id)
                 .in_("issue_id", changes_ids)
+                .not_.in_("enrichment_type", ENRICHMENT_BLACKLIST)
                 .execute()
             )
             if not len(change_data_response.data):
                 return None
 
+            truncate_evidences_entities_if_necessary(change_data_response.data)
+
         except Exception:
             logging.exception("Supabase error while retrieving change content")
             return None
@@ -323,17 +332,17 @@ class SupabaseDal:
             return data
 
     def extract_relevant_issues(self, evidence):
-        enrichment_blacklist = {"text_file", "graph", "ai_analysis", "holmes"}
         data = [
             enrich
             for enrich in evidence.data
-            if enrich.get("enrichment_type") not in enrichment_blacklist
+            if enrich.get("enrichment_type") not in ENRICHMENT_BLACKLIST_SET
         ]
 
         unzipped_files = [
             self.unzip_evidence_file(enrich)
             for enrich in evidence.data
             if enrich.get("enrichment_type") == "text_file"
+            or enrich.get("enrichment_type") == "alert_raw_data"
         ]
 
         data.extend(unzipped_files)
@@ -370,12 +379,14 @@ class SupabaseDal:
         evidence = (
             self.client.table(EVIDENCE_TABLE)
             .select("*")
-            .filter("issue_id", "eq", issue_id)
+            .eq("issue_id", issue_id)
+            .not_.in_("enrichment_type", ENRICHMENT_BLACKLIST)
             .execute()
         )
-        data = self.extract_relevant_issues(evidence)
+        relevant_evidence = self.extract_relevant_issues(evidence)
+        truncate_evidences_entities_if_necessary(relevant_evidence)
 
-        issue_data["evidence"] = data
+        issue_data["evidence"] = relevant_evidence
 
         # build issue investigation dates
         started_at = issue_data.get("starts_at")
@@ -518,10 +529,13 @@ class SupabaseDal:
                 self.client.table(EVIDENCE_TABLE)
                 .select("data, enrichment_type")
                 .in_("issue_id", unique_issues)
+                .not_.in_("enrichment_type", ENRICHMENT_BLACKLIST)
                 .execute()
             )
 
-            return self.extract_relevant_issues(res)
+            relevant_issues = self.extract_relevant_issues(res)
+            truncate_evidences_entities_if_necessary(relevant_issues)
+            return relevant_issues
 
         except Exception:
             logging.exception("failed to fetch workload issues data", exc_info=True)