holmesgpt 0.13.2__py3-none-any.whl → 0.16.2a0__py3-none-any.whl

holmes/core/conversations.py

@@ -1,7 +1,6 @@
 from typing import Dict, List, Optional
 
 import sentry_sdk
-
 from holmes.config import Config
 from holmes.core.models import (
     ToolCallConversationResult,
@@ -10,9 +9,10 @@ from holmes.core.models import (
 )
 from holmes.plugins.prompts import load_and_render_prompt
 from holmes.core.tool_calling_llm import ToolCallingLLM
+from holmes.plugins.runbooks import RunbookCatalog
 from holmes.utils.global_instructions import (
     Instructions,
-    add_global_instructions_to_user_prompt,
+    add_runbooks_to_user_prompt,
 )
 
 DEFAULT_TOOL_SIZE = 10000
@@ -26,7 +26,8 @@ def calculate_tool_size(
         return DEFAULT_TOOL_SIZE
 
     context_window = ai.llm.get_context_window_size()
-    message_size_without_tools = ai.llm.count_tokens_for_message(messages_without_tools)
+    tokens = ai.llm.count_tokens(messages_without_tools)
+    message_size_without_tools = tokens.total_tokens
     maximum_output_token = ai.llm.get_maximum_output_token()
 
     tool_size = min(
@@ -63,6 +64,7 @@ def build_issue_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """
     This function generates a list of messages for issue conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -119,8 +121,10 @@ def build_issue_chat_messages(
     tools_for_investigation = issue_chat_request.investigation_result.tools
 
     if not conversation_history or len(conversation_history) == 0:
-        user_prompt = add_global_instructions_to_user_prompt(
-            user_prompt, global_instructions
+        user_prompt = add_runbooks_to_user_prompt(
+            user_prompt=user_prompt,
+            runbook_catalog=runbooks,
+            global_instructions=global_instructions,
         )
 
         number_of_tools_for_investigation = len(tools_for_investigation)  # type: ignore
@@ -133,6 +137,7 @@ def build_issue_chat_messages(
                     "issue": issue_chat_request.issue_type,
                     "toolsets": ai.tool_executor.toolsets,
                     "cluster_name": config.cluster_name,
+                    "runbooks_enabled": True if runbooks else False,
                 },
             )
             messages = [
@@ -153,6 +158,7 @@ def build_issue_chat_messages(
             "issue": issue_chat_request.issue_type,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_without_tools = load_and_render_prompt(
             template_path, template_context_without_tools
@@ -186,6 +192,7 @@ def build_issue_chat_messages(
             "issue": issue_chat_request.issue_type,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_with_truncated_tools = load_and_render_prompt(
             template_path, truncated_template_context
@@ -201,8 +208,10 @@ def build_issue_chat_messages(
             },
         ]
 
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
     )
 
     conversation_history.append(
@@ -227,6 +236,7 @@ def build_issue_chat_messages(
         "issue": issue_chat_request.issue_type,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_without_tools = load_and_render_prompt(
         template_path, template_context_without_tools
@@ -250,6 +260,7 @@ def build_issue_chat_messages(
         "issue": issue_chat_request.issue_type,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_with_truncated_tools = load_and_render_prompt(
         template_path, template_context
@@ -262,7 +273,11 @@ def build_issue_chat_messages(
 
 
 def add_or_update_system_prompt(
-    conversation_history: List[Dict[str, str]], ai: ToolCallingLLM, config: Config
+    conversation_history: List[Dict[str, str]],
+    ai: ToolCallingLLM,
+    config: Config,
+    additional_system_prompt: Optional[str] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """Either add the system prompt or replace an existing system prompt.
     As a 'defensive' measure, this code will only replace an existing system prompt if it is the
@@ -274,9 +289,12 @@ def add_or_update_system_prompt(
     context = {
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
 
     system_prompt = load_and_render_prompt(template_path, context)
+    if additional_system_prompt:
+        system_prompt = system_prompt + "\n" + additional_system_prompt
 
     if not conversation_history or len(conversation_history) == 0:
         conversation_history.append({"role": "system", "content": system_prompt})
@@ -303,6 +321,8 @@ def build_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    additional_system_prompt: Optional[str] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ) -> List[dict]:
     """
     This function generates a list of messages for general chat conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -358,10 +378,18 @@ def build_chat_messages(
         conversation_history = conversation_history.copy()
 
     conversation_history = add_or_update_system_prompt(
-        conversation_history=conversation_history, ai=ai, config=config
+        conversation_history=conversation_history,
+        ai=ai,
+        config=config,
+        additional_system_prompt=additional_system_prompt,
+        runbooks=runbooks,
     )
 
-    ask = add_global_instructions_to_user_prompt(ask, global_instructions)
+    ask = add_runbooks_to_user_prompt(
+        user_prompt=ask,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
+    )
 
     conversation_history.append(  # type: ignore
         {
@@ -369,6 +397,7 @@ def build_chat_messages(
             "content": ask,
         },
     )
+
     number_of_tools = len(
         [message for message in conversation_history if message.get("role") == "tool"]  # type: ignore
     )
@@ -393,6 +422,7 @@ def build_workload_health_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """
     This function generates a list of messages for workload health conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -451,8 +481,10 @@ def build_workload_health_chat_messages(
     resource = workload_health_chat_request.resource
 
     if not conversation_history or len(conversation_history) == 0:
-        user_prompt = add_global_instructions_to_user_prompt(
-            user_prompt, global_instructions
+        user_prompt = add_runbooks_to_user_prompt(
+            user_prompt=user_prompt,
+            runbook_catalog=runbooks,
+            global_instructions=global_instructions,
         )
 
         number_of_tools_for_workload = len(tools_for_workload)  # type: ignore
@@ -465,6 +497,7 @@ def build_workload_health_chat_messages(
                     "resource": resource,
                     "toolsets": ai.tool_executor.toolsets,
                     "cluster_name": config.cluster_name,
+                    "runbooks_enabled": True if runbooks else False,
                 },
             )
             messages = [
@@ -485,6 +518,7 @@ def build_workload_health_chat_messages(
             "resource": resource,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_without_tools = load_and_render_prompt(
             template_path, template_context_without_tools
@@ -518,6 +552,7 @@ def build_workload_health_chat_messages(
             "resource": resource,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_with_truncated_tools = load_and_render_prompt(
             template_path, truncated_template_context
@@ -533,8 +568,10 @@ def build_workload_health_chat_messages(
             },
         ]
 
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
     )
 
     conversation_history.append(
@@ -559,6 +596,7 @@ def build_workload_health_chat_messages(
         "resource": resource,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_without_tools = load_and_render_prompt(
         template_path, template_context_without_tools
@@ -582,6 +620,7 @@ def build_workload_health_chat_messages(
         "resource": resource,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_with_truncated_tools = load_and_render_prompt(
         template_path, template_context

holmes/core/feedback.py

@@ -0,0 +1,191 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Callable, Optional
+
+from .llm import LLM
+
+DEFAULT_PRIVACY_NOTICE_BANNER = "Your feedback will be used to improve Holmesgpt's performance. Please avoid sharing sensitive personal information. By continuing, you consent to this data usage."
+PRIVACY_NOTICE_BANNER = os.environ.get(
+    "PRIVACY_NOTICE_BANNER", DEFAULT_PRIVACY_NOTICE_BANNER
+)
+
+
+class FeedbackInfoBase(ABC):
+    """Abstract base class for all feedback-related classes that must implement to_dict()."""
+
+    @abstractmethod
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation. Must be implemented by all subclasses."""
+        pass
+
+
+class FeedbackLLM(FeedbackInfoBase):
+    """Class to represent a LLM in the feedback."""
+
+    def __init__(self, model: str, max_context_size: int):
+        self.model = model
+        self.max_context_size = max_context_size
+
+    def update_from_llm(self, llm: LLM):
+        self.model = llm.model
+        self.max_context_size = llm.get_context_window_size()
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return self.__dict__
+
+
+# TODO: extend the FeedbackLLMResponse to include each tool call results details used for evaluate the overall response.
+# Currenlty tool call details in plan:
+# - toolcall parameter and success/failure, toolcall truncation size
+# - Holmes plan (todo list)
+# - Holmes intermediate output
+class FeedbackLLMResponse(FeedbackInfoBase):
+    """Class to represent a LLM response in the feedback"""
+
+    def __init__(self, user_ask: str, response: str):
+        self.user_ask = user_ask
+        self.response = response
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return self.__dict__
+
+
+class FeedbackMetadata(FeedbackInfoBase):
+    """Class to store feedback metadata."""
+
+    def __init__(self):
+        # In iteration mode, there can be multiple ask and response pairs.
+        self.llm_responses = []
+        self.llm = FeedbackLLM("", 0)
+
+    def add_llm_response(self, user_ask: str, response: str) -> None:
+        """Add a LLM response to the metadata."""
+        llm_response = FeedbackLLMResponse(user_ask, response)
+        self.llm_responses.append(llm_response)
+
+    def update_llm(self, llm: LLM) -> None:
+        """Update the LLM information in the metadata."""
+        self.llm.update_from_llm(llm)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "llm_responses": [resp.to_dict() for resp in self.llm_responses],
+            "llm": self.llm.to_dict(),
+        }
+
+
+class UserFeedback(FeedbackInfoBase):
+    """Class to store user rate and comment to the AI response."""
+
+    def __init__(self, is_positive: bool, comment: Optional[str]):
+        self.is_positive = is_positive
+        self.comment = comment
+
+    @property
+    def rating_text(self) -> str:
+        """Return human-readable rating text."""
+        return "useful" if self.is_positive else "not useful"
+
+    @property
+    def rating_emoji(self) -> str:
+        """Return emoji representation of the rating."""
+        return "👍" if self.is_positive else "👎"
+
+    def __str__(self) -> str:
+        """Return string representation of the feedback."""
+        if self.comment:
+            return f"Rating: {self.rating_text}. Comment: {self.comment}"
+        else:
+            return f"Rating: {self.rating_text}. No additional comment."
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "is_positive": self.is_positive,
+            "comment": self.comment,
+        }
+
+
+class Feedback(FeedbackInfoBase):
+    """Class to store overall feedback data used to evaluate the AI response."""
+
+    def __init__(self):
+        self.metadata = FeedbackMetadata()
+        self.user_feedback: Optional[UserFeedback] = None
+
+    def set_user_feedback(self, user_feedback: UserFeedback) -> None:
+        """Set the user feedback."""
+        self.user_feedback = user_feedback
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary representation."""
+        return {
+            "metadata": self.metadata.to_dict(),
+            "user_feedback": self.user_feedback.to_dict()
+            if self.user_feedback
+            else None,
+        }
+
+
+FeedbackCallback = Callable[[Feedback], None]
+
+
+def feedback_callback_example(feedback: Feedback) -> None:
+    """
+    Example implementation of a feedback callback function.
+
+    This function demonstrates how to process feedback data using to_dict() methods
+    and could be used for:
+    - Logging feedback to files or databases
+    - Sending feedback to analytics services
+    - Training data collection
+    - User satisfaction monitoring
+
+    Args:
+        feedback: Feedback object containing user feedback and metadata
+    """
+    print("\n=== Feedback Received ===")
+
+    # Convert entire feedback to dict first - this is the main data structure
+    feedback_dict = feedback.to_dict()
+    print(f"Complete feedback dictionary keys: {list(feedback_dict.keys())}")
+
+    # How to check user feedback using to_dict()
+    print("\n1. Checking User Feedback:")
+    user_feedback_dict = (
+        feedback.user_feedback.to_dict() if feedback.user_feedback else None
+    )
+    if user_feedback_dict:
+        print(f"   User feedback dict: {user_feedback_dict}")
+        print(f"   Is positive: {user_feedback_dict['is_positive']}")
+        print(f"   Comment: {user_feedback_dict['comment'] or 'None'}")
+        # You can also access properties through the object:
+        print(f"   Rating emoji: {feedback.user_feedback.rating_emoji}")  # type: ignore
+        print(f"   Rating text: {feedback.user_feedback.rating_text}")  # type: ignore
+    else:
+        print("   No user feedback provided (user_feedback is None)")
+
+    # How to check LLM information using to_dict()
+    print("\n2. Checking LLM Information:")
+    metadata_dict = feedback.metadata.to_dict()
+    llm_dict = metadata_dict["llm"]
+    print(f"   LLM dict: {llm_dict}")
+    print(f"   Model: {llm_dict['model']}")
+    print(f"   Max context size: {llm_dict['max_context_size']}")
+
+    # How to check ask and response pairs using to_dict()
+    print("\n3. Checking Ask and Response History:")
+    llm_responses_dict = metadata_dict["llm_responses"]
+    print(f"   Number of exchanges: {len(llm_responses_dict)}")
+
+    for i, response_dict in enumerate(llm_responses_dict, 1):
+        print(f"   Exchange {i} dict: {list(response_dict.keys())}")
+        user_ask = response_dict["user_ask"]
+        ai_response = response_dict["response"]
+        print(f"     User ask: {user_ask}")
+        print(f"     AI response: {ai_response}")
+
+    print("=== End Feedback ===\n")

holmes/core/investigation.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional
 
+
 from holmes.common.env_vars import HOLMES_POST_PROCESSING_PROMPT
 from holmes.config import Config
 from holmes.core.investigation_structured_output import process_response_into_sections
@@ -8,7 +9,8 @@ from holmes.core.issue import Issue
 from holmes.core.models import InvestigateRequest, InvestigationResult
 from holmes.core.supabase_dal import SupabaseDal
 from holmes.core.tracing import DummySpan, SpanType
-from holmes.utils.global_instructions import add_global_instructions_to_user_prompt
+from holmes.plugins.runbooks import RunbookCatalog
+from holmes.utils.global_instructions import add_runbooks_to_user_prompt
 
 from holmes.core.investigation_structured_output import (
     DEFAULT_SECTIONS,
@@ -25,8 +27,8 @@ def investigate_issues(
     config: Config,
     model: Optional[str] = None,
     trace_span=DummySpan(),
+    runbooks: Optional[RunbookCatalog] = None,
 ) -> InvestigationResult:
-    config.load_robusta_api_key(dal=dal)
     context = dal.get_issue_data(investigate_request.context.get("robusta_issue_id"))
 
     resource_instructions = dal.get_resource_instructions(
@@ -61,6 +63,7 @@ def investigate_issues(
         global_instructions=global_instructions,
         sections=investigate_request.sections,
         trace_span=trace_span,
+        runbooks=runbooks,
     )
 
     (text_response, sections) = process_response_into_sections(investigation.result)
@@ -71,6 +74,7 @@ def investigate_issues(
         sections=sections,
         tool_calls=investigation.tool_calls or [],
         instructions=investigation.instructions,
+        metadata=investigation.metadata,
     )
 
 
@@ -80,7 +84,6 @@ def get_investigation_context(
     config: Config,
     request_structured_output_from_llm: Optional[bool] = None,
 ):
-    config.load_robusta_api_key(dal=dal)
     ai = config.create_issue_investigator(dal=dal, model=investigate_request.model)
 
     raw_data = investigate_request.model_dump()
@@ -96,18 +99,11 @@ def get_investigation_context(
         raw=raw_data,
     )
 
-    runbooks = ai.runbook_manager.get_instructions_for_issue(issue)
+    issue_instructions = ai.runbook_manager.get_instructions_for_issue(issue)
 
-    instructions = dal.get_resource_instructions(
+    resource_instructions = dal.get_resource_instructions(
         "alert", investigate_request.context.get("issue_type")
     )
-    if instructions is not None and instructions.instructions:
-        runbooks.extend(instructions.instructions)
-    if instructions is not None and len(instructions.documents) > 0:
-        docPrompts = []
-        for document in instructions.documents:
-            docPrompts.append(f"* fetch information from this URL: {document.url}\n")
-        runbooks.extend(docPrompts)
 
     # This section is about setting vars to request the LLM to return structured output.
     # It does not mean that Holmes will not return structured sections for investigation as it is
@@ -132,6 +128,7 @@ def get_investigation_context(
     else:
         logging.info("Structured output is disabled for this request")
 
+    runbook_catalog = config.get_runbook_catalog()
     system_prompt = load_and_render_prompt(
         investigate_request.prompt_template,
         {
@@ -140,21 +137,20 @@ def get_investigation_context(
             "structured_output": request_structured_output_from_llm,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbook_catalog else False,
         },
     )
-
     user_prompt = ""
-    if runbooks:
-        for runbook_str in runbooks:
-            user_prompt += f"* {runbook_str}\n"
-
-        user_prompt = f'My instructions to check \n"""{user_prompt}"""'
 
     global_instructions = dal.get_global_instructions_for_account()
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbook_catalog,
+        global_instructions=global_instructions,
+        issue_instructions=issue_instructions,
+        resource_instructions=resource_instructions,
     )
 
-    user_prompt = f"{user_prompt}\n This is context from the issue {issue.raw}"
+    user_prompt = f"{user_prompt}\n #This is context from the issue:\n{issue.raw}"
 
-    return ai, system_prompt, user_prompt, response_format, sections, runbooks
+    return ai, system_prompt, user_prompt, response_format, sections, issue_instructions