holmesgpt 0.13.2__py3-none-any.whl → 0.18.4__py3-none-any.whl

holmes/core/toolset_manager.py

@@ -2,7 +2,7 @@ import concurrent.futures
 import json
 import logging
 import os
-from typing import Any, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from benedict import benedict
 from pydantic import FilePath
@@ -13,8 +13,31 @@ from holmes.core.tools import Toolset, ToolsetStatusEnum, ToolsetTag, ToolsetTyp
 from holmes.plugins.toolsets import load_builtin_toolsets, load_toolsets_from_config
 from holmes.utils.definitions import CUSTOM_TOOLSET_LOCATION
 
+if TYPE_CHECKING:
+    pass
+
 DEFAULT_TOOLSET_STATUS_LOCATION = os.path.join(config_path_dir, "toolsets_status.json")
 
+# Mapping of deprecated toolset names to their new names
+DEPRECATED_TOOLSET_NAMES: dict[str, str] = {
+    "coralogix/logs": "coralogix",
+}
+
+
+def handle_deprecated_toolset_name(
+    toolset_name: str, builtin_toolset_names: list[str]
+) -> str:
+    if toolset_name in DEPRECATED_TOOLSET_NAMES:
+        new_name = DEPRECATED_TOOLSET_NAMES[toolset_name]
+        if new_name in builtin_toolset_names:
+            logging.warning(
+                f"The toolset name '{toolset_name}' is deprecated. "
+                f"Please use '{new_name}' instead. "
+                "The old name will continue to work but may be removed in a future version."
+            )
+            return new_name
+    return toolset_name
+
 
 class ToolsetManager:
     """
@@ -30,14 +53,18 @@ class ToolsetManager:
         custom_toolsets: Optional[List[FilePath]] = None,
         custom_toolsets_from_cli: Optional[List[FilePath]] = None,
         toolset_status_location: Optional[FilePath] = None,
+        global_fast_model: Optional[str] = None,
+        custom_runbook_catalogs: Optional[List[Union[str, FilePath]]] = None,
     ):
         self.toolsets = toolsets
         self.toolsets = toolsets or {}
+        self.custom_runbook_catalogs = custom_runbook_catalogs
         if mcp_servers is not None:
             for _, mcp_server in mcp_servers.items():
                 mcp_server["type"] = ToolsetType.MCP.value
         self.toolsets.update(mcp_servers or {})
         self.custom_toolsets = custom_toolsets
+        self.global_fast_model = global_fast_model
 
         if toolset_status_location is None:
             toolset_status_location = FilePath(DEFAULT_TOOLSET_STATUS_LOCATION)
@@ -81,7 +108,15 @@ class ToolsetManager:
         3. custom toolset from config can override both built-in and add new custom toolsets # for backward compatibility
         """
         # Load built-in toolsets
-        builtin_toolsets = load_builtin_toolsets(dal)
+        # Extract search paths from custom catalog files
+        additional_search_paths = None
+        if self.custom_runbook_catalogs:
+            additional_search_paths = [
+                os.path.dirname(os.path.abspath(str(catalog_path)))
+                for catalog_path in self.custom_runbook_catalogs
+            ]
+
+        builtin_toolsets = load_builtin_toolsets(dal, additional_search_paths)
         toolsets_by_name: dict[str, Toolset] = {
             toolset.name: toolset for toolset in builtin_toolsets
         }
@@ -118,9 +153,13 @@ class ToolsetManager:
                 if any(tag in toolset_tags for tag in toolset.tags)
             }
 
+        # Inject global fast_model into all toolsets
+        final_toolsets = list(toolsets_by_name.values())
+        self._inject_fast_model_into_transformers(final_toolsets)
+
         # check_prerequisites against each enabled toolset
         if not check_prerequisites:
-            return list(toolsets_by_name.values())
+            return final_toolsets
 
         enabled_toolsets: List[Toolset] = []
         for _, toolset in toolsets_by_name.items():
@@ -130,7 +169,7 @@ class ToolsetManager:
                 toolset.status = ToolsetStatusEnum.DISABLED
         self.check_toolset_prerequisites(enabled_toolsets)
 
-        return list(toolsets_by_name.values())
+        return final_toolsets
 
     @classmethod
     def check_toolset_prerequisites(cls, toolsets: list[Toolset]):
@@ -155,6 +194,10 @@ class ToolsetManager:
         builtin_toolsets_dict: dict[str, dict[str, Any]] = {}
         custom_toolsets_dict: dict[str, dict[str, Any]] = {}
         for toolset_name, toolset_config in toolsets.items():
+            toolset_name = handle_deprecated_toolset_name(
+                toolset_name, builtin_toolset_names
+            )
+
             if toolset_name in builtin_toolset_names:
                 # build-in types was assigned when loaded
                 builtin_toolsets_dict[toolset_name] = toolset_config
@@ -266,7 +309,11 @@ class ToolsetManager:
                 toolset.path = cached_status.get("path", None)
             # check prerequisites for only enabled toolset when the toolset is loaded from cache. When the toolset is
             # not loaded from cache, the prerequisites are checked in the refresh_toolset_status method.
-            if toolset.enabled and toolset.status == ToolsetStatusEnum.ENABLED:
+            if toolset.enabled and (
+                toolset.status == ToolsetStatusEnum.ENABLED
+                or toolset.type == ToolsetType.MCP
+            ):
+                # MCP servers need to reload their tools even if previously failed, so rerun prerequisites
                 enabled_toolsets_from_cache.append(toolset)
         self.check_toolset_prerequisites(enabled_toolsets_from_cache)
 
@@ -276,6 +323,10 @@ class ToolsetManager:
             list(toolsets_status_by_name.keys()),
             check_conflict_default=True,
         )
+
+        # Inject fast_model into CLI custom toolsets
+        self._inject_fast_model_into_transformers(custom_toolsets_from_cli)
+
         # custom toolsets from cli as experimental toolset should not override custom toolsets from config
         enabled_toolsets_from_cli: List[Toolset] = []
         for custom_toolset_from_cli in custom_toolsets_from_cli:
@@ -438,3 +489,138 @@ class ToolsetManager:
             else:
                 existing_toolsets_by_name[new_toolset.name] = new_toolset
                 existing_toolsets_by_name[new_toolset.name] = new_toolset
+
+    def _inject_fast_model_into_transformers(self, toolsets: List[Toolset]) -> None:
+        """
+        Inject global fast_model setting into all llm_summarize transformers that don't already have fast_model.
+        This ensures --fast-model reaches all tools regardless of toolset-level transformer configuration.
+
+        IMPORTANT: This also forces recreation of transformer instances since they may already be created.
+        """
+        import logging
+
+        from holmes.core.transformers import registry
+
+        logger = logging.getLogger(__name__)
+
+        logger.debug(
+            f"Starting fast_model injection. global_fast_model={self.global_fast_model}"
+        )
+
+        if not self.global_fast_model:
+            logger.debug("No global_fast_model configured, skipping injection")
+            return
+
+        injected_count = 0
+        toolset_count = 0
+
+        for toolset in toolsets:
+            toolset_count += 1
+            toolset_injected = 0
+            logger.debug(
+                f"Processing toolset '{toolset.name}', has toolset transformers: {toolset.transformers is not None}"
+            )
+
+            # Inject into toolset-level transformers
+            if toolset.transformers:
+                logger.debug(
+                    f"Toolset '{toolset.name}' has {len(toolset.transformers)} toolset-level transformers"
+                )
+                for transformer in toolset.transformers:
+                    logger.debug(
+                        f"  Toolset transformer: name='{transformer.name}', config keys={list(transformer.config.keys())}"
+                    )
+                    if (
+                        transformer.name == "llm_summarize"
+                        and "fast_model" not in transformer.config
+                    ):
+                        transformer.config["global_fast_model"] = self.global_fast_model
+                        injected_count += 1
+                        toolset_injected += 1
+                        logger.info(
+                            f"  ✓ Injected global_fast_model into toolset '{toolset.name}' transformer"
+                        )
+                    elif transformer.name == "llm_summarize":
+                        logger.debug(
+                            f"  - Toolset transformer already has fast_model: {transformer.config.get('fast_model')}"
+                        )
+            else:
+                logger.debug(
+                    f"Toolset '{toolset.name}' has no toolset-level transformers"
+                )
+
+            # Inject into tool-level transformers
+            if hasattr(toolset, "tools") and toolset.tools:
+                logger.debug(f"Toolset '{toolset.name}' has {len(toolset.tools)} tools")
+                for tool in toolset.tools:
+                    logger.debug(
+                        f"  Processing tool '{tool.name}', has transformers: {tool.transformers is not None}"
+                    )
+                    if tool.transformers:
+                        logger.debug(
+                            f"    Tool '{tool.name}' has {len(tool.transformers)} transformers"
+                        )
+                        tool_updated = False
+                        for transformer in tool.transformers:
+                            logger.debug(
+                                f"      Tool transformer: name='{transformer.name}', config keys={list(transformer.config.keys())}"
+                            )
+                            if (
+                                transformer.name == "llm_summarize"
+                                and "fast_model" not in transformer.config
+                            ):
+                                transformer.config["global_fast_model"] = (
+                                    self.global_fast_model
+                                )
+                                injected_count += 1
+                                toolset_injected += 1
+                                tool_updated = True
+                                logger.info(
+                                    f"      ✓ Injected global_fast_model into tool '{tool.name}' transformer"
+                                )
+                            elif transformer.name == "llm_summarize":
+                                logger.debug(
+                                    f"      - Tool transformer already has fast_model: {transformer.config.get('fast_model')}"
+                                )
+
+                        # CRITICAL: Force recreation of transformer instances if we updated the config
+                        if tool_updated:
+                            logger.info(
+                                f"      🔄 Recreating transformer instances for tool '{tool.name}' after injection"
+                            )
+                            if tool.transformers:
+                                tool._transformer_instances = []
+                                for transformer in tool.transformers:
+                                    if not transformer:
+                                        continue
+                                    try:
+                                        # Create transformer instance with updated config
+                                        transformer_instance = (
+                                            registry.create_transformer(
+                                                transformer.name, transformer.config
+                                            )
+                                        )
+                                        tool._transformer_instances.append(
+                                            transformer_instance
+                                        )
+                                        logger.debug(
+                                            f"        Recreated transformer '{transformer.name}' for tool '{tool.name}' with config: {transformer.config}"
+                                        )
+                                    except Exception as e:
+                                        logger.warning(
+                                            f"        Failed to recreate transformer '{transformer.name}' for tool '{tool.name}': {e}"
+                                        )
+                                        continue
+                    else:
+                        logger.debug(f"    Tool '{tool.name}' has no transformers")
+            else:
+                logger.debug(f"Toolset '{toolset.name}' has no tools")
+
+            if toolset_injected > 0:
+                logger.info(
+                    f"Toolset '{toolset.name}': injected into {toolset_injected} transformers"
+                )
+
+        logger.info(
+            f"Fast_model injection complete: {injected_count} transformers updated across {toolset_count} toolsets"
+        )

holmes/core/tracing.py

@@ -41,7 +41,18 @@ def readable_timestamp():
 
 def get_active_branch_name():
     try:
-        # First check if .git is a file (worktree case)
+        # First check GitHub Actions environment variables (CI)
+        github_head_ref = os.environ.get("GITHUB_HEAD_REF")  # Set for PRs
+        if github_head_ref:
+            return github_head_ref
+
+        github_ref = os.environ.get(
+            "GITHUB_REF", ""
+        )  # Set for pushes: refs/heads/branch-name
+        if github_ref.startswith("refs/heads/"):
+            return github_ref.replace("refs/heads/", "")
+
+        # Check if .git is a file (worktree case)
         git_path = Path(".git")
         if git_path.is_file():
             # Read the worktree git directory path
@@ -101,7 +112,7 @@ class SpanType(Enum):
 class DummySpan:
     """A no-op span implementation for when tracing is disabled."""
 
-    def start_span(self, name: str, span_type=None, **kwargs):
+    def start_span(self, name: Optional[str] = None, span_type=None, **kwargs):
         return DummySpan()
 
     def log(self, *args, **kwargs):
@@ -110,6 +121,11 @@ class DummySpan:
     def end(self):
         pass
 
+    def set_attributes(
+        self, name: Optional[str] = None, type=None, span_attributes=None
+    ) -> None:
+        pass
+
     def __enter__(self):
         return self
 
@@ -231,7 +247,7 @@ class BraintrustTracer:
         else:
             logging.warning("No active span found in Braintrust context")
 
-        return f"https://www.braintrust.dev/app/robustadev/p/{self.project}/experiments/{experiment_name}"
+        return f"https://www.braintrust.dev/app/{BRAINTRUST_ORG}/p/{self.project}/experiments/{experiment_name}"
 
     def wrap_llm(self, llm_module):
         """Wrap LiteLLM with Braintrust tracing if in active context, otherwise return unwrapped."""

holmes/core/transformers/__init__.py

@@ -0,0 +1,23 @@
+"""
+Transformer system for processing tool outputs.
+
+This module provides the infrastructure for transforming tool outputs
+before they are passed to the LLM for analysis.
+"""
+
+from .base import BaseTransformer, TransformerError
+from .llm_summarize import LLMSummarizeTransformer
+from .registry import TransformerRegistry, registry
+from .transformer import Transformer
+
+# Register built-in transformers
+registry.register(LLMSummarizeTransformer)
+
+__all__ = [
+    "BaseTransformer",
+    "TransformerError",
+    "TransformerRegistry",
+    "registry",
+    "LLMSummarizeTransformer",
+    "Transformer",
+]

holmes/core/transformers/base.py

@@ -0,0 +1,63 @@
+"""
+Base transformer abstract class for tool output transformation.
+"""
+
+__all__ = ["BaseTransformer", "TransformerError"]
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel
+
+
+class TransformerError(Exception):
+    """Exception raised when transformer operations fail."""
+
+    pass
+
+
+class BaseTransformer(BaseModel, ABC):
+    """
+    Abstract base class for all tool output transformers.
+
+    Transformers process tool outputs before they are returned to the LLM,
+    enabling operations like summarization, filtering, or format conversion.
+    """
+
+    @abstractmethod
+    def transform(self, input_text: str) -> str:
+        """
+        Transform the input text and return the transformed output.
+
+        Args:
+            input_text: The raw tool output to transform
+
+        Returns:
+            The transformed output text
+
+        Raises:
+            TransformerError: If transformation fails
+        """
+        pass
+
+    @abstractmethod
+    def should_apply(self, input_text: str) -> bool:
+        """
+        Determine whether this transformer should be applied to the input.
+
+        Args:
+            input_text: The raw tool output to check
+
+        Returns:
+            True if the transformer should be applied, False otherwise
+        """
+        pass
+
+    @property
+    def name(self) -> str:
+        """
+        Get the transformer name.
+
+        Returns:
+            The transformer name (class name by default)
+        """
+        return self.__class__.__name__

holmes/core/transformers/llm_summarize.py

@@ -0,0 +1,175 @@
+"""
+LLM Summarize Transformer for fast model summarization of large tool outputs.
+"""
+
+import logging
+from typing import ClassVar, Optional
+
+from pydantic import Field, PrivateAttr, StrictStr
+
+from ..llm import LLM, DefaultLLM
+from .base import BaseTransformer, TransformerError
+
+logger = logging.getLogger(__name__)
+
+
+class LLMSummarizeTransformer(BaseTransformer):
+    """
+    Transformer that uses a fast LLM model to summarize large tool outputs.
+
+    This transformer applies summarization when:
+    1. A fast model is available
+    2. The input length exceeds the configured threshold
+
+    Configuration options:
+    - input_threshold: Minimum input length to trigger summarization (default: 1000)
+    - prompt: Custom prompt template for summarization (optional)
+    - fast_model: Fast model name for summarization (e.g., "gpt-4o-mini")
+    - api_key: API key for the fast model (optional, uses default if not provided)
+    """
+
+    DEFAULT_PROMPT: ClassVar[str] = """Summarize this operational data focusing on:
+- What needs attention or immediate action
+- Group similar entries into a single line and description
+- Make sure to mention outliers, errors, and non-standard patterns
+- List normal/healthy patterns as aggregate descriptions
+- When listing problematic entries, also try to use aggregate descriptions when possible
+- When possible, mention exact keywords, IDs, or patterns so the user can filter/search the original data and drill down on the parts they care about (extraction over abstraction)"""
+
+    # Pydantic fields with validation
+    input_threshold: int = Field(
+        default=1000, ge=0, description="Minimum input length to trigger summarization"
+    )
+    prompt: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Custom prompt template for summarization",
+    )
+    fast_model: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Fast model name for summarization (e.g., 'gpt-4o-mini')",
+    )
+    global_fast_model: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Global fast model name fallback when fast_model is not set",
+    )
+    api_key: Optional[str] = Field(
+        default=None,
+        description="API key for the fast model (optional, uses default if not provided)",
+    )
+
+    # Private attribute for the LLM instance (not serialized)
+    _fast_llm: Optional[LLM] = PrivateAttr(default=None)
+
+    def model_post_init(self, __context) -> None:
+        """Initialize the fast LLM instance after model validation."""
+        logger = logging.getLogger(__name__)
+
+        self._fast_llm = None
+
+        # Determine which fast model to use: fast_model takes precedence over global_fast_model
+        effective_fast_model = self.fast_model or self.global_fast_model
+
+        logger.debug(
+            f"LLMSummarizeTransformer initialization: fast_model='{self.fast_model}', global_fast_model='{self.global_fast_model}', effective='{effective_fast_model}'"
+        )
+
+        # Create fast LLM instance if a fast model is available
+        if effective_fast_model:
+            try:
+                self._fast_llm = DefaultLLM(effective_fast_model, self.api_key)
+                logger.info(
+                    f"Created fast LLM instance with model: {effective_fast_model}"
+                )
+            except Exception as e:
+                logger.warning(f"Failed to create fast LLM instance: {e}")
+                self._fast_llm = None
+        else:
+            logger.debug(
+                "No fast model configured (neither fast_model nor global_fast_model)"
+            )
+
+    def should_apply(self, input_text: str) -> bool:
+        """
+        Determine if summarization should be applied to the input.
+
+        Args:
+            input_text: The tool output to check
+
+        Returns:
+            True if summarization should be applied, False otherwise
+        """
+        logger = logging.getLogger(__name__)
+
+        # Skip if no fast model is configured
+        if self._fast_llm is None:
+            logger.debug(
+                f"Skipping summarization: no fast model configured (fast_model='{self.fast_model}', global_fast_model='{self.global_fast_model}')"
+            )
+            return False
+
+        # Check if input exceeds threshold
+        input_length = len(input_text)
+
+        if input_length <= self.input_threshold:
+            logger.debug(
+                f"Skipping summarization: input length {input_length} <= threshold {self.input_threshold}"
+            )
+            return False
+
+        logger.debug(
+            f"Applying summarization: input length {input_length} > threshold {self.input_threshold}"
+        )
+        return True
+
+    def transform(self, input_text: str) -> str:
+        """
+        Transform the input text by summarizing it with the fast model.
+
+        Args:
+            input_text: The tool output to summarize
+
+        Returns:
+            Summarized text
+
+        Raises:
+            TransformerError: If summarization fails
+        """
+        if self._fast_llm is None:
+            raise TransformerError("Cannot transform: no fast model configured")
+
+        try:
+            # Get the prompt to use
+            prompt = self.prompt or self.DEFAULT_PROMPT
+
+            # Construct the full prompt with the content
+            full_prompt = f"{prompt}\n\nContent to summarize:\n{input_text}"
+
+            # Perform the summarization
+            logger.debug(f"Summarizing {len(input_text)} characters with fast model")
+
+            response = self._fast_llm.completion(
+                [{"role": "user", "content": full_prompt}]
+            )
+            summarized_text = response.choices[0].message.content  # type: ignore
+
+            if not summarized_text or not summarized_text.strip():
+                raise TransformerError("Fast model returned empty summary")
+
+            logger.debug(
+                f"Summarization complete: {len(input_text)} -> {len(summarized_text)} characters"
+            )
+
+            return summarized_text.strip()
+
+        except Exception as e:
+            error_msg = f"Failed to summarize content with fast model: {e}"
+            logger.error(error_msg)
+            raise TransformerError(error_msg) from e
+
+    @property
+    def name(self) -> str:
+        """Get the transformer name."""
+        return "llm_summarize"