holmesgpt 0.13.3a0__py3-none-any.whl → 0.14.1a0__py3-none-any.whl

holmes/core/toolset_manager.py

@@ -2,7 +2,7 @@ import concurrent.futures
 import json
 import logging
 import os
-from typing import Any, List, Optional
+from typing import Any, List, Optional, TYPE_CHECKING
 
 from benedict import benedict
 from pydantic import FilePath
@@ -13,6 +13,9 @@ from holmes.core.tools import Toolset, ToolsetStatusEnum, ToolsetTag, ToolsetTyp
 from holmes.plugins.toolsets import load_builtin_toolsets, load_toolsets_from_config
 from holmes.utils.definitions import CUSTOM_TOOLSET_LOCATION
 
+if TYPE_CHECKING:
+    pass
+
 DEFAULT_TOOLSET_STATUS_LOCATION = os.path.join(config_path_dir, "toolsets_status.json")
 
 
@@ -30,6 +33,7 @@ class ToolsetManager:
         custom_toolsets: Optional[List[FilePath]] = None,
         custom_toolsets_from_cli: Optional[List[FilePath]] = None,
         toolset_status_location: Optional[FilePath] = None,
+        global_fast_model: Optional[str] = None,
     ):
         self.toolsets = toolsets
         self.toolsets = toolsets or {}
@@ -38,6 +42,7 @@ class ToolsetManager:
                 mcp_server["type"] = ToolsetType.MCP.value
         self.toolsets.update(mcp_servers or {})
         self.custom_toolsets = custom_toolsets
+        self.global_fast_model = global_fast_model
 
         if toolset_status_location is None:
             toolset_status_location = FilePath(DEFAULT_TOOLSET_STATUS_LOCATION)
@@ -118,9 +123,13 @@ class ToolsetManager:
                 if any(tag in toolset_tags for tag in toolset.tags)
             }
 
+        # Inject global fast_model into all toolsets
+        final_toolsets = list(toolsets_by_name.values())
+        self._inject_fast_model_into_transformers(final_toolsets)
+
         # check_prerequisites against each enabled toolset
         if not check_prerequisites:
-            return list(toolsets_by_name.values())
+            return final_toolsets
 
         enabled_toolsets: List[Toolset] = []
         for _, toolset in toolsets_by_name.items():
@@ -130,7 +139,7 @@ class ToolsetManager:
                 toolset.status = ToolsetStatusEnum.DISABLED
         self.check_toolset_prerequisites(enabled_toolsets)
 
-        return list(toolsets_by_name.values())
+        return final_toolsets
 
     @classmethod
     def check_toolset_prerequisites(cls, toolsets: list[Toolset]):
@@ -276,6 +285,10 @@ class ToolsetManager:
             list(toolsets_status_by_name.keys()),
             check_conflict_default=True,
         )
+
+        # Inject fast_model into CLI custom toolsets
+        self._inject_fast_model_into_transformers(custom_toolsets_from_cli)
+
         # custom toolsets from cli as experimental toolset should not override custom toolsets from config
         enabled_toolsets_from_cli: List[Toolset] = []
         for custom_toolset_from_cli in custom_toolsets_from_cli:
@@ -438,3 +451,137 @@ class ToolsetManager:
             else:
                 existing_toolsets_by_name[new_toolset.name] = new_toolset
                 existing_toolsets_by_name[new_toolset.name] = new_toolset
+
+    def _inject_fast_model_into_transformers(self, toolsets: List[Toolset]) -> None:
+        """
+        Inject global fast_model setting into all llm_summarize transformers that don't already have fast_model.
+        This ensures --fast-model reaches all tools regardless of toolset-level transformer configuration.
+
+        IMPORTANT: This also forces recreation of transformer instances since they may already be created.
+        """
+        import logging
+        from holmes.core.transformers import registry
+
+        logger = logging.getLogger(__name__)
+
+        logger.info(
+            f"Starting fast_model injection. global_fast_model={self.global_fast_model}"
+        )
+
+        if not self.global_fast_model:
+            logger.info("No global_fast_model configured, skipping injection")
+            return
+
+        injected_count = 0
+        toolset_count = 0
+
+        for toolset in toolsets:
+            toolset_count += 1
+            toolset_injected = 0
+            logger.debug(
+                f"Processing toolset '{toolset.name}', has toolset transformers: {toolset.transformers is not None}"
+            )
+
+            # Inject into toolset-level transformers
+            if toolset.transformers:
+                logger.debug(
+                    f"Toolset '{toolset.name}' has {len(toolset.transformers)} toolset-level transformers"
+                )
+                for transformer in toolset.transformers:
+                    logger.debug(
+                        f"  Toolset transformer: name='{transformer.name}', config keys={list(transformer.config.keys())}"
+                    )
+                    if (
+                        transformer.name == "llm_summarize"
+                        and "fast_model" not in transformer.config
+                    ):
+                        transformer.config["global_fast_model"] = self.global_fast_model
+                        injected_count += 1
+                        toolset_injected += 1
+                        logger.info(
+                            f"  ✓ Injected global_fast_model into toolset '{toolset.name}' transformer"
+                        )
+                    elif transformer.name == "llm_summarize":
+                        logger.debug(
+                            f"  - Toolset transformer already has fast_model: {transformer.config.get('fast_model')}"
+                        )
+            else:
+                logger.debug(
+                    f"Toolset '{toolset.name}' has no toolset-level transformers"
+                )
+
+            # Inject into tool-level transformers
+            if hasattr(toolset, "tools") and toolset.tools:
+                logger.debug(f"Toolset '{toolset.name}' has {len(toolset.tools)} tools")
+                for tool in toolset.tools:
+                    logger.debug(
+                        f"  Processing tool '{tool.name}', has transformers: {tool.transformers is not None}"
+                    )
+                    if tool.transformers:
+                        logger.debug(
+                            f"    Tool '{tool.name}' has {len(tool.transformers)} transformers"
+                        )
+                        tool_updated = False
+                        for transformer in tool.transformers:
+                            logger.debug(
+                                f"      Tool transformer: name='{transformer.name}', config keys={list(transformer.config.keys())}"
+                            )
+                            if (
+                                transformer.name == "llm_summarize"
+                                and "fast_model" not in transformer.config
+                            ):
+                                transformer.config["global_fast_model"] = (
+                                    self.global_fast_model
+                                )
+                                injected_count += 1
+                                toolset_injected += 1
+                                tool_updated = True
+                                logger.info(
+                                    f"      ✓ Injected global_fast_model into tool '{tool.name}' transformer"
+                                )
+                            elif transformer.name == "llm_summarize":
+                                logger.debug(
+                                    f"      - Tool transformer already has fast_model: {transformer.config.get('fast_model')}"
+                                )
+
+                        # CRITICAL: Force recreation of transformer instances if we updated the config
+                        if tool_updated:
+                            logger.info(
+                                f"      🔄 Recreating transformer instances for tool '{tool.name}' after injection"
+                            )
+                            if tool.transformers:
+                                tool._transformer_instances = []
+                                for transformer in tool.transformers:
+                                    if not transformer:
+                                        continue
+                                    try:
+                                        # Create transformer instance with updated config
+                                        transformer_instance = (
+                                            registry.create_transformer(
+                                                transformer.name, transformer.config
+                                            )
+                                        )
+                                        tool._transformer_instances.append(
+                                            transformer_instance
+                                        )
+                                        logger.debug(
+                                            f"        Recreated transformer '{transformer.name}' for tool '{tool.name}' with config: {transformer.config}"
+                                        )
+                                    except Exception as e:
+                                        logger.warning(
+                                            f"        Failed to recreate transformer '{transformer.name}' for tool '{tool.name}': {e}"
+                                        )
+                                        continue
+                    else:
+                        logger.debug(f"    Tool '{tool.name}' has no transformers")
+            else:
+                logger.debug(f"Toolset '{toolset.name}' has no tools")
+
+            if toolset_injected > 0:
+                logger.info(
+                    f"Toolset '{toolset.name}': injected into {toolset_injected} transformers"
+                )
+
+        logger.info(
+            f"Fast_model injection complete: {injected_count} transformers updated across {toolset_count} toolsets"
+        )

holmes/core/tracing.py

@@ -101,7 +101,7 @@ class SpanType(Enum):
 class DummySpan:
     """A no-op span implementation for when tracing is disabled."""
 
-    def start_span(self, name: str, span_type=None, **kwargs):
+    def start_span(self, name: Optional[str] = None, span_type=None, **kwargs):
         return DummySpan()
 
     def log(self, *args, **kwargs):
@@ -110,6 +110,11 @@ class DummySpan:
     def end(self):
         pass
 
+    def set_attributes(
+        self, name: Optional[str] = None, type=None, span_attributes=None
+    ) -> None:
+        pass
+
     def __enter__(self):
         return self
 

holmes/core/transformers/__init__.py

@@ -0,0 +1,23 @@
+"""
+Transformer system for processing tool outputs.
+
+This module provides the infrastructure for transforming tool outputs
+before they are passed to the LLM for analysis.
+"""
+
+from .base import BaseTransformer, TransformerError
+from .registry import TransformerRegistry, registry
+from .llm_summarize import LLMSummarizeTransformer
+from .transformer import Transformer
+
+# Register built-in transformers
+registry.register(LLMSummarizeTransformer)
+
+__all__ = [
+    "BaseTransformer",
+    "TransformerError",
+    "TransformerRegistry",
+    "registry",
+    "LLMSummarizeTransformer",
+    "Transformer",
+]

holmes/core/transformers/base.py

@@ -0,0 +1,62 @@
+"""
+Base transformer abstract class for tool output transformation.
+"""
+
+__all__ = ["BaseTransformer", "TransformerError"]
+
+from abc import ABC, abstractmethod
+from pydantic import BaseModel
+
+
+class TransformerError(Exception):
+    """Exception raised when transformer operations fail."""
+
+    pass
+
+
+class BaseTransformer(BaseModel, ABC):
+    """
+    Abstract base class for all tool output transformers.
+
+    Transformers process tool outputs before they are returned to the LLM,
+    enabling operations like summarization, filtering, or format conversion.
+    """
+
+    @abstractmethod
+    def transform(self, input_text: str) -> str:
+        """
+        Transform the input text and return the transformed output.
+
+        Args:
+            input_text: The raw tool output to transform
+
+        Returns:
+            The transformed output text
+
+        Raises:
+            TransformerError: If transformation fails
+        """
+        pass
+
+    @abstractmethod
+    def should_apply(self, input_text: str) -> bool:
+        """
+        Determine whether this transformer should be applied to the input.
+
+        Args:
+            input_text: The raw tool output to check
+
+        Returns:
+            True if the transformer should be applied, False otherwise
+        """
+        pass
+
+    @property
+    def name(self) -> str:
+        """
+        Get the transformer name.
+
+        Returns:
+            The transformer name (class name by default)
+        """
+        return self.__class__.__name__

holmes/core/transformers/llm_summarize.py

@@ -0,0 +1,174 @@
+"""
+LLM Summarize Transformer for fast model summarization of large tool outputs.
+"""
+
+import logging
+from typing import Optional, ClassVar
+from pydantic import Field, PrivateAttr, StrictStr
+
+from .base import BaseTransformer, TransformerError
+from ..llm import DefaultLLM, LLM
+
+logger = logging.getLogger(__name__)
+
+
+class LLMSummarizeTransformer(BaseTransformer):
+    """
+    Transformer that uses a fast LLM model to summarize large tool outputs.
+
+    This transformer applies summarization when:
+    1. A fast model is available
+    2. The input length exceeds the configured threshold
+
+    Configuration options:
+    - input_threshold: Minimum input length to trigger summarization (default: 1000)
+    - prompt: Custom prompt template for summarization (optional)
+    - fast_model: Fast model name for summarization (e.g., "gpt-4o-mini")
+    - api_key: API key for the fast model (optional, uses default if not provided)
+    """
+
+    DEFAULT_PROMPT: ClassVar[str] = """Summarize this operational data focusing on:
+- What needs attention or immediate action
+- Group similar entries into a single line and description
+- Make sure to mention outliers, errors, and non-standard patterns
+- List normal/healthy patterns as aggregate descriptions
+- When listing problematic entries, also try to use aggregate descriptions when possible
+- When possible, mention exact keywords, IDs, or patterns so the user can filter/search the original data and drill down on the parts they care about (extraction over abstraction)"""
+
+    # Pydantic fields with validation
+    input_threshold: int = Field(
+        default=1000, ge=0, description="Minimum input length to trigger summarization"
+    )
+    prompt: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Custom prompt template for summarization",
+    )
+    fast_model: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Fast model name for summarization (e.g., 'gpt-4o-mini')",
+    )
+    global_fast_model: Optional[StrictStr] = Field(
+        default=None,
+        min_length=1,
+        description="Global fast model name fallback when fast_model is not set",
+    )
+    api_key: Optional[str] = Field(
+        default=None,
+        description="API key for the fast model (optional, uses default if not provided)",
+    )
+
+    # Private attribute for the LLM instance (not serialized)
+    _fast_llm: Optional[LLM] = PrivateAttr(default=None)
+
+    def model_post_init(self, __context) -> None:
+        """Initialize the fast LLM instance after model validation."""
+        logger = logging.getLogger(__name__)
+
+        self._fast_llm = None
+
+        # Determine which fast model to use: fast_model takes precedence over global_fast_model
+        effective_fast_model = self.fast_model or self.global_fast_model
+
+        logger.debug(
+            f"LLMSummarizeTransformer initialization: fast_model='{self.fast_model}', global_fast_model='{self.global_fast_model}', effective='{effective_fast_model}'"
+        )
+
+        # Create fast LLM instance if a fast model is available
+        if effective_fast_model:
+            try:
+                self._fast_llm = DefaultLLM(effective_fast_model, self.api_key)
+                logger.info(
+                    f"Created fast LLM instance with model: {effective_fast_model}"
+                )
+            except Exception as e:
+                logger.warning(f"Failed to create fast LLM instance: {e}")
+                self._fast_llm = None
+        else:
+            logger.debug(
+                "No fast model configured (neither fast_model nor global_fast_model)"
+            )
+
+    def should_apply(self, input_text: str) -> bool:
+        """
+        Determine if summarization should be applied to the input.
+
+        Args:
+            input_text: The tool output to check
+
+        Returns:
+            True if summarization should be applied, False otherwise
+        """
+        logger = logging.getLogger(__name__)
+
+        # Skip if no fast model is configured
+        if self._fast_llm is None:
+            logger.debug(
+                f"Skipping summarization: no fast model configured (fast_model='{self.fast_model}', global_fast_model='{self.global_fast_model}')"
+            )
+            return False
+
+        # Check if input exceeds threshold
+        input_length = len(input_text)
+
+        if input_length <= self.input_threshold:
+            logger.debug(
+                f"Skipping summarization: input length {input_length} <= threshold {self.input_threshold}"
+            )
+            return False
+
+        logger.debug(
+            f"Applying summarization: input length {input_length} > threshold {self.input_threshold}"
+        )
+        return True
+
+    def transform(self, input_text: str) -> str:
+        """
+        Transform the input text by summarizing it with the fast model.
+
+        Args:
+            input_text: The tool output to summarize
+
+        Returns:
+            Summarized text
+
+        Raises:
+            TransformerError: If summarization fails
+        """
+        if self._fast_llm is None:
+            raise TransformerError("Cannot transform: no fast model configured")
+
+        try:
+            # Get the prompt to use
+            prompt = self.prompt or self.DEFAULT_PROMPT
+
+            # Construct the full prompt with the content
+            full_prompt = f"{prompt}\n\nContent to summarize:\n{input_text}"
+
+            # Perform the summarization
+            logger.debug(f"Summarizing {len(input_text)} characters with fast model")
+
+            response = self._fast_llm.completion(
+                [{"role": "user", "content": full_prompt}]
+            )
+            summarized_text = response.choices[0].message.content  # type: ignore
+
+            if not summarized_text or not summarized_text.strip():
+                raise TransformerError("Fast model returned empty summary")
+
+            logger.debug(
+                f"Summarization complete: {len(input_text)} -> {len(summarized_text)} characters"
+            )
+
+            return summarized_text.strip()
+
+        except Exception as e:
+            error_msg = f"Failed to summarize content with fast model: {e}"
+            logger.error(error_msg)
+            raise TransformerError(error_msg) from e
+
+    @property
+    def name(self) -> str:
+        """Get the transformer name."""
+        return "llm_summarize"

holmes/core/transformers/registry.py

@@ -0,0 +1,122 @@
+"""
+Transformer registry for managing available transformers.
+"""
+
+from typing import Dict, Type, Optional, Any, List
+from .base import BaseTransformer, TransformerError
+
+
+class TransformerRegistry:
+    """
+    Registry for managing transformer types and creating transformer instances.
+
+    This registry provides a centralized way to register transformer classes
+    and create instances based on configuration.
+    """
+
+    def __init__(self):
+        self._transformers: Dict[str, Type[BaseTransformer]] = {}
+
+    def register(self, transformer_class: Type[BaseTransformer]) -> None:
+        """
+        Register a transformer class, using the transformer's name property.
+
+        Args:
+            transformer_class: The transformer class to register
+
+        Raises:
+            ValueError: If name is already registered or transformer_class is invalid
+        """
+        if not issubclass(transformer_class, BaseTransformer):
+            raise ValueError(
+                f"Transformer class must inherit from BaseTransformer, got {transformer_class}"
+            )
+
+        # Get name from the transformer class
+        try:
+            temp_instance = transformer_class()
+            name = temp_instance.name
+        except Exception:
+            # Fallback to class name if instantiation fails
+            name = transformer_class.__name__
+
+        if name in self._transformers:
+            raise ValueError(f"Transformer '{name}' is already registered")
+
+        self._transformers[name] = transformer_class
+
+    def unregister(self, name: str) -> None:
+        """
+        Unregister a transformer by name.
+
+        Args:
+            name: The name of the transformer to unregister
+
+        Raises:
+            KeyError: If transformer name is not registered
+        """
+        if name not in self._transformers:
+            raise KeyError(f"Transformer '{name}' is not registered")
+
+        del self._transformers[name]
+
+    def create_transformer(
+        self, name: str, config: Optional[Dict[str, Any]] = None
+    ) -> BaseTransformer:
+        """
+        Create a transformer instance by name.
+
+        Args:
+            name: The name of the transformer to create
+            config: Optional configuration for the transformer
+
+        Returns:
+            A new transformer instance
+
+        Raises:
+            KeyError: If transformer name is not registered
+            TransformerError: If transformer creation fails
+        """
+        if name not in self._transformers:
+            raise KeyError(f"Transformer '{name}' is not registered")
+
+        transformer_class = self._transformers[name]
+
+        try:
+            # Handle both old-style dict config and new Pydantic models
+            if config is None:
+                return transformer_class()
+            else:
+                # For Pydantic models, pass config as keyword arguments
+                return transformer_class(**config)
+        except Exception as e:
+            raise TransformerError(f"Failed to create transformer '{name}': {e}") from e
+
+    def is_registered(self, name: str) -> bool:
+        """
+        Check if a transformer is registered.
+
+        Args:
+            name: The name to check
+
+        Returns:
+            True if the transformer is registered, False otherwise
+        """
+        return name in self._transformers
+
+    def list_transformers(self) -> List[str]:
+        """
+        Get a list of all registered transformer names.
+
+        Returns:
+            List of registered transformer names
+        """
+        return list(self._transformers.keys())
+
+    def clear(self) -> None:
+        """Clear all registered transformers."""
+        self._transformers.clear()
+
+
+# Global transformer registry instance
+registry = TransformerRegistry()

holmes/core/transformers/transformer.py

@@ -0,0 +1,31 @@
+"""
+Configuration class for tool transformers.
+"""
+
+import logging
+from typing import Any, Dict
+from pydantic import BaseModel, Field, model_validator
+
+from .registry import registry
+
+
+class Transformer(BaseModel):
+    """
+    Configuration for a tool transformer.
+
+    Each transformer config specifies a transformer type and its parameters.
+    This replaces the previous dict-based configuration with proper type safety.
+    """
+
+    name: str = Field(description="Name of the transformer (e.g., 'llm_summarize')")
+    config: Dict[str, Any] = Field(
+        default_factory=dict, description="Configuration parameters for the transformer"
+    )
+
+    @model_validator(mode="after")
+    def validate_transformer(self):
+        """Validate that the transformer name is known to the registry."""
+        if not registry.is_registered(self.name):
+            # Log warning but don't fail validation - allows for graceful degradation
+            logging.warning(f"Transformer '{self.name}' is not registered")
+        return self

holmes/main.py

@@ -76,6 +76,9 @@ opt_api_key: Optional[str] = typer.Option(
     help="API key to use for the LLM (if not given, uses environment variables OPENAI_API_KEY or AZURE_API_KEY)",
 )
 opt_model: Optional[str] = typer.Option(None, help="Model to use for the LLM")
+opt_fast_model: Optional[str] = typer.Option(
+    None, help="Optional fast model for summarization tasks"
+)
 opt_config_file: Optional[Path] = typer.Option(
     DEFAULT_CONFIG_LOCATION,  # type: ignore
     "--config",
@@ -177,6 +180,7 @@ def ask(
     # common options
     api_key: Optional[str] = opt_api_key,
     model: Optional[str] = opt_model,
+    fast_model: Optional[str] = opt_fast_model,
     config_file: Optional[Path] = opt_config_file,
     custom_toolsets: Optional[List[Path]] = opt_custom_toolsets,
     max_steps: Optional[int] = opt_max_steps,
@@ -244,6 +248,7 @@ def ask(
         config_file,
         api_key=api_key,
         model=model,
+        fast_model=fast_model,
         max_steps=max_steps,
         custom_toolsets_from_cli=custom_toolsets,
         slack_token=slack_token,