holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/toolsets/azure_sql/tools/get_top_log_io_queries.py

@@ -0,0 +1,180 @@
+import logging
+from typing import Dict, List, Tuple
+
+from holmes.core.tools import StructuredToolResult, ToolParameter, ToolResultStatus
+from holmes.plugins.toolsets.azure_sql.azure_base_toolset import (
+    BaseAzureSQLTool,
+    BaseAzureSQLToolset,
+    AzureSQLDatabaseConfig,
+)
+from holmes.plugins.toolsets.azure_sql.apis.azure_sql_api import AzureSQLAPIClient
+from holmes.plugins.toolsets.azure_sql.utils import format_timing
+
+
+class GetTopLogIOQueries(BaseAzureSQLTool):
+    def __init__(self, toolset: "BaseAzureSQLToolset"):
+        super().__init__(
+            name="get_top_log_io_queries",
+            description="Identifies queries consuming the most transaction log I/O from Query Store. Use this to find queries causing transaction log performance issues and write-heavy workload problems.",
+            parameters={
+                "top_count": ToolParameter(
+                    description="Number of top queries to return. Use 15 for detailed analysis, 5-10 for quick overview (default: 15)",
+                    type="integer",
+                    required=False,
+                ),
+                "hours_back": ToolParameter(
+                    description="Time window for analysis in hours. Use 2 for recent issues, 24+ for trend analysis (default: 2)",
+                    type="integer",
+                    required=False,
+                ),
+            },
+            toolset=toolset,
+        )
+
+    def _format_log_io_queries_report(
+        self,
+        queries: List[Dict],
+        db_config: AzureSQLDatabaseConfig,
+        top_count: int,
+        hours_back: int,
+    ) -> str:
+        """Format the log I/O queries data into a readable report."""
+        report_sections = []
+
+        # Header
+        report_sections.append("# Top Log I/O Consuming Queries Report")
+        report_sections.append(f"**Database:** {db_config.database_name}")
+        report_sections.append(f"**Server:** {db_config.server_name}")
+        report_sections.append(f"**Analysis Period:** Last {hours_back} hours")
+        report_sections.append(f"**Top Queries:** {top_count}")
+        report_sections.append("")
+
+        if not queries:
+            report_sections.append("No queries found for the specified time period.")
+            return "\n".join(report_sections)
+
+        # Summary
+        total_log_writes = sum(float(q.get("total_log_bytes_used", 0)) for q in queries)
+        total_executions = sum(int(q.get("execution_count", 0)) for q in queries)
+
+        report_sections.append("## Summary")
+        report_sections.append(f"- **Total Queries Analyzed:** {len(queries)}")
+        report_sections.append(
+            f"- **Total Log Bytes Used:** {total_log_writes:,.0f} bytes"
+        )
+        report_sections.append(f"- **Total Executions:** {total_executions:,}")
+        report_sections.append("")
+
+        # Query Details
+        report_sections.append("## Query Details")
+
+        for i, query in enumerate(queries[:top_count], 1):
+            avg_log_bytes = float(query.get("avg_log_bytes_used", 0))
+            execution_count = int(query.get("execution_count", 0))
+            total_log_bytes = float(query.get("total_log_bytes_used", 0))
+            max_log_bytes = float(query.get("max_log_bytes_used", 0))
+            avg_cpu = float(query.get("avg_cpu_time", 0))
+            avg_duration = float(query.get("avg_duration", 0))
+            query_text = query.get("query_sql_text", "N/A")
+            last_execution = query.get("last_execution_time", "N/A")
+
+            # Truncate long queries
+            if len(query_text) > 200:
+                query_text = query_text[:200] + "..."
+
+            report_sections.append(f"### Query #{i}")
+            report_sections.append(
+                f"- **Average Log Bytes Used:** {avg_log_bytes:,.0f} bytes"
+            )
+            report_sections.append(
+                f"- **Total Log Bytes Used:** {total_log_bytes:,.0f} bytes"
+            )
+            report_sections.append(
+                f"- **Max Log Bytes Used:** {max_log_bytes:,.0f} bytes"
+            )
+            report_sections.append(f"- **Execution Count:** {execution_count:,}")
+            report_sections.append(f"- **Average CPU Time:** {format_timing(avg_cpu)}")
+            report_sections.append(
+                f"- **Average Duration:** {format_timing(avg_duration)}"
+            )
+            report_sections.append(f"- **Last Execution:** {last_execution}")
+            report_sections.append("- **Query Text:**")
+            report_sections.append("```sql")
+            report_sections.append(query_text)
+            report_sections.append("```")
+            report_sections.append("")
+
+        return "\n".join(report_sections)
+
+    def _invoke(self, params: Dict) -> StructuredToolResult:
+        try:
+            top_count = params.get("top_count", 15)
+            hours_back = params.get("hours_back", 2)
+
+            db_config = self.toolset.database_config()
+            client = self.toolset.api_client()
+
+            # Get top log I/O queries
+            queries = client.get_top_log_io_queries(
+                db_config.subscription_id,
+                db_config.resource_group,
+                db_config.server_name,
+                db_config.database_name,
+                top_count,
+                hours_back,
+            )
+
+            # Format the report
+            report_text = self._format_log_io_queries_report(
+                queries, db_config, top_count, hours_back
+            )
+
+            return StructuredToolResult(
+                status=ToolResultStatus.SUCCESS,
+                data=report_text,
+                params=params,
+            )
+        except Exception as e:
+            error_msg = f"Failed to get top log I/O queries: {str(e)}"
+            logging.error(error_msg)
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=error_msg,
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict) -> str:
+        db_config = self.toolset.database_config()
+        return f"Fetch top log I/O consuming queries for database {db_config.server_name}/{db_config.database_name}"
+
+    @staticmethod
+    def validate_config(
+        api_client: AzureSQLAPIClient, database_config: AzureSQLDatabaseConfig
+    ) -> Tuple[bool, str]:
+        errors = []
+
+        try:
+            # Test direct database connection for Query Store access
+            test_query = (
+                "SELECT TOP 1 query_id FROM sys.query_store_query WHERE query_id > 0"
+            )
+            api_client.execute_query(
+                database_config.server_name, database_config.database_name, test_query
+            )
+        except Exception as e:
+            error_msg = str(e)
+            if (
+                "login failed" in error_msg.lower()
+                or "authentication" in error_msg.lower()
+            ):
+                errors.append(f"Database authentication failed: {error_msg}")
+            elif "permission" in error_msg.lower() or "denied" in error_msg.lower():
+                errors.append(f"Query Store access denied: {error_msg}")
+            elif "query store" in error_msg.lower():
+                errors.append(f"Query Store not available or disabled: {error_msg}")
+            else:
+                errors.append(f"Database connection failed: {error_msg}")
+
+        if errors:
+            return False, "\n".join(errors)
+        return True, ""

holmes/plugins/toolsets/azure_sql/utils.py

@@ -0,0 +1,83 @@
+"""
+Shared utility functions for Azure SQL toolset.
+"""
+
+
+def format_timing(microseconds: float) -> str:
+    """Format timing values with appropriate units.
+
+    Args:
+        microseconds: Time value in microseconds
+
+    Returns:
+        Formatted string with appropriate units (s, ms, or μs)
+    """
+    if microseconds >= 1_000_000:  # >= 1 second
+        return f"{microseconds / 1_000_000:.2f} s"
+    elif microseconds >= 1_000:  # >= 1 millisecond
+        return f"{microseconds / 1_000:.2f} ms"
+    else:  # < 1 millisecond
+        return f"{microseconds:.0f} μs"
+
+
+def format_size_bytes(bytes_value: float) -> str:
+    """Format byte values with appropriate units.
+
+    Args:
+        bytes_value: Size value in bytes
+
+    Returns:
+        Formatted string with appropriate units (GB, MB, KB, or B)
+    """
+    if bytes_value >= 1_073_741_824:  # >= 1 GB
+        return f"{bytes_value / 1_073_741_824:.2f} GB"
+    elif bytes_value >= 1_048_576:  # >= 1 MB
+        return f"{bytes_value / 1_048_576:.2f} MB"
+    elif bytes_value >= 1_024:  # >= 1 KB
+        return f"{bytes_value / 1_024:.2f} KB"
+    else:  # < 1 KB
+        return f"{bytes_value:.0f} B"
+
+
+def format_percentage(value: float, decimal_places: int = 2) -> str:
+    """Format percentage values.
+
+    Args:
+        value: Percentage value (0-100)
+        decimal_places: Number of decimal places to show
+
+    Returns:
+        Formatted percentage string
+    """
+    return f"{value:.{decimal_places}f}%"
+
+
+def safe_divide(numerator: float, denominator: float, default: float = 0.0) -> float:
+    """Safely divide two numbers, handling division by zero.
+
+    Args:
+        numerator: The dividend
+        denominator: The divisor
+        default: Value to return if denominator is zero
+
+    Returns:
+        Division result or default if denominator is zero
+    """
+    if denominator == 0 or denominator is None:
+        return default
+    return numerator / denominator
+
+
+def truncate_text(text: str, max_length: int = 100) -> str:
+    """Truncate text to a maximum length with ellipsis.
+
+    Args:
+        text: Text to truncate
+        max_length: Maximum length including ellipsis
+
+    Returns:
+        Truncated text with ellipsis if needed
+    """
+    if not text or len(text) <= max_length:
+        return text
+    return text[: max_length - 3] + "..."

holmes/plugins/toolsets/bash/bash_instructions.jinja2

@@ -0,0 +1,14 @@
+# Bash commands
+
+The tool `run_bash_command` allows you to run many kubectl commands:
+
+- `kubectl get ...`
+- `kubectl describe ...`
+- `kubectl events ...`
+- `kubectl top ...`
+
+It is also possible to combine `kubectl` with `grep`. For example:
+- `kubectl get pods | grep holmes`
+
+The tool `kubectl_run_image` will run an image:
+- `kubectl run <name> --image=<image> --rm --attach --restart=Never --i --tty -- <command>`

holmes/plugins/toolsets/bash/bash_toolset.py

@@ -0,0 +1,208 @@
+import argparse
+import logging
+import os
+import random
+import re
+import string
+from typing import Dict, Any, Optional
+
+
+from holmes.core.tools import (
+    CallablePrerequisite,
+    StructuredToolResult,
+    Tool,
+    ToolParameter,
+    ToolResultStatus,
+    Toolset,
+    ToolsetTag,
+)
+from holmes.plugins.toolsets.bash.common.bash import execute_bash_command
+from holmes.plugins.toolsets.bash.common.config import BashExecutorConfig
+from holmes.plugins.toolsets.bash.kubectl.constants import SAFE_NAMESPACE_PATTERN
+from holmes.plugins.toolsets.bash.kubectl.kubectl_run import validate_image_and_commands
+from holmes.plugins.toolsets.bash.parse_command import make_command_safe
+from holmes.plugins.toolsets.utils import get_param_or_raise
+
+
+class BaseBashExecutorToolset(Toolset):
+    config: Optional[BashExecutorConfig] = None
+
+    def get_example_config(self):
+        example_config = BashExecutorConfig()
+        return example_config.model_dump()
+
+
+class BaseBashTool(Tool):
+    toolset: BaseBashExecutorToolset
+
+
+class KubectlRunImageCommand(BaseBashTool):
+    def __init__(self, toolset: BaseBashExecutorToolset):
+        super().__init__(
+            name="kubectl_run_image",
+            description=(
+                "Executes `kubectl run <name> --image=<image> ... -- <command>` return the result"
+            ),
+            parameters={
+                "image": ToolParameter(
+                    description="The image to run",
+                    type="string",
+                    required=True,
+                ),
+                "command": ToolParameter(
+                    description="The command to execute on the deployed pod",
+                    type="string",
+                    required=True,
+                ),
+                "namespace": ToolParameter(
+                    description="The namespace in which to deploy the temporary pod",
+                    type="string",
+                    required=False,
+                ),
+                "timeout": ToolParameter(
+                    description=(
+                        "Optional timeout in seconds for the command execution. "
+                        "Defaults to 60s."
+                    ),
+                    type="integer",
+                    required=False,
+                ),
+            },
+            toolset=toolset,
+        )
+
+    def _build_kubectl_command(self, params: dict, pod_name: str) -> str:
+        namespace = params.get("namespace", "default")
+        image = get_param_or_raise(params, "image")
+        command_str = get_param_or_raise(params, "command")
+        return f"kubectl run {pod_name} --image={image} --namespace={namespace} --rm --attach --restart=Never -i -- {command_str}"
+
+    def _invoke(self, params: Dict[str, Any]) -> StructuredToolResult:
+        timeout = params.get("timeout", 60)
+
+        image = get_param_or_raise(params, "image")
+        command_str = get_param_or_raise(params, "command")
+
+        namespace = params.get("namespace")
+
+        if namespace and not re.match(SAFE_NAMESPACE_PATTERN, namespace):
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"Error: The namespace is invalid. Valid namespaces must match the following regexp: {SAFE_NAMESPACE_PATTERN}",
+                params=params,
+            )
+
+        validate_image_and_commands(
+            image=image, container_command=command_str, config=self.toolset.config
+        )
+
+        pod_name = (
+            "holmesgpt-debug-pod-"
+            + "".join(random.choices(string.ascii_letters, k=8)).lower()
+        )
+        full_kubectl_command = self._build_kubectl_command(params, pod_name)
+        return execute_bash_command(
+            cmd=full_kubectl_command, timeout=timeout, params=params
+        )
+
+    def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str:
+        return self._build_kubectl_command(params, "<pod_name>")
+
+
+class RunBashCommand(BaseBashTool):
+    def __init__(self, toolset: BaseBashExecutorToolset):
+        super().__init__(
+            name="run_bash_command",
+            description=(
+                "Executes a given bash command and returns its standard output, "
+                "standard error, and exit code."
+                "The command is executed via 'bash -c \"<command>\"'."
+                "Only some commands are allowed."
+            ),
+            parameters={
+                "command": ToolParameter(
+                    description="The bash command string to execute.",
+                    type="string",
+                    required=True,
+                ),
+                "timeout": ToolParameter(
+                    description=(
+                        "Optional timeout in seconds for the command execution. "
+                        "Defaults to 60s."
+                    ),
+                    type="integer",
+                    required=False,
+                ),
+            },
+            toolset=toolset,
+        )
+
+    def _invoke(self, params: Dict[str, Any]) -> StructuredToolResult:
+        command_str = params.get("command")
+        timeout = params.get("timeout", 60)
+
+        if not command_str:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error="The 'command' parameter is required and was not provided.",
+                params=params,
+            )
+
+        if not isinstance(command_str, str):
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"The 'command' parameter must be a string, got {type(command_str).__name__}.",
+                params=params,
+            )
+        try:
+            safe_command_str = make_command_safe(command_str, self.toolset.config)
+            return execute_bash_command(
+                cmd=safe_command_str, timeout=timeout, params=params
+            )
+        except (argparse.ArgumentError, ValueError) as e:
+            logging.info(f"Refusing LLM tool call {command_str}", exc_info=True)
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"Refusing to execute bash command. Only some commands are supported and this is likely because requested command is unsupported. Error: {str(e)}",
+                params=params,
+            )
+
+    def get_parameterized_one_liner(self, params: Dict[str, Any]) -> str:
+        command = params.get("command", "N/A")
+        display_command = command[:200] + "..." if len(command) > 200 else command
+        return display_command
+
+
+class BashExecutorToolset(BaseBashExecutorToolset):
+    def __init__(self):
+        super().__init__(
+            name="bash",
+            enabled=False,
+            description=(
+                "Toolset for executing arbitrary bash commands on the system where Holmes is running. "
+                "WARNING: This toolset provides powerful capabilities and should be "
+                "enabled and used with extreme caution due to significant security risks. "
+                "Ensure that only trusted users have access to this tool."
+            ),
+            docs_url="",  # TODO: Add relevant documentation URL
+            icon_url="https://upload.wikimedia.org/wikipedia/commons/thumb/4/4b/Bash_Logo_Colored.svg/120px-Bash_Logo_Colored.svg.png",  # Example Bash icon
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            tools=[RunBashCommand(self), KubectlRunImageCommand(self)],
+            tags=[ToolsetTag.CORE],
+            is_default=False,
+        )
+
+        self._reload_llm_instructions()
+
+    def _reload_llm_instructions(self):
+        template_file_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "bash_instructions.jinja2")
+        )
+        self._load_llm_instructions(jinja_template=f"file://{template_file_path}")
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> tuple[bool, str]:
+        if config:
+            self.config = BashExecutorConfig(**config)
+        else:
+            self.config = BashExecutorConfig()
+        return True, ""

holmes/plugins/toolsets/bash/common/bash.py

@@ -0,0 +1,52 @@
+import subprocess
+from holmes.core.tools import StructuredToolResult, ToolResultStatus
+
+
+def execute_bash_command(cmd: str, timeout: int, params: dict) -> StructuredToolResult:
+    try:
+        process = subprocess.run(
+            cmd,
+            shell=True,
+            executable="/bin/bash",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+
+        stdout = process.stdout.strip() if process.stdout else ""
+        result_data = f"{cmd}\n" f"{stdout}"
+
+        status = ToolResultStatus.ERROR
+        if process.returncode == 0 and stdout:
+            status = ToolResultStatus.SUCCESS
+        elif not stdout:
+            status = ToolResultStatus.NO_DATA
+
+        return StructuredToolResult(
+            status=status,
+            data=result_data,
+            params=params,
+            invocation=cmd,
+            return_code=process.returncode,
+        )
+    except subprocess.TimeoutExpired:
+        return StructuredToolResult(
+            status=ToolResultStatus.ERROR,
+            error=f"Error: Command '{cmd}' timed out after {timeout} seconds.",
+            params=params,
+        )
+    except FileNotFoundError:
+        # This might occur if /bin/bash is not found, or if shell=False and command is not found
+        return StructuredToolResult(
+            status=ToolResultStatus.ERROR,
+            error="Error: Bash executable or command not found. Ensure bash is installed and the command is valid.",
+            params=params,
+        )
+    except Exception as e:
+        return StructuredToolResult(
+            status=ToolResultStatus.ERROR,
+            error=f"Error executing command '{cmd}': {str(e)}",
+            params=params,
+        )

holmes/plugins/toolsets/bash/common/config.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+
+class KubectlImageConfig(BaseModel):
+    image: str
+    allowed_commands: list[str]
+
+
+class KubectlConfig(BaseModel):
+    allowed_images: list[KubectlImageConfig] = []
+
+
+class BashExecutorConfig(BaseModel):
+    kubectl: KubectlConfig = KubectlConfig()

holmes/plugins/toolsets/bash/common/stringify.py

@@ -0,0 +1,25 @@
+import shlex
+
+SAFE_SHELL_CHARS = frozenset(".-_=/,:")
+
+
+def escape_shell_args(args: list[str]) -> list[str]:
+    """
+    Escape shell arguments to prevent injection.
+    Uses shlex.quote for safe shell argument quoting.
+    """
+    escaped_args = []
+
+    for arg in args:
+        # If argument is safe (contains only alphanumeric, hyphens, dots, underscores, equals, slash, comma, colon)
+        # no escaping needed
+        if arg and all(c.isalnum() or c in SAFE_SHELL_CHARS for c in arg):
+            escaped_args.append(arg)
+        # If argument starts with -- or - (flag), no escaping needed
+        elif arg.startswith("-"):
+            escaped_args.append(arg)
+        # For everything else, use shlex.quote for proper escaping
+        else:
+            escaped_args.append(shlex.quote(arg))
+
+    return escaped_args

holmes/plugins/toolsets/bash/common/validators.py

@@ -0,0 +1,24 @@
+import argparse
+import re
+from typing import Union
+
+
+def regex_validator(field_name: str, pattern: Union[str, re.Pattern]):
+    def check_regex(arg_value):
+        if not re.match(pattern, arg_value):
+            raise argparse.ArgumentTypeError(f"invalid {field_name} value")
+        return arg_value
+
+    return check_regex
+
+
+def whitelist_validator(field_name: str, whitelisted_values: set[str]):
+    def validate_value(value: str) -> str:
+        if value not in whitelisted_values:
+            whitelisted_values_str = ", ".join(whitelisted_values)
+            raise argparse.ArgumentTypeError(
+                f"Invalid {field_name} format: '{value}'. Must be one of [{whitelisted_values_str}]"
+            )
+        return value
+
+    return validate_value

holmes/plugins/toolsets/bash/grep/__init__.py

@@ -0,0 +1,52 @@
+import argparse
+import re
+from typing import Any
+
+from holmes.plugins.toolsets.bash.common.stringify import escape_shell_args
+
+MAX_GREP_SIZE = 100
+SAFE_GREP_PATTERN = re.compile(r"^[a-zA-Z0-9\-_. :*()]+$")
+
+
+def create_grep_parser(main_parser: Any):
+    parser = main_parser.add_parser(
+        "grep",
+        help="Search text patterns in files or input",
+        exit_on_error=False,
+    )
+    parser.add_argument(
+        "keyword",
+        type=lambda x: validate_grep_keyword(x),
+        help="The pattern to search for",
+    )
+    parser.add_argument(
+        "-i", "--ignore-case", action="store_true", help="Ignore case distinctions"
+    )
+
+
+def validate_grep_keyword(value: str) -> str:
+    """Validate the grep keyword parameter."""
+
+    if not value:
+        raise argparse.ArgumentTypeError("Grep keyword cannot be empty")
+
+    if not SAFE_GREP_PATTERN.match(value):
+        raise argparse.ArgumentTypeError(f"Unsafe grep keyword: {value}")
+
+    if len(value) > MAX_GREP_SIZE:
+        raise argparse.ArgumentTypeError(
+            f"Grep keyword too long. Max allowed size is {MAX_GREP_SIZE} but received {len(value)}"
+        )
+
+    return value
+
+
+def stringify_grep_command(cmd: Any) -> str:
+    """Stringify grep command."""
+    parts = ["grep"]
+
+    if cmd.ignore_case:
+        parts.append("-i")
+
+    parts.append(cmd.keyword)
+    return " ".join(escape_shell_args(parts))