holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/toolsets/bash/parse_command.py

@@ -0,0 +1,103 @@
+import argparse
+import shlex
+from typing import Any, Optional
+
+from holmes.plugins.toolsets.bash.common.config import BashExecutorConfig
+from holmes.plugins.toolsets.bash.grep import create_grep_parser, stringify_grep_command
+from holmes.plugins.toolsets.bash.kubectl import (
+    create_kubectl_parser,
+    stringify_kubectl_command,
+)
+
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Parser for commands", exit_on_error=False
+    )
+    commands_parser = parser.add_subparsers(
+        dest="cmd", required=True, help="The tool to command (e.g., kubectl)"
+    )
+
+    create_kubectl_parser(commands_parser)
+    create_grep_parser(commands_parser)
+    return parser
+
+
+def stringify_command(
+    command: Any, original_command: str, config: Optional[BashExecutorConfig]
+) -> str:
+    if command.cmd == "kubectl":
+        return stringify_kubectl_command(command, config)
+    elif command.cmd == "grep":
+        return stringify_grep_command(command)
+    else:
+        # This code path should not happen b/c the parsing of the command should catch an unsupported command
+        raise ValueError(
+            f"Unsupported command '{command.cmd}' in {original_command}. Supported commands are: kubectl, grep"
+        )
+
+
+command_parser = create_parser()
+
+
+def split_into_separate_commands(command_str: str) -> list[list[str]]:
+    """
+    Splits a single bash command into sub commands based on the pipe '|' delimiter.
+
+    Example:
+        >>> "ls -l" -> [
+            ['ls', '-l']
+        ]
+        >>> "kubectl get pods | grep holmes" -> [
+            ['kubectl', 'get', 'pods'],
+            ['grep', 'holmes']
+        ]
+    """
+    parts = shlex.split(command_str)
+    if not parts:
+        return []
+
+    commands_list: list[list[str]] = []
+    current_command: list[str] = []
+
+    for part in parts:
+        if part == "|":
+            if current_command:
+                commands_list.append(current_command)
+            current_command = []
+        elif part == "&&":
+            raise ValueError(
+                'Double ampersand "&&" is not a supported way to chain commands. Run each command separately.'
+            )
+        else:
+            current_command.append(part)
+
+    if current_command:
+        commands_list.append(current_command)
+
+    return commands_list
+
+
+def make_command_safe(command_str: str, config: Optional[BashExecutorConfig]) -> str:
+    commands = split_into_separate_commands(command_str)
+
+    try:
+        safe_commands = [
+            command_parser.parse_args(command_parts) for command_parts in commands
+        ]
+        if safe_commands and safe_commands[0].cmd == "grep":
+            raise ValueError(
+                "The command grep can only be used after another command using the pipe `|` character to connect both commands"
+            )
+        safe_commands_str = [
+            stringify_command(cmd, original_command=command_str, config=config)
+            for cmd in safe_commands
+        ]
+
+        return " | ".join(safe_commands_str)
+    except SystemExit:
+        # argparse throws a SystemExit error when it can't parse command or arguments
+        # This ideally should be captured differently by ensuring all possible args
+        # are accounted for in the implementation for each command.
+        # When falling back, we raise a generic error
+        raise ValueError("The command failed to be parsed for safety") from None

holmes/plugins/toolsets/confluence.yaml

@@ -0,0 +1,19 @@
+toolsets:
+  confluence:
+    description: "Fetch confluence pages"
+    docs_url: "https://docs.robusta.dev/master/configuration/holmesgpt/toolsets/confluence.html"
+    icon_url: "https://platform.robusta.dev/demos/confluence.svg"
+    tags:
+      - core
+    prerequisites:
+      - command: "curl --version"
+      - env:
+          - CONFLUENCE_USER
+          - CONFLUENCE_API_KEY
+          - CONFLUENCE_BASE_URL
+
+    tools:
+      - name: "fetch_confluence_url"
+        description: "Fetch a page in confluence.  Use this to fetch confluence runbooks if they are present before starting your investigation."
+        user_description: "fetch confluence page {{ confluence_page_id }}"
+        command: "curl -u ${CONFLUENCE_USER}:${CONFLUENCE_API_KEY} -X GET -H 'Content-Type: application/json' ${CONFLUENCE_BASE_URL}/wiki/rest/api/content/{{ confluence_page_id }}?expand=body.storage"

holmes/plugins/toolsets/consts.py

@@ -0,0 +1,5 @@
+TOOLSET_CONFIG_MISSING_ERROR = "The toolset is missing its configuration"
+
+STANDARD_END_DATETIME_TOOL_PARAM_DESCRIPTION = (
+    "End datetime, inclusive. Should be formatted in rfc3339. Defaults to NOW"
+)

holmes/plugins/toolsets/coralogix/api.py

@@ -0,0 +1,158 @@
+from enum import Enum
+import logging
+from typing import Any, Tuple
+from urllib.parse import urljoin
+
+import requests  # type: ignore
+
+from holmes.plugins.toolsets.coralogix.utils import (
+    CoralogixConfig,
+    CoralogixQueryResult,
+    merge_log_results,
+    parse_logs,
+    CoralogixLogsMethodology,
+)
+from holmes.plugins.toolsets.logging_utils.logging_api import FetchPodLogsParams
+from holmes.plugins.toolsets.utils import (
+    process_timestamps_to_rfc3339,
+)
+
+
+DEFAULT_TIME_SPAN_SECONDS = 86400
+DEFAULT_LOG_COUNT = 2000  # Coralogix's default is 2000
+
+
+class CoralogixTier(str, Enum):
+    FREQUENT_SEARCH = "TIER_FREQUENT_SEARCH"
+    ARCHIVE = "TIER_ARCHIVE"
+
+
+def get_dataprime_base_url(domain: str) -> str:
+    return f"https://ng-api-http.{domain}"
+
+
+def execute_http_query(domain: str, api_key: str, query: dict[str, Any]):
+    base_url = get_dataprime_base_url(domain)
+    url = urljoin(base_url, "api/v1/dataprime/query")
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    return requests.post(url, headers=headers, json=query)
+
+
+def health_check(domain: str, api_key: str) -> Tuple[bool, str]:
+    query = {"query": "source logs | limit 1"}
+
+    response = execute_http_query(domain=domain, api_key=api_key, query=query)
+
+    if response.status_code == 200:
+        return True, ""
+    else:
+        return False, f"Failed with status_code={response.status_code}. {response.text}"
+
+
+def build_query_string(config: CoralogixConfig, params: FetchPodLogsParams) -> str:
+    query_filters = []
+    query_filters.append(f'{config.labels.namespace}:"{params.namespace}"')
+    query_filters.append(f'{config.labels.pod}:"{params.pod_name}"')
+
+    if params.filter:
+        query_filters.append(f'{config.labels.log_message}:"{params.filter}"')
+
+    query_string = " AND ".join(query_filters)
+    query_string = f"source logs | lucene '{query_string}' | limit {params.limit or DEFAULT_LOG_COUNT}"
+    return query_string
+
+
+def get_start_end(params: FetchPodLogsParams):
+    (start, end) = process_timestamps_to_rfc3339(
+        start_timestamp=params.start_time,
+        end_timestamp=params.end_time,
+        default_time_span_seconds=DEFAULT_TIME_SPAN_SECONDS,
+    )
+    return (start, end)
+
+
+def build_query(
+    config: CoralogixConfig, params: FetchPodLogsParams, tier: CoralogixTier
+):
+    (start, end) = get_start_end(params)
+
+    query_string = build_query_string(config, params)
+    return {
+        "query": query_string,
+        "metadata": {
+            "tier": tier.value,
+            "syntax": "QUERY_SYNTAX_DATAPRIME",
+            "startDate": start,
+            "endDate": end,
+        },
+    }
+
+
+def query_logs_for_tier(
+    config: CoralogixConfig, params: FetchPodLogsParams, tier: CoralogixTier
+) -> CoralogixQueryResult:
+    http_status = None
+    try:
+        query = build_query(config, params, tier)
+
+        response = execute_http_query(
+            domain=config.domain,
+            api_key=config.api_key,
+            query=query,
+        )
+        http_status = response.status_code
+        if http_status == 200:
+            logs = parse_logs(raw_logs=response.text.strip())
+            return CoralogixQueryResult(logs=logs, http_status=http_status, error=None)
+        else:
+            return CoralogixQueryResult(
+                logs=[], http_status=http_status, error=response.text
+            )
+    except Exception as e:
+        logging.error("Failed to fetch coralogix logs", exc_info=True)
+        return CoralogixQueryResult(logs=[], http_status=http_status, error=str(e))
+
+
+def query_logs_for_all_tiers(
+    config: CoralogixConfig, params: FetchPodLogsParams
+) -> CoralogixQueryResult:
+    methodology = config.logs_retrieval_methodology
+    result: CoralogixQueryResult
+
+    if methodology in [
+        CoralogixLogsMethodology.FREQUENT_SEARCH_ONLY,
+        CoralogixLogsMethodology.BOTH_FREQUENT_SEARCH_AND_ARCHIVE,
+        CoralogixLogsMethodology.ARCHIVE_FALLBACK,
+    ]:
+        result = query_logs_for_tier(
+            config=config, params=params, tier=CoralogixTier.FREQUENT_SEARCH
+        )
+
+        if (
+            methodology == CoralogixLogsMethodology.ARCHIVE_FALLBACK and not result.logs
+        ) or methodology == CoralogixLogsMethodology.BOTH_FREQUENT_SEARCH_AND_ARCHIVE:
+            archive_search_results = query_logs_for_tier(
+                config=config, params=params, tier=CoralogixTier.ARCHIVE
+            )
+            result = merge_log_results(result, archive_search_results)
+
+    else:
+        # methodology in [CoralogixLogsMethodology.ARCHIVE_ONLY, CoralogixLogsMethodology.FREQUENT_SEARCH_FALLBACK]:
+        result = query_logs_for_tier(
+            config=config, params=params, tier=CoralogixTier.ARCHIVE
+        )
+
+        if (
+            methodology == CoralogixLogsMethodology.FREQUENT_SEARCH_FALLBACK
+            and not result.logs
+        ):
+            frequent_search_results = query_logs_for_tier(
+                config=config, params=params, tier=CoralogixTier.FREQUENT_SEARCH
+            )
+            result = merge_log_results(result, frequent_search_results)
+
+    return result

holmes/plugins/toolsets/coralogix/toolset_coralogix_logs.py

@@ -0,0 +1,103 @@
+from typing import Any, Optional, Tuple
+
+from holmes.core.tools import (
+    CallablePrerequisite,
+    StructuredToolResult,
+    ToolResultStatus,
+    ToolsetTag,
+)
+from holmes.plugins.toolsets.consts import (
+    TOOLSET_CONFIG_MISSING_ERROR,
+)
+from holmes.plugins.toolsets.coralogix.api import (
+    build_query_string,
+    get_start_end,
+    health_check,
+    query_logs_for_all_tiers,
+)
+from holmes.plugins.toolsets.coralogix.utils import (
+    CoralogixConfig,
+    build_coralogix_link_to_logs,
+    stringify_flattened_logs,
+)
+from holmes.plugins.toolsets.logging_utils.logging_api import (
+    BasePodLoggingToolset,
+    FetchPodLogsParams,
+    PodLoggingTool,
+)
+
+
+class CoralogixLogsToolset(BasePodLoggingToolset):
+    def __init__(self):
+        super().__init__(
+            name="coralogix/logs",
+            description="Toolset for interacting with Coralogix to fetch logs",
+            docs_url="https://docs.robusta.dev/master/configuration/holmesgpt/toolsets/coralogix_logs.html",
+            icon_url="https://avatars.githubusercontent.com/u/35295744?s=200&v=4",
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            tools=[
+                PodLoggingTool(self),
+            ],
+            tags=[ToolsetTag.CORE],
+        )
+
+    def get_example_config(self):
+        example_config = CoralogixConfig(
+            api_key="<cxuw_...>", team_hostname="my-team", domain="eu2.coralogix.com"
+        )
+        return example_config.model_dump()
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
+        if not config:
+            return False, TOOLSET_CONFIG_MISSING_ERROR
+
+        self.config = CoralogixConfig(**config)
+
+        if not self.config.api_key:
+            return False, "Missing configuration field 'api_key'"
+
+        return health_check(domain=self.config.domain, api_key=self.config.api_key)
+
+    @property
+    def coralogix_config(self) -> Optional[CoralogixConfig]:
+        return self.config
+
+    def fetch_pod_logs(self, params: FetchPodLogsParams) -> StructuredToolResult:
+        if not self.coralogix_config:
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                error=f"The {self.name} toolset is not configured",
+                params=params.model_dump(),
+            )
+
+        logs_data = query_logs_for_all_tiers(
+            config=self.coralogix_config, params=params
+        )
+        (start, end) = get_start_end(params=params)
+        query_string = build_query_string(config=self.coralogix_config, params=params)
+
+        url = build_coralogix_link_to_logs(
+            config=self.coralogix_config,
+            lucene_query=query_string,
+            start=start,
+            end=end,
+        )
+
+        data: str
+        if logs_data.error:
+            data = logs_data.error
+        else:
+            logs = stringify_flattened_logs(logs_data.logs)
+            # Remove link and query from results once the UI and slackbot properly handle the URL from the StructuredToolResult
+            data = f"link: {url}\nquery: {query_string}\n{logs}"
+
+        return StructuredToolResult(
+            status=(
+                ToolResultStatus.ERROR if logs_data.error else ToolResultStatus.SUCCESS
+            ),
+            error=logs_data.error,
+            data=data,
+            url=url,
+            invocation=query_string,
+            params=params.model_dump(),
+        )

holmes/plugins/toolsets/coralogix/utils.py

@@ -0,0 +1,181 @@
+from enum import Enum
+import json
+import logging
+import urllib.parse
+from datetime import datetime
+from typing import Any, NamedTuple, Optional, Dict, List
+
+from pydantic import BaseModel
+
+
+class FlattenedLog(NamedTuple):
+    timestamp: str
+    log_message: str
+
+
+class CoralogixQueryResult(BaseModel):
+    logs: List[FlattenedLog]
+    http_status: Optional[int]
+    error: Optional[str]
+
+
+class CoralogixLabelsConfig(BaseModel):
+    pod: str = "kubernetes.pod_name"
+    namespace: str = "kubernetes.namespace_name"
+    log_message: str = "log"
+
+
+class CoralogixLogsMethodology(str, Enum):
+    FREQUENT_SEARCH_ONLY = "FREQUENT_SEARCH_ONLY"
+    ARCHIVE_ONLY = "ARCHIVE_ONLY"
+    ARCHIVE_FALLBACK = "ARCHIVE_FALLBACK"
+    FREQUENT_SEARCH_FALLBACK = "FREQUENT_SEARCH_FALLBACK"
+    BOTH_FREQUENT_SEARCH_AND_ARCHIVE = "BOTH_FREQUENT_SEARCH_AND_ARCHIVE"
+
+
+class CoralogixConfig(BaseModel):
+    team_hostname: str
+    domain: str
+    api_key: str
+    labels: CoralogixLabelsConfig = CoralogixLabelsConfig()
+    logs_retrieval_methodology: CoralogixLogsMethodology = (
+        CoralogixLogsMethodology.ARCHIVE_FALLBACK
+    )
+
+
+def parse_json_lines(raw_text) -> List[Dict[str, Any]]:
+    """Parses JSON objects from a raw text response."""
+    json_objects = []
+    for line in raw_text.strip().split("\n"):  # Split by newlines
+        try:
+            json_objects.append(json.loads(line))
+        except json.JSONDecodeError:
+            logging.error(f"Failed to decode JSON from line: {line}")
+    return json_objects
+
+
+def normalize_datetime(date_str: Optional[str]) -> str:
+    """takes a date string as input and attempts to convert it into a standardized ISO 8601 format with UTC timezone (“Z” suffix) and microsecond precision.
+    if any error occurs during parsing or formatting, it returns the original input string.
+    The method specifically handles older Python versions by removing a trailing “Z” and truncating microseconds to 6 digits before parsing.
+    """
+    if not date_str:
+        return "UNKNOWN_TIMESTAMP"
+
+    try:
+        # older versions of python do not support `Z` appendix nor more than 6 digits of microsecond precision
+        date_str_no_z = date_str.rstrip("Z")
+
+        parts = date_str_no_z.split(".")
+        if len(parts) > 1 and len(parts[1]) > 6:
+            date_str_no_z = f"{parts[0]}.{parts[1][:6]}"
+
+        date = datetime.fromisoformat(date_str_no_z)
+
+        normalized_date_time = date.strftime("%Y-%m-%dT%H:%M:%S.%f")
+        return normalized_date_time + "Z"
+    except Exception:
+        return date_str
+
+
+def flatten_structured_log_entries(
+    log_entries: List[Dict[str, Any]],
+) -> List[FlattenedLog]:
+    flattened_logs = []
+    for log_entry in log_entries:
+        try:
+            user_data = json.loads(log_entry.get("userData", "{}"))
+            timestamp = normalize_datetime(user_data.get("time"))
+            log_message = user_data.get("log", "")
+            if log_message:
+                flattened_logs.append(
+                    FlattenedLog(timestamp=timestamp, log_message=log_message)
+                )  # Store as tuple for sorting
+
+        except json.JSONDecodeError:
+            logging.error(
+                f"Failed to decode userData JSON: {log_entry.get('userData')}"
+            )
+    return flattened_logs
+
+
+def stringify_flattened_logs(log_entries: List[FlattenedLog]) -> str:
+    formatted_logs = []
+    for entry in log_entries:
+        formatted_logs.append(entry.log_message)
+
+    return "\n".join(formatted_logs) if formatted_logs else "No logs found."
+
+
+def parse_json_objects(json_objects: List[Dict[str, Any]]) -> List[FlattenedLog]:
+    """Extracts timestamp and log values from parsed JSON objects, sorted in ascending order (oldest first)."""
+    logs: List[FlattenedLog] = []
+
+    for data in json_objects:
+        if isinstance(data, dict) and "result" in data and "results" in data["result"]:
+            logs += flatten_structured_log_entries(
+                log_entries=data["result"]["results"]
+            )
+        elif isinstance(data, dict) and data.get("warning"):
+            logging.info(
+                f"Received the following warning when fetching coralogix logs: {data}"
+            )
+        else:
+            logging.debug(f"Unrecognised partial response from coralogix logs: {data}")
+
+    logs.sort(key=lambda x: x[0])
+
+    return logs
+
+
+def parse_logs(raw_logs: str) -> List[FlattenedLog]:
+    """Processes the HTTP response and extracts only log outputs."""
+    try:
+        json_objects = parse_json_lines(raw_logs)
+        if not json_objects:
+            raise Exception("No valid JSON objects found.")
+        return parse_json_objects(json_objects)
+    except Exception as e:
+        logging.error(
+            f"Unexpected error in format_logs for a coralogix API response: {str(e)}"
+        )
+        raise e
+
+
+def build_coralogix_link_to_logs(
+    config: CoralogixConfig, lucene_query: str, start: str, end: str
+) -> str:
+    query_param = urllib.parse.quote_plus(lucene_query)
+
+    return f"https://{config.team_hostname}.app.{config.domain}/#/query-new/logs?query={query_param}&querySyntax=dataprime&time=from:{start},to:{end}"
+
+
+def merge_log_results(
+    a: CoralogixQueryResult, b: CoralogixQueryResult
+) -> CoralogixQueryResult:
+    """
+    Merges two CoralogixQueryResult objects, deduplicating logs and sorting them by timestamp.
+
+    """
+    if a.error is None and b.error:
+        return a
+    elif b.error is None and a.error:
+        return b
+    elif a.error and b.error:
+        return a
+
+    combined_logs = a.logs + b.logs
+
+    if not combined_logs:
+        deduplicated_logs_set = set()
+    else:
+        deduplicated_logs_set = set(combined_logs)
+
+    # Assumes timestamps are in a format sortable as strings (e.g., ISO 8601)
+    sorted_logs = sorted(list(deduplicated_logs_set), key=lambda log: log.timestamp)
+
+    return CoralogixQueryResult(
+        logs=sorted_logs,
+        http_status=a.http_status if a.http_status is not None else b.http_status,
+        error=a.error,
+    )