holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/toolsets/servicenow/install.md

@@ -0,0 +1,37 @@
+## Configuration
+
+[Full guide for reference](https://www.servicenow.com/docs/bundle/yokohama-platform-security/page/integrate/authentication/task/configure-api-key.html)
+
+### Create an inbound authentication profile.
+
+ 1. Navigate to All > System Web Services > API Access Policies > Inbound Authentication Profiles.
+ 2. Select New.
+ 3. Select Create API Key authentication profiles
+ 4. Auth Parameter > add x-sn-apikey: Auth Header
+ 5. Submit the form.
+
+### Create a REST API key
+
+ 1. Navigate to All > System Web Services > API Access Policies > REST API Key.
+ 2. Select New.
+ 3. Set name, description and user. Set expiry date if desired. > Submit.
+ 4. Open the record that was created to view the token generated by the ServiceNow AI Platform for the user.
+
+### Create a REST API Access policy
+
+ 1. Navigate to All > System Web Services > REST API Access Policies.
+ 2. Select New.
+ 3. REST API = Table API
+ 4. Uncheck Apply to all tables > Select table > change_request
+ 5. in select profile from step 1 (API Key)
+
+
+Use your `instance name` and `api_key` to set up Service Now configuration.
+```yaml
+toolsets:
+  ServiceNow:
+    enabled: true
+    config:
+      api_key: <api-token>
+      instance: <dev1234..>
+```

holmes/plugins/toolsets/servicenow/instructions.jinja2

@@ -0,0 +1,3 @@
+* ALWAYS fetch changes from servicenow, USE servicenow_return_changes_in_timerange to see changes in the relevant time range.
+* If you are investigating an issue on some subject , USE servicenow_return_changes_with_keyword with the object name to find related changes.
+* If you find a ServiceNow change that seems relevant to your investigation or the user question, USE servicenow_return_change_details with the change sys_id to get further information and improve your answer if possible.

holmes/plugins/toolsets/servicenow/servicenow.py

@@ -0,0 +1,198 @@
+import requests  # type: ignore
+import logging
+import os
+from typing import Any, Dict, Tuple, List
+from holmes.core.tools import (
+    CallablePrerequisite,
+    Tool,
+    ToolParameter,
+    Toolset,
+    ToolsetTag,
+)
+
+from pydantic import BaseModel, PrivateAttr
+from holmes.core.tools import StructuredToolResult, ToolResultStatus
+from holmes.plugins.toolsets.utils import (
+    process_timestamps_to_rfc3339,
+    standard_start_datetime_tool_param_description,
+)
+
+DEFAULT_TIME_SPAN_SECONDS = 3600
+
+
+class ServiceNowConfig(BaseModel):
+    api_key: str
+    instance: str
+
+
+class ServiceNowToolset(Toolset):
+    name: str = "ServiceNow"
+    description: str = "Database containing changes information related to keys, workloads or any service."
+    tags: List[ToolsetTag] = [ToolsetTag.CORE]
+    _session: requests.Session = PrivateAttr(default=requests.Session())
+
+    def __init__(self):
+        super().__init__(
+            prerequisites=[CallablePrerequisite(callable=self.prerequisites_callable)],
+            experimental=True,
+            tools=[
+                ReturnChangesInTimerange(toolset=self),
+                ReturnChange(toolset=self),
+                ReturnChangesWithKeyword(toolset=self),
+            ],
+        )
+        instructions_filepath = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "instructions.jinja2")
+        )
+        self._load_llm_instructions(jinja_template=f"file://{instructions_filepath}")
+
+    def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
+        if not config:
+            return False, "Missing config credentials."
+
+        try:
+            self.config: Dict = ServiceNowConfig(**config).model_dump()
+            self._session.headers.update(
+                {
+                    "x-sn-apikey": self.config.get("api_key"),
+                }
+            )
+
+            url = f"https://{self.config.get('instance')}.service-now.com/api/now/v2/table/change_request"
+            response = self._session.get(url=url, params={"sysparm_limit": 1})
+
+            return response.ok, ""
+        except Exception as e:
+            logging.exception(
+                "Invalid ServiceNow config. Failed to set up ServiceNow toolset"
+            )
+            return False, f"Invalid ServiceNow config {e}"
+
+    def get_example_config(self) -> Dict[str, Any]:
+        example_config = ServiceNowConfig(
+            api_key="now_xxxxxxxxxxxxxxxx", instance="dev12345"
+        )
+        return example_config.model_dump()
+
+
+class ServiceNowBaseTool(Tool):
+    toolset: ServiceNowToolset
+
+    def return_result(
+        self, response: requests.Response, params: Any, field: str = "result"
+    ) -> StructuredToolResult:
+        response.raise_for_status()
+        res = response.json()
+        return StructuredToolResult(
+            status=ToolResultStatus.SUCCESS
+            if res.get(field, [])
+            else ToolResultStatus.NO_DATA,
+            data=res,
+            params=params,
+        )
+
+    def get_parameterized_one_liner(self, params) -> str:
+        return f"ServiceNow {self.name} {params}"
+
+
+class ReturnChangesInTimerange(ServiceNowBaseTool):
+    name: str = "servicenow_return_changes_in_timerange"
+    description: str = "Returns all changes requests from a specific time range. These changes tickets can apply to all components. default to changes from the last 1 hour."
+    parameters: Dict[str, ToolParameter] = {
+        "start": ToolParameter(
+            description=standard_start_datetime_tool_param_description(
+                DEFAULT_TIME_SPAN_SECONDS
+            ),
+            type="string",
+            required=False,
+        )
+    }
+
+    def _invoke(self, params: Any) -> StructuredToolResult:
+        parsed_params = {}
+        try:
+            (start, _) = process_timestamps_to_rfc3339(
+                start_timestamp=params.get("start"),
+                end_timestamp=None,
+                default_time_span_seconds=DEFAULT_TIME_SPAN_SECONDS,
+            )
+
+            url = f"https://{self.toolset.config.get('instance')}.service-now.com/api/now/v2/table/change_request"
+            parsed_params.update(
+                {
+                    "sysparm_fields": "sys_id,number,short_description,type,active,sys_updated_on"
+                }
+            )
+            parsed_params.update({"sysparm_query": f"sys_updated_on>={start}"})
+
+            response = self.toolset._session.get(url=url, params=parsed_params)
+            return self.return_result(response, parsed_params)
+        except Exception as e:
+            logging.exception(self.get_parameterized_one_liner(params))
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                data=f"Exception {self.name}: {str(e)}",
+                params=params,
+            )
+
+
+class ReturnChange(ServiceNowBaseTool):
+    name: str = "servicenow_return_change_details"
+    description: str = "Returns detailed information for one specific ServiceNow change"
+    parameters: Dict[str, ToolParameter] = {
+        "sys_id": ToolParameter(
+            description="The unique identifier of the change. Use servicenow_return_changes_in_timerange tool to fetch list of changes and use 'sys_id' for further information",
+            type="string",
+            required=True,
+        )
+    }
+
+    def _invoke(self, params: Any) -> StructuredToolResult:
+        try:
+            url = "https://{instance}.service-now.com/api/now/v2/table/change_request/{sys_id}".format(
+                instance=self.toolset.config.get("instance"),
+                sys_id=params.get("sys_id"),
+            )
+            response = self.toolset._session.get(url=url)
+            return self.return_result(response, params)
+        except Exception as e:
+            logging.exception(self.get_parameterized_one_liner(params))
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                data=f"Exception {self.name}: {str(e)}",
+                params=params,
+            )
+
+
+class ReturnChangesWithKeyword(ServiceNowBaseTool):
+    name: str = "servicenow_return_changes_with_keyword"
+    description: str = "Returns all changes requests where a keyword is contained in the description. good for finding changes related to a key, workload or any object."
+    parameters: Dict[str, ToolParameter] = {
+        "keyword": ToolParameter(
+            description="key, workload or object name. Keyword that will filter service now changes that are related to this keyword or object.",
+            type="string",
+            required=True,
+        )
+    }
+
+    def _invoke(self, params: Any) -> StructuredToolResult:
+        parsed_params = {}
+        try:
+            url = f"https://{self.toolset.config.get('instance')}.service-now.com/api/now/v2/table/change_request"
+            parsed_params.update(
+                {
+                    "sysparm_fields": "sys_id,number,short_description,type,active,sys_updated_on"
+                }
+            )
+            parsed_params.update(
+                {"sysparm_query": f"short_descriptionLIKE{params.get('keyword')}"}
+            )
+            response = self.toolset._session.get(url=url, params=parsed_params)
+            return self.return_result(response, parsed_params)
+        except Exception as e:
+            logging.exception(self.get_parameterized_one_liner(params))
+            return StructuredToolResult(
+                status=ToolResultStatus.ERROR,
+                data=f"Exception {self.name}: {str(e)}",
+                params=params,
+            )

holmes/plugins/toolsets/slab.yaml

@@ -0,0 +1,20 @@
+toolsets:
+  slab:
+    description: "Fetches slab pages"
+    docs_url: "https://docs.robusta.dev/master/configuration/holmesgpt/toolsets/slab.html"
+    icon_url: "https://platform.robusta.dev/demos/slab-mark.svg"
+    tags:
+      - core
+    prerequisites:
+      - command: "curl --version"
+      - env:
+          - SLAB_API_KEY
+    tools:
+      - name: "fetch_slab_document"
+        description: "Fetch a document from slab. Use this to fetch runbooks if they are present before starting your investigation."
+        command: |
+          curl -X POST \
+          -H "Authorization: ${SLAB_API_KEY}" \
+          -H "Content-Type: application/json" \
+          -d '{"query":"query { post(id: \"{{ post_id }}\") { id title content } }"}' \
+          https://api.slab.com/v1/graphql

holmes/plugins/toolsets/utils.py

@@ -0,0 +1,137 @@
+import datetime
+import time
+from typing import Dict, Optional, Tuple, Union
+
+from dateutil import parser  # type: ignore
+
+
+def standard_start_datetime_tool_param_description(time_span_seconds: int):
+    return f"Start datetime, inclusive. Should be formatted in rfc3339. If negative integer, the number of seconds relative to end. Defaults to -{time_span_seconds}"
+
+
+def is_int(val):
+    try:
+        int(val)
+    except ValueError:
+        return False
+    else:
+        return True
+
+
+def is_rfc3339(timestamp_str: str) -> bool:
+    """Check if a string is in RFC3339 format."""
+    try:
+        parser.parse(timestamp_str)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def to_unix(timestamp_str: str) -> int:
+    dt = parser.parse(timestamp_str)
+    return int(dt.timestamp())
+
+
+def to_unix_ms(timestamp_str: str) -> int:
+    dt = parser.parse(timestamp_str)
+    return int(dt.timestamp() * 1000)
+
+
+def unix_nano_to_rfc3339(unix_nano: int) -> str:
+    unix_seconds = unix_nano / 1_000_000_000
+
+    seconds_part = int(unix_seconds)
+    milliseconds_part = int((unix_seconds - seconds_part) * 1000)
+
+    dt = datetime.datetime.fromtimestamp(seconds_part, datetime.timezone.utc)
+    return f"{dt.strftime('%Y-%m-%dT%H:%M:%S')}.{milliseconds_part:03d}Z"
+
+
+def datetime_to_unix(timestamp_or_datetime_str):
+    if timestamp_or_datetime_str and is_int(timestamp_or_datetime_str):
+        return int(timestamp_or_datetime_str)
+    else:
+        return to_unix(timestamp_or_datetime_str)
+
+
+def unix_to_rfc3339(timestamp: int) -> str:
+    dt = datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc)
+    return f"{dt.strftime('%Y-%m-%dT%H:%M:%S')}Z"
+
+
+def datetime_to_rfc3339(timestamp):
+    if isinstance(timestamp, int):
+        return unix_to_rfc3339(timestamp)
+    else:
+        return timestamp
+
+
+def process_timestamps_to_rfc3339(
+    start_timestamp: Optional[Union[int, str]],
+    end_timestamp: Optional[Union[int, str]],
+    default_time_span_seconds: int,
+) -> Tuple[str, str]:
+    (start_timestamp, end_timestamp) = process_timestamps_to_int(
+        start_timestamp,
+        end_timestamp,
+        default_time_span_seconds=default_time_span_seconds,
+    )
+    parsed_start_timestamp = datetime_to_rfc3339(start_timestamp)
+    parsed_end_timestamp = datetime_to_rfc3339(end_timestamp)
+    return (parsed_start_timestamp, parsed_end_timestamp)
+
+
+def process_timestamps_to_int(
+    start: Optional[Union[int, str]],
+    end: Optional[Union[int, str]],
+    default_time_span_seconds: int,
+) -> Tuple[int, int]:
+    """
+    Process and normalize start and end timestamps.
+
+    Supports:
+    - Integer timestamps (Unix time)
+    - RFC3339 formatted timestamps
+    - Negative integers as relative time from the other timestamp
+    - Auto-inversion if start is after end
+
+    Returns:
+    Tuple of (start_timestamp, end_timestamp)
+    """
+    # If no end_timestamp provided, use current time
+    if not end or end == "0" or end == 0:
+        end = int(time.time())
+
+    # If no start provided, default to one hour before end
+    if not start:
+        start = -1 * abs(default_time_span_seconds)
+
+    start = datetime_to_unix(start)
+    end = datetime_to_unix(end)
+
+    # Handle negative timestamps (relative to the other timestamp)
+    if isinstance(start, int) and isinstance(end, int):
+        if start < 0 and end < 0:
+            # end is relative to now()
+            end = int(time.time()) + end
+            start = end + start
+        elif start < 0:
+            start = end + start
+        elif end < 0:
+            # start/end are inverted. end should be after start_timestamp
+            delta = end
+            end = start
+            start = start + delta
+
+    # Invert timestamps if start is after end
+    if isinstance(start, int) and isinstance(end, int) and start > end:
+        start, end = end, start
+
+    return (start, end)  # type: ignore
+
+
+def get_param_or_raise(dict: Dict, param: str) -> str:
+    value = dict.get(param)
+    if not value:
+        raise Exception(f'Missing param "{param}"')
+    return value

holmes/plugins/utils.py

@@ -0,0 +1,14 @@
+# this file contains utilities that plugin writers are likely to use - not utilities that are only relevant for core
+from typing import Dict
+
+
+def dict_to_markdown(items: Dict[str, str]) -> str:
+    if not items:
+        return ""
+
+    text = ""
+    for k, v in items.items():
+        # TODO: if v is a url, linkify it
+        text += f"• *{k}*: {v}\n"
+
+    return text

holmes/utils/cache.py

@@ -0,0 +1,84 @@
+import time
+from threading import Timer
+from typing import Any, Dict, Optional
+import json
+import bz2
+
+
+class SetEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, set):
+            return list(o)
+        return json.JSONEncoder.default(self, o)
+
+
+def compress(data):
+    json_str = json.dumps(data, cls=SetEncoder)
+    json_bytes = json_str.encode("utf-8")
+    compressed = bz2.compress(json_bytes, compresslevel=1)
+
+    return compressed
+
+
+def decompress(compressed_data):
+    try:
+        decompressed = bz2.decompress(compressed_data)
+        json_str = decompressed.decode("utf-8")
+        data = json.loads(json_str)
+        return data
+    except Exception as e:
+        raise Exception(f"Decompression failed: {str(e)}")
+
+
+class TTLCache:
+    def __init__(self, ttl_seconds: int):
+        self._cache: Dict[str, Dict[str, Any]] = {}
+        self._ttl = ttl_seconds
+        self._evict_interval = max(self._ttl / 10, 60)
+        self._evict_timer = None
+        self._start_evict_timer()
+
+    def _start_evict_timer(self):
+        self._evict_timer = Timer(self._evict_interval, self._evict)
+        self._evict_timer.daemon = (
+            True  # Allow the program to exit even if timer is alive
+        )
+        self._evict_timer.start()
+
+    def _evict(self):
+        current_time = time.time()
+        expired_keys = [
+            key for key, item in self._cache.items() if item["expiry"] <= current_time
+        ]
+
+        for key in expired_keys:
+            del self._cache[key]
+
+        self._start_evict_timer()
+
+    def set(self, key: str, value: Any) -> None:
+        expiry = time.time() + self._ttl
+
+        self._cache[key] = {"value": compress(value), "expiry": expiry}
+
+    def get(self, key: str) -> Optional[Any]:
+        item = self._cache.get(key)
+
+        if item is None:
+            return None
+
+        if item["expiry"] <= time.time():
+            del self._cache[key]
+            return None
+
+        return decompress(item["value"])
+
+    def delete(self, key: str) -> None:
+        self._cache.pop(key, None)
+
+    def clear(self) -> None:
+        self._cache.clear()
+
+    def __del__(self):
+        if self._evict_timer:
+            self._evict_timer.cancel()

holmes/utils/cert_utils.py

@@ -0,0 +1,40 @@
+import base64
+import os
+
+import certifi
+
+CUSTOM_CERTIFICATE_PATH = "/tmp/custom_ca.pem"
+
+
+def append_custom_certificate(custom_ca: str) -> None:
+    with open(certifi.where(), "ab") as outfile:
+        outfile.write(base64.b64decode(custom_ca))
+
+    os.environ["WEBSOCKET_CLIENT_CA_BUNDLE"] = certifi.where()
+
+
+def create_temporary_certificate(custom_ca: str) -> None:
+    with open(certifi.where(), "rb") as base_cert:
+        base_cert_content = base_cert.read()
+
+    with open(CUSTOM_CERTIFICATE_PATH, "wb") as outfile:
+        outfile.write(base_cert_content)
+        outfile.write(base64.b64decode(custom_ca))
+
+    os.environ["REQUESTS_CA_BUNDLE"] = CUSTOM_CERTIFICATE_PATH
+    os.environ["WEBSOCKET_CLIENT_CA_BUNDLE"] = CUSTOM_CERTIFICATE_PATH
+    certifi.where = lambda: CUSTOM_CERTIFICATE_PATH
+
+
+def add_custom_certificate(custom_ca: str) -> bool:
+    if not custom_ca:
+        return False
+
+    # NOTE: Sometimes (Openshift) the certifi.where() is not writable, so we need to
+    #       use a temporary file in case of PermissionError.
+    try:
+        append_custom_certificate(custom_ca)
+    except PermissionError:
+        create_temporary_certificate(custom_ca)
+
+    return True

holmes/utils/default_toolset_installation_guide.jinja2

@@ -0,0 +1,44 @@
+{% if enabled %}
+{% if is_default %}
+This integration is enabled by default.
+
+If you would like to disable this toolset (not recommended), you need to update the `generated_values.yaml` configuration.
+{% else %}
+To disable this integration, you need to update the `generated_values.yaml` configuration.
+{% endif %}
+
+```yaml
+holmes:
+  toolsets:
+    {{toolset_name}}:
+        enabled: false
+```
+
+{% else %}
+To enable this integration, update the Helm values for Robusta (`generated_values.yaml`):
+
+```yaml
+holmes:
+{% if env_vars %}
+  additionalEnvVars:
+{% for env in env_vars %}
+    - name: {{ env }}
+      value: <{{ env }}>
+{% endfor %}
+{% endif %}
+  toolsets:
+    {{toolset_name}}:
+      enabled: true
+      {% if example_config %}
+      config:
+        {{ example_config | indent(8) }}
+      {% endif %}
+```
+
+{% endif %}
+
+And deploy the updated configuration using Helm:
+
+```bash
+helm upgrade robusta robusta/robusta --values=generated_values.yaml --set clusterName=<YOUR_CLUSTER_NAME>
+```

holmes/utils/definitions.py

@@ -0,0 +1,13 @@
+import os
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+CUSTOM_TOOLSET_LOCATION = os.environ.get(
+    "CUSTOM_TOOLSET_LOCATION", "/etc/holmes/config/custom_toolset.yaml"
+)
+
+
+class RobustaConfig(BaseModel):
+    sinks_config: List[Dict[str, Dict]]
+    global_config: dict

holmes/utils/env.py

@@ -0,0 +1,53 @@
+import logging
+import os
+import re
+from typing import Any, Optional
+
+from pydantic import SecretStr
+
+
+def get_env_replacement(value: str) -> Optional[str]:
+    env_patterns = re.findall(r"{{\s*env\.([^}]*)\s*}}", value)
+
+    result = value
+
+    # Replace env patterns with their values or raise exception
+    for env_var_key in env_patterns:
+        env_var_key = env_var_key.strip()
+        pattern_regex = r"{{\s*env\." + re.escape(env_var_key) + r"\s*}}"
+        if env_var_key in os.environ:
+            replacement = os.environ[env_var_key]
+        else:
+            msg = f"ENV var replacement {env_var_key} does not exist"
+            logging.error(msg)
+            raise ValueError(msg)
+        result = re.sub(pattern_regex, replacement, result)
+
+    return result
+
+
+def replace_env_vars_values(values: dict[str, Any]) -> dict[str, Any]:
+    for key, value in values.items():
+        if isinstance(value, str):
+            env_var_value = get_env_replacement(value)
+            if env_var_value:
+                values[key] = env_var_value
+        elif isinstance(value, SecretStr):
+            env_var_value = get_env_replacement(value.get_secret_value())
+            if env_var_value:
+                values[key] = SecretStr(env_var_value)
+        elif isinstance(value, dict):
+            replace_env_vars_values(value)
+        elif isinstance(value, list):
+            # can be a list of strings
+            values[key] = [
+                (
+                    replace_env_vars_values(item)
+                    if isinstance(item, dict)
+                    else get_env_replacement(item)
+                    if isinstance(item, str)
+                    else item
+                )
+                for item in value
+            ]
+    return values