holmesgpt 0.11.5__py3-none-any.whl

holmes/plugins/sources/jira/__init__.py

@@ -0,0 +1,123 @@
+import logging
+from typing import List, Optional
+
+import requests  # type: ignore
+from requests.auth import HTTPBasicAuth  # type: ignore
+
+from holmes.core.issue import Issue
+from holmes.core.tool_calling_llm import LLMResult
+from holmes.plugins.interfaces import SourcePlugin
+
+
+class JiraSource(SourcePlugin):
+    def __init__(self, url: str, username: str, api_key: str, jql_query: str):
+        self.url = url
+        self.username = username
+        self.api_key = api_key
+        self.jql_query = jql_query
+
+    def fetch_issues(self) -> List[Issue]:
+        logging.info(f"Fetching issues from {self.url} with JQL='{self.jql_query}'")
+        try:
+            response = requests.get(
+                f"{self.url}/rest/api/2/search",
+                params={"jql": self.jql_query},
+                auth=HTTPBasicAuth(self.username, self.api_key),
+                headers={"Accept": "application/json"},
+            )
+            if response.status_code != 200:
+                raise Exception(
+                    f"Failed to get issues: {response.status_code} {response.text}"
+                )
+            logging.info(f"Got {response}")
+            response.raise_for_status()
+            data = response.json()
+            return [self.convert_to_issue(issue) for issue in data.get("issues", [])]
+        except requests.RequestException as e:
+            raise ConnectionError("Failed to fetch data from Jira.") from e
+
+    def convert_to_issue(self, jira_issue, description: Optional[str] = None):
+        description = self.extract_description(jira_issue)
+        return Issue(
+            id=jira_issue["id"],
+            name=jira_issue["fields"]["summary"],
+            source_type="jira",
+            source_instance_id=self.url,
+            url=f"{self.url}/browse/{jira_issue['key']}",
+            description=description,
+            raw=jira_issue,
+        )
+
+        # status=jira_issue["fields"]["status"]["name"],
+
+    def extract_description(self, jira_issue) -> str:
+        """
+        Extracts and formats the issue description.
+        """
+        description_blocks = (
+            jira_issue.get("fields", {}).get("description", {}).get("content", [])
+        )
+        description_text = []
+
+        for block in description_blocks:
+            if block["type"] == "paragraph":
+                text = " ".join(
+                    [c["text"] for c in block.get("content", []) if "text" in c]
+                )
+                description_text.append(text)
+            elif block["type"] == "orderedList":
+                for idx, item in enumerate(block["content"], start=1):
+                    text = " ".join(
+                        [
+                            c["text"]
+                            for c in item["content"][0].get("content", [])
+                            if "text" in c
+                        ]
+                    )
+                    description_text.append(f"{idx}. {text}")
+
+        return (
+            "\n".join(description_text)
+            if description_text
+            else "No description available."
+        )
+
+    def write_back_result(self, issue_id: str, result_data: LLMResult) -> None:
+        # TODO: upload files and show tool usage
+        comment_url = f"{self.url}/rest/api/2/issue/{issue_id}/comment"
+        comment_data = {
+            "body": f"Automatic AI Investigation by Robusta:\n\n{result_data.result}\n"
+        }
+        response = requests.post(
+            comment_url,
+            json=comment_data,
+            auth=HTTPBasicAuth(self.username, self.api_key),
+            headers={"Accept": "application/json"},
+        )
+        response.raise_for_status()
+        data = response.json()
+        logging.debug(f"Comment added to issue {issue_id}: {data}")
+
+
+class JiraServiceManagementSource(JiraSource):
+    def __init__(self, url: str, username: str, api_key: str, jql_query: str):
+        super().__init__(url, username, api_key, jql_query)
+
+    def fetch_issue(self, id: str) -> Issue:
+        """
+        Might also be the same in jira, needs additional testing
+        """
+        logging.info(f"Fetching Jira Service Management issue {id} from {self.url}")
+
+        try:
+            response = requests.get(
+                f"{self.url}/rest/api/3/issue/{id}",
+                auth=HTTPBasicAuth(self.username, self.api_key),
+                headers={"Accept": "application/json"},
+            )
+            response.raise_for_status()
+            jsm_issue = response.json()
+            description = self.extract_description(jsm_issue)
+            return self.convert_to_issue(jsm_issue, description)
+        except requests.RequestException as e:
+            raise ConnectionError(f"Failed to fetch JSM ticket {id}") from e

holmes/plugins/sources/opsgenie/__init__.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Dict, List, Optional, Union
+
+import markdown  # type: ignore
+import requests  # type: ignore
+
+from holmes.core.issue import Issue
+from holmes.core.tool_calling_llm import LLMResult
+from holmes.plugins.interfaces import SourcePlugin
+
+OPSGENIE_TEAM_INTEGRATION_KEY_HELP = "OpsGenie Team Integration key for writing back results. (NOT a normal API Key.) Get it from Teams > YourTeamName > Integrations > Add Integration > API Key. Don't forget to turn on the integration!"
+
+
+class OpsGenieSource(SourcePlugin):
+    def __init__(
+        self, api_key: str, query: str, team_integration_key: Optional[str] = None
+    ):
+        self.api_key = api_key
+        self.query = query
+        self.team_integration_key = team_integration_key
+
+    def fetch_issues(self) -> List[Issue]:
+        logging.info(f"Fetching alerts from OpsGenie with query: {self.query}")
+        try:
+            data = []
+            url = "https://api.opsgenie.com/v2/alerts"
+            headers = {
+                "Authorization": f"GenieKey {self.api_key}",
+                "Content-Type": "application/json",
+            }
+            params: Dict[str, Union[int, str]] = {"query": self.query, "limit": 100}
+            while url:
+                # TODO: also fetch notes and description
+                response = requests.get(url, headers=headers, params=params)
+                logging.debug(f"Got {response.json()}")
+                if response.status_code != 200:
+                    raise Exception(
+                        f"Failed to get alerts: {response.status_code} {response.text}"
+                    )
+                response.raise_for_status()
+                data.extend(response.json().get("data", []))
+                next_url = response.json().get("paging", {}).get("next", None)
+                url = next_url if next_url else None  # type: ignore
+            return [self.convert_to_issue(alert) for alert in data]
+        except requests.RequestException as e:
+            raise ConnectionError("Failed to fetch data from OpsGenie.") from e
+
+    def convert_to_issue(self, opsgenie_alert):
+        return Issue(
+            id=str(opsgenie_alert["id"]),
+            name=opsgenie_alert["message"],
+            source_type="opsgenie",
+            source_instance_id="opsgenie",
+            url=opsgenie_alert["tinyId"],
+            raw=opsgenie_alert,
+        )
+
+    def write_back_result(self, issue_id: str, result_data: LLMResult) -> None:
+        if self.team_integration_key is None:
+            raise Exception(
+                f"Please set '--opsgenie-team-integration-key' to write back results. This is an {OPSGENIE_TEAM_INTEGRATION_KEY_HELP}"
+            )
+
+        # TODO: update description to make this more visible (right now we add a comment)
+        html_output = markdown.markdown(result_data.result)
+        logging.debug(f"HTML output: {html_output}")
+
+        url = f"https://api.opsgenie.com/v2/alerts/{issue_id}/notes?identifierType=id"
+        headers = {
+            "Authorization": f"GenieKey {self.team_integration_key}",
+            "Content-Type": "application/json",
+        }
+        response = requests.post(
+            url=url,
+            json={"note": f"Automatic AI Investigation by Robusta:\n\n{html_output}\n"},
+            headers=headers,
+        )
+        logging.debug(f"Response: {response.json()}")
+        response.raise_for_status()
+
+        # We get back a response like: {'result': 'Request will be processed', 'took': 0.006, 'requestId': '<request_id>'}
+        # Now we need to lookup the request to see if it succeeded
+        request_id = response.json().get("requestId", None)
+        url = f"https://api.opsgenie.com/v2/alerts/requests/{request_id}"
+        response = requests.get(url=url, headers=headers)
+
+        logging.debug(f"Response: {response.json()}")
+        response.raise_for_status()
+        json_response = response.json()
+        if not json_response["data"]["success"]:
+            raise Exception(
+                f"Failed to write back result to OpsGenie: {json_response['data']['status']}"
+            )

holmes/plugins/sources/pagerduty/__init__.py

@@ -0,0 +1,147 @@
+import logging
+from typing import List, Optional
+
+import requests  # type: ignore
+
+from holmes.core.issue import Issue
+from holmes.core.tool_calling_llm import LLMResult
+from holmes.plugins.interfaces import SourcePlugin
+from holmes.utils.markdown_utils import markdown_to_plain_text
+
+
+class PagerDutySource(SourcePlugin):
+    def __init__(
+        self, api_key: str, user_email: str, incident_key: Optional[str] = None
+    ):
+        self.api_url = (
+            "https://api.pagerduty.com"  # currently hard-coded, can expose it if useful
+        )
+        self.api_key = api_key
+        self.user_email = user_email
+        self.incident_key = incident_key
+
+    def fetch_issues(self) -> List[Issue]:
+        logging.info(f"Fetching issues from {self.api_url}")
+        try:
+            headers = {
+                "Authorization": f"Token token={self.api_key}",
+                "Accept": "application/vnd.pagerduty+json;version=2",
+            }
+
+            # excludes resolved
+            query_params = "?statuses[]=triggered&statuses[]=acknowledged"
+
+            if self.incident_key:
+                query_params = f"{query_params}&incident_key={self.incident_key}"
+
+            response = requests.get(
+                f"{self.api_url}/incidents{query_params}", headers=headers
+            )
+            if response.status_code != 200:
+                print(f"Got response: {response}")
+                raise Exception(
+                    f"Failed to get issues: {response.status_code} {response.text}"
+                )
+            logging.debug(f"Got response: {response}")
+            response.raise_for_status()
+            data = response.json()
+            return [self.convert_to_issue(issue) for issue in data.get("incidents", [])]
+        except requests.RequestException as e:
+            raise ConnectionError("Failed to fetch data from PagerDuty.") from e
+
+    def fetch_issue(self, id: str) -> Optional[Issue]:  # type: ignore
+        """
+        Fetch a single issue from PagerDuty using the incident ID and convert it to an Issue object.
+
+        :param incident_id: The ID of the incident to fetch.
+        :return: An Issue object if found, otherwise None.
+        """
+        logging.info(f"Fetching issue {id} from {self.api_url}")
+
+        headers = {
+            "Authorization": f"Token token={self.api_key}",
+            "Accept": "application/vnd.pagerduty+json;version=2",
+        }
+
+        try:
+            response = requests.get(f"{self.api_url}/incidents/{id}", headers=headers)
+
+            if response.status_code == 404:
+                logging.warning(f"Incident {id} not found.")
+                return None
+
+            if response.status_code != 200:
+                logging.error(
+                    f"Failed to get issue: {response.status_code} {response.text}"
+                )
+                raise Exception(
+                    f"Failed to get issue: {response.status_code} {response.text}"
+                )
+
+            logging.debug(f"Got response: {response.json()}")
+            incident_data = response.json().get("incident")
+
+            if incident_data:
+                return self.convert_to_issue(incident_data)
+            else:
+                logging.warning(f"No incident data found for {id}.")
+                return None
+
+        except requests.RequestException as e:
+            logging.error(f"Connection error while fetching issue {id}: {e}")
+            raise ConnectionError("Failed to fetch data from PagerDuty.") from e
+
+    def convert_to_issue(self, source_issue):
+        return Issue(
+            id=source_issue["id"],
+            name=source_issue["summary"],
+            source_type="pagerduty",
+            source_instance_id=self.api_url,
+            url=f"{source_issue['html_url']}",
+            raw=source_issue,
+        )
+
+    def write_back_result(self, issue_id: str, result_data: LLMResult) -> None:
+        logging.info(f"Writing back result to issue {issue_id}")
+        if not self.user_email:
+            raise Exception(
+                "When using --update mode, --pagerduty-user-email must be provided"
+            )
+
+        try:
+            url = f"{self.api_url}/incidents/{issue_id}/notes"
+            headers = {
+                "Authorization": f"Token token={self.api_key}",
+                "Content-Type": "application/json",
+                "From": self.user_email,
+            }
+            comment = markdown_to_plain_text(result_data.result)
+            comment_data = {
+                "note": {
+                    "content": f"Automatic AI Investigation by HolmesGPT:\n\n{comment}"
+                }
+            }
+            response = requests.post(url, json=comment_data, headers=headers)
+            response.raise_for_status()
+            data = response.json()
+            logging.debug(f"Comment added to issue {issue_id}: {data}")
+        except requests.RequestException as e:
+            if e.response is not None:
+                logging.error(
+                    f"Failed to write back result to PagerDuty: {e}; {e.response.text}"
+                )
+            else:
+                logging.error(f"Failed to write back result to PagerDuty: {e}")
+            raise
+
+
+# Run with:
+# poetry run python3 -m holmes.plugins.sources.pagerduty <api-key> <user-email>
+if __name__ == "__main__":
+    import sys
+
+    pd_source = PagerDutySource(api_key=sys.argv[1], user_email=sys.argv[2])
+    issues = pd_source.fetch_issues()
+    for issue in issues:
+        pd_source.write_back_result(issue.id, LLMResult(result="This is a test"))
+        print(issue)

holmes/plugins/sources/prometheus/models.py

@@ -0,0 +1,104 @@
+import html
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Union
+from urllib.parse import parse_qs, unquote, urlparse
+from pydantic import BaseModel, computed_field
+
+
+# these models are used by AlertManager's push API (when alertmanager pushes alerts to us by webhook)
+# this is the standard format we use internally
+class PrometheusAlert(BaseModel):
+    status: str
+    labels: Dict[str, str]
+    annotations: Dict[str, str]
+    startsAt: datetime
+    endsAt: datetime
+    generatorURL: Optional[str] = None
+    fingerprint: str
+
+    @computed_field  # type: ignore
+    @property
+    def unique_id(self) -> str:
+        return f"{self.name}-{self.fingerprint}-{self.startsAt}"
+
+    @computed_field  # type: ignore
+    @property
+    def duration(self) -> Union[timedelta, str]:
+        if self.endsAt.year == 1:
+            return "Ongoing"
+        else:
+            duration = self.endsAt - self.startsAt
+            return duration
+
+    @computed_field  # type: ignore
+    @property
+    def name(self) -> str:
+        return self.labels["alertname"]
+
+    @computed_field  # type: ignore
+    @property
+    def definition(self) -> str:
+        """
+        Returns the promql definition of this alert
+        """
+        url = self.generatorURL
+        if not url:
+            return ""
+
+        # decode HTML entities to convert + like representations to characters
+        url = html.unescape(url)
+        parsed_url = urlparse(url)
+        query_params = parse_qs(parsed_url.query)
+
+        q_expr = query_params.get("g0.expr", [])
+        if len(q_expr) < 1 or not q_expr[0]:
+            return ""
+
+        return unquote(q_expr[0])
+
+
+class PrometheusAlertGroup(BaseModel):
+    receiver: str
+    status: str
+    alerts: List[PrometheusAlert]
+    groupLabels: Dict[str, str]
+    commonLabels: Dict[str, str]
+    commonAnnotations: Dict[str, str]
+    externalURL: str
+    version: str
+    groupKey: str
+    truncatedAlerts: int
+
+
+# these models are used by AlertManager's pull API (when pulling alerts from alertmanager via API)
+class PrometheusReceiver(BaseModel):
+    name: str
+
+
+class PrometheusAlertStatus(BaseModel):
+    state: str
+    silencedBy: List[str]
+    inhibitedBy: List[str]
+
+
+class PrometheusGettableAlert(BaseModel):
+    labels: Dict[str, str]
+    generatorURL: Optional[str] = ""
+    annotations: Dict[str, str]
+    receivers: List[PrometheusReceiver]
+    fingerprint: str
+    startsAt: datetime
+    updatedAt: datetime
+    endsAt: datetime
+    status: PrometheusAlertStatus
+
+    def to_regular_prometheus_alert(self) -> PrometheusAlert:
+        return PrometheusAlert(
+            status=self.status.state,
+            labels=self.labels,
+            annotations=self.annotations,
+            startsAt=self.startsAt,
+            endsAt=self.endsAt,
+            generatorURL=self.generatorURL,
+            fingerprint=self.fingerprint,
+        )

holmes/plugins/sources/prometheus/plugin.py

@@ -0,0 +1,154 @@
+import json
+import logging
+import re
+from pathlib import Path
+from typing import List, Optional, Pattern
+
+import humanize
+import requests  # type: ignore
+import rich
+import rich.segment
+from pydantic import parse_obj_as
+from pydantic.json import pydantic_encoder
+from requests.auth import HTTPBasicAuth  # type: ignore
+
+from holmes.core.issue import Issue
+from holmes.plugins.interfaces import SourcePlugin
+from holmes.plugins.utils import dict_to_markdown
+
+from .models import PrometheusAlert, PrometheusGettableAlert
+
+
+class AlertManagerSource(SourcePlugin):
+    """
+    Issue IDs are of the format {alert_name}-{alert_fingerprint}-{starts_at} which is both unique and allows
+    quickly identifying the alertname and using it to filter on issue_id
+    """
+
+    def __init__(
+        self,
+        url: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        alertname_filter: Optional[Pattern] = None,
+        label_filter: Optional[str] = None,
+        filepath: Optional[Path] = None,
+    ):
+        super().__init__()
+        self.url = url
+        self.username = username
+        self.password = password
+        self.alertname_filter = alertname_filter
+        self.label_filter = label_filter
+        self.filepath = filepath
+
+        if self.url is None and self.filepath is None:
+            # we don't mention --alertmanager-file to avoid confusing users - most users wont care about it
+            raise ValueError("--alertmanager-url must be specified")
+        if self.url is not None and self.filepath is not None:
+            logging.warning(
+                "Ignoring --alertmanager-url because --alertmanager-file is specified"
+            )
+        if self.label_filter and self.filepath is not None:
+            logging.warning(
+                "Ignoring --label-filter because --alertmanager-file is specified"
+            )
+        if self.url and not (
+            self.url.startswith("http://") or self.url.startswith("https://")
+        ):
+            raise ValueError("--alertmanager-url must start with http:// or https://")
+
+    def __fetch_issues_from_api(self) -> List[PrometheusAlert]:
+        fetch_alerts_url = f"{self.url}/api/v2/alerts"
+        params = {
+            "active": "true",
+            "silenced": "false",
+            "inhibited": "false",
+        }
+        if self.label_filter:
+            params["filter"] = self.label_filter
+            logging.info(f"Filtering alerts by {self.label_filter}")
+
+        if self.username is not None or self.password is not None:
+            auth = HTTPBasicAuth(self.username, self.password)  # type: ignore
+        else:
+            auth = None
+
+        logging.info(f"Loading alerts from url {fetch_alerts_url}")
+        response = requests.get(fetch_alerts_url, params=params, auth=auth)
+        if response.status_code != 200:
+            raise Exception(
+                f"Failed to get live alerts: {response.status_code} {response.text}"
+            )
+        data = response.json()
+        return [
+            a.to_regular_prometheus_alert()
+            for a in parse_obj_as(List[PrometheusGettableAlert], data)
+        ]
+
+    def __fetch_issues_from_file(self) -> List[PrometheusAlert]:
+        logging.info(f"Loading alerts from file {self.filepath}")
+        with open(self.filepath, "r") as f:  # type: ignore
+            data = json.load(f)
+        return parse_obj_as(List[PrometheusAlert], data)
+
+    def fetch_issues(self) -> List[Issue]:
+        if self.filepath is not None:
+            alerts = self.__fetch_issues_from_file()
+        else:
+            alerts = self.__fetch_issues_from_api()
+
+        if self.alertname_filter is not None:
+            alertname_filter = re.compile(self.alertname_filter)
+            alerts = [a for a in alerts if alertname_filter.match(a.unique_id)]
+
+        return [
+            Issue(
+                id=alert.unique_id,
+                name=alert.name,
+                source_type="prometheus",
+                source_instance_id=self.filepath if self.filepath else self.url,  # type: ignore
+                url=alert.generatorURL,
+                presentation_key_metadata=f"*Severity*: {alert.labels['severity']}\n*Start Time*: {alert.startsAt.strftime('%Y-%m-%d %H:%M:%S UTC')}\n*Duration*: {humanize.naturaldelta(alert.duration)}",  # type: ignore
+                presentation_all_metadata=self.__format_issue_metadata(alert),
+                raw=alert.model_dump(),
+            )
+            for alert in alerts
+        ]
+
+    def dump_raw_alerts_to_file(self, path: Path) -> None:
+        """
+        Useful for generating test data
+        """
+        alerts = self.__fetch_issues_from_api()
+        with open(path, "w") as f:
+            f.write(json.dumps(alerts, default=pydantic_encoder, indent=2))
+
+    def output_curl_commands(self, console: rich.console.Console) -> None:
+        """
+        Outputs curl commands to send each alert to Alertmanager via the API.
+        """
+        alerts = self.__fetch_issues_from_api()
+        for alert in alerts:
+            alert_json = json.dumps(
+                [alert.model_dump()], default=pydantic_encoder
+            )  # Wrap in a list
+            curl_command = (
+                f"curl -X POST -H 'Content-Type: application/json' "
+                f"-d '{alert_json}' {self.url}/api/v2/alerts"
+            )
+            console.print(f"[green]{alert.name} alert[/green]")
+            console.print(f"[yellow]{curl_command}[/yellow]", soft_wrap=True)
+
+    @staticmethod
+    def __format_issue_metadata(alert: PrometheusAlert) -> Optional[str]:
+        if not alert.labels and not alert.annotations:
+            return None
+        text = ""
+        if alert.labels:
+            text += "*Labels:*\n"
+            text += dict_to_markdown(alert.labels)
+        if alert.annotations:
+            text += "*Annotations:*\n"
+            text += dict_to_markdown(alert.annotations)
+        return text