intelmq-extensions 1.8.1__py3-none-any.whl

intelmq_extensions/bots/collectors/blackkite/_client.py

@@ -0,0 +1,167 @@
+"""Client to access the BlackKite library
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import logging
+import math
+from enum import Enum
+from typing import Iterator, Sequence
+
+import requests
+
+from intelmq_extensions.lib.api_helpers import (
+    DEFAULT_REFRESH_WINDOW,
+    OAuthAccessMixin,
+    RateLimiter,
+)
+
+from ....lib.blackkite import Category
+
+default_logger = logging.getLogger(__name__)
+
+
+class Status(str, Enum):
+    ACTIVE = "Active"
+    FALSE_POSITIVE = "FalsePositive"
+    SUPPRESSED = "Suppressed"
+    ACKNOWLEDGED = "Acknowledged"
+    DELETED = "Deleted"
+
+
+class Severity(str, Enum):
+    INFO = "Info"
+    LOW = "Low"
+    MEDIUM = "Medium"
+    HIGH = "High"
+    CRITICAL = "Critical"
+
+
+class Output(str, Enum):
+    INFO = "Info"
+    PASSED = "Passed"
+    WARNING = "Warning"
+    FAILED = "Failed"
+
+
+_DEFAULT_STATUSES = [Status.ACTIVE]
+_DEFAULT_SEVERITY = [Severity.CRITICAL]
+
+CATEGORIES_WITH_OUTPUT = [
+    Category.DNSHealth,
+    Category.ApplicationSecurity,
+    Category.EmailSecurity,
+    Category.NetworkSecurity,
+    Category.DDoSResiliency,
+    Category.SSLTLSstrength,
+    Category.InformationDisclosure,
+]
+
+
+class BlackKiteClient(OAuthAccessMixin):
+    def __init__(
+        self,
+        url: str,
+        client_id: str,
+        client_secret: str,
+        refresh_before: str = DEFAULT_REFRESH_WINDOW,
+        session: requests.Session = None,
+        logger: logging.Logger = default_logger,
+        limit_requests: int = 60,
+        limit_period: int = 60,
+        page_size: int = 100,
+    ) -> None:
+        self.url = url
+        self.logger = logger
+        self._session = session
+        self._page_size = page_size
+
+        self.limiter = RateLimiter(limit_requests, limit_period)
+        self.init_oauth(
+            oauth_url=f"{url}/oauth/token",
+            oauth_clientid=client_id,
+            oauth_clientsecret=client_secret,
+            session=session,
+            logger=logger,
+            refresh_before=refresh_before,
+            limiter=self.limiter,
+        )
+
+    def get(self, path: str, params: dict = None, raw: bool = False):
+        with self.limiter.call():
+            response = self._session.get(
+                f"{self.url}/{path}",
+                params=params,
+                headers={"Authorization": f"Bearer {self.access_token}"},
+            )
+        if not response.ok:
+            self.logger.error(
+                "Request %s failed with error %s, message: %s.",
+                path,
+                response.status_code,
+                response.text,
+            )
+            raise RuntimeError(f"Request to {path} failed with {response.status_code}")
+        return response if raw else response.json()
+
+    def get_paginated(self, path: str, params: dict = None) -> Iterator[dict]:
+        last = False
+        page = 1
+        params = params or {}
+        while not last:
+            params.update({"page_number": page, "page_size": self._page_size})
+            response = self.get(path, params, raw=True)
+
+            total_items = int(response.headers.get("X-Total-Items", "0"))
+            last = page >= math.ceil(total_items / self._page_size)
+
+            for element in response.json():
+                yield element
+            page += 1
+
+    def list_findings(
+        self,
+        path: str,
+        company_id: int,
+        severities: Sequence[Severity] = None,
+        statuses: Sequence[Status] = None,
+        outputs: Sequence[Output] = None,
+    ) -> Iterator[dict]:
+        severities = severities or _DEFAULT_SEVERITY
+        statuses = statuses or _DEFAULT_STATUSES
+
+        params = {"status": ",".join(statuses), "severity": ",".join(severities)}
+        if outputs:
+            params["output"] = ",".join(outputs)
+
+        return self.get_paginated(f"companies/{company_id}/findings/{path}", params)
+
+    def status(self) -> dict:
+        return self.get("status")
+
+    def companies(self) -> Iterator[dict]:
+        return self.get_paginated("companies")
+
+    def get_findings_from_category(
+        self,
+        category: Category,
+        company_id: int,
+        severities: Sequence[Severity] = None,
+        statuses: Sequence[Status] = None,
+        outputs: Sequence[Output] = None,
+    ):
+        if category not in CATEGORIES_WITH_OUTPUT:
+            outputs = None
+        return self.list_findings(
+            category.name.lower(), company_id, severities, statuses, outputs
+        )
+
+    def acknowledge_finding(self, company_id: int, finding_id: int):
+        with self.limiter.call():
+            result = self._session.patch(
+                f"{self.url}/companies/{company_id}/findings/{finding_id}",
+                headers={"Authorization": f"Bearer {self.access_token}"},
+                json={"Status": Status.ACKNOWLEDGED.value},
+            )
+            self.logger.debug("ACK finding %s, result: %s.", finding_id, result.text)

intelmq_extensions/bots/collectors/blackkite/collector.py

@@ -0,0 +1,182 @@
+"""Collector of data from BlackKite API
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+Parameters:
+
+url
+client_id
+client_secret
+refresh_before
+
+categories (dict):
+    {category-code, e.g. PATCH}:
+        severities: [{list of names}] (optional, override)
+        outputs: [{list of names}] (optional, override)
+        statuses: [{list of names}] (optional, override)
+        include: [{list of ids, eg. XXX-001}] (optional, mutual exclusive with exclude)
+        exclude: [{list of ids, eg. XXX-001}] (optional, mutual exclusive with include)
+        acknowledge: bool (optional, whether change finding's status to acknowledged or not
+                     (default: false))
+severities: []{list of names}
+outputs: [{list of names}]
+statuses: [{list of names}]
+"""
+
+import json
+
+from intelmq.lib.bot import CollectorBot
+from intelmq.lib.utils import create_request_session
+
+from intelmq_extensions.lib.blackkite import Category
+
+from ._client import CATEGORIES_WITH_OUTPUT, BlackKiteClient, Output, Severity, Status
+
+
+class BlackKiteCollectorBot(CollectorBot):
+    url: str = ""
+    client_id: str = ""
+    client_secret: str = ""
+    # refresh access token when it's less than 10 minutes to expire
+    refresh_token_before: int = 10
+    # BlackKite has API rate limit 60 req./1 minute
+    limit_requests: int = 60
+    limit_period: int = 60  # 1 minute
+    page_size: int = 100
+
+    categories: dict = {}
+    severities: list = [Severity.CRITICAL.value]
+    outputs: list = [Output.FAILED.value]
+    statuses: list = [Status.ACTIVE.value]
+
+    def init(self):
+        self.set_request_parameters()
+        self.session = create_request_session(self)
+        self.client = BlackKiteClient(
+            url=self.url,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            refresh_before=self.refresh_token_before,
+            session=self.session,
+            logger=self.logger,
+            limit_requests=self.limit_requests,
+            limit_period=self.limit_period,
+            page_size=self.page_size,
+        )
+
+        self._process_settings()
+
+    def _process_settings(self):
+        self._default_config = {
+            "severities": [Severity(value) for value in self.severities],
+            "outputs": [Output(value) for value in self.outputs],
+            "statuses": [Status(value) for value in self.statuses],
+        }
+
+        self._categories = dict(
+            self._process_category_config(k, v) for k, v in self.categories.items()
+        )
+
+    @staticmethod
+    def _process_category_config(key: str, data: dict):
+        data = data or {}
+        category = Category(key)
+        if "include" in data and "exclude" in data:
+            raise ValueError("Including and excluding at the same time isn't possible")
+        processed_data = {}
+        if severities := data.get("severities"):
+            processed_data["severities"] = [Severity(v) for v in severities]
+        if statuses := data.get("statuses"):
+            processed_data["statuses"] = [Status(v) for v in statuses]
+        if outputs := data.get("outputs"):
+            if category not in CATEGORIES_WITH_OUTPUT:
+                raise ValueError(f"{category.value} does not support output filtering.")
+            processed_data["outputs"] = [Output(v) for v in outputs]
+        if include := data.get("include"):
+            if any(filter(lambda v: not v.startswith(f"{category.value}-"), include)):
+                raise ValueError(f"Category {category.value} includes incorrect IDs")
+            processed_data["include"] = include
+        if exclude := data.get("exclude"):
+            if any(filter(lambda v: not v.startswith(f"{category.value}-"), exclude)):
+                raise ValueError(f"Category {category.value} excludes incorrect IDs")
+            processed_data["exclude"] = exclude
+        processed_data["acknowledge"] = data.get("acknowledge", False)
+
+        return category, processed_data
+
+    def process(self):
+        for company in self.client.companies():
+            for category, config in self._categories.items():
+                try:
+                    self._process_category(
+                        company, category, {**self._default_config, **config}
+                    )
+                except Exception as exc:
+                    self.logger.error(
+                        "Error when processing a category: %s.", exc, exc_info=True
+                    )
+
+    def _process_category(self, company: dict, category: Category, config: dict):
+        def _should_send(finding):
+            if include := config.get("include"):
+                return finding.get("ControlId") in include
+            elif exclude := config.get("exclude"):
+                return finding.get("ControlId") not in exclude
+
+            return True
+
+        for finding in self.client.get_findings_from_category(
+            category,
+            company["CompanyId"],
+            statuses=config.get("statuses"),
+            severities=config.get("severities"),
+            outputs=config.get("outputs"),
+        ):
+            if not _should_send(finding):
+                continue
+            report = self.new_report()
+            report.add("raw", json.dumps({"company": company, "finding": finding}))
+            self.send_message(report)
+            if config.get("acknowledge"):
+                self.client.acknowledge_finding(
+                    company.get("CompanyId"), finding.get("FindingId")
+                )
+
+    @staticmethod
+    def check(parameters: dict) -> list[list[str]] or None:
+        errors = []
+        definitions = [
+            ("severities", Severity),
+            ("outputs", Output),
+            ("statuses", Status),
+        ]
+        for key, type_ in definitions:
+            try:
+                [type_(value) for value in parameters.get(key, [])]
+            except ValueError as exc:
+                errors.append(["error", f"Error processing '{key}': {exc}."])
+
+        allowed_keys = {
+            "severities",
+            "outputs",
+            "statuses",
+            "include",
+            "exclude",
+            "acknowledge",
+        }
+        for category, value in parameters.get("categories", {}).items():
+            value = value or {}
+            try:
+                if set(value.keys()) - allowed_keys:
+                    raise ValueError("Unsupported config key")
+                BlackKiteCollectorBot._process_category_config(category, value)
+            except (ValueError, TypeError) as exc:
+                errors.append(
+                    ["error", f"Error processing category {category}: {exc}."]
+                )
+
+        return errors or None
+
+
+BOT = BlackKiteCollectorBot

intelmq_extensions/bots/collectors/disp/_client.py

@@ -0,0 +1,121 @@
+"""Client to access the DISP API
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+import logging
+from datetime import datetime
+from urllib.parse import quote, urlencode
+
+import requests
+
+from intelmq_extensions.lib.api_helpers import DEFAULT_REFRESH_WINDOW, OAuthAccessMixin
+
+_SCOPE = "https://gateway.disp.deloitte.com/.default"
+_GRANT_TYPE = "client_credentials"
+
+
+default_logger = logging.getLogger(__name__)
+
+
+class DISPClient(OAuthAccessMixin):
+    def __init__(
+        self,
+        api_url: str,
+        auth_token: str,
+        oauth_clientid: str,
+        oauth_clientsecret: str,
+        oauth_url: str,
+        session: requests.Session,
+        refresh_before: int = DEFAULT_REFRESH_WINDOW,
+        logger: logging.Logger = default_logger,
+    ) -> None:
+        self.api_url = api_url
+        self.auth_token = auth_token
+        self._access_token = None
+        self._session = session
+        self._page_size = 10
+        self.logger = logger
+
+        self.init_oauth(
+            oauth_clientid=oauth_clientid,
+            oauth_clientsecret=oauth_clientsecret,
+            oauth_url=oauth_url,
+            oauth_scope=_SCOPE,
+            oauth_grant_type=_GRANT_TYPE,
+            logger=self.logger,
+            refresh_before=refresh_before,
+            session=self._session,
+        )
+
+    def _auth(self):
+        return {
+            "Authorization": f"Bearer {self.auth_token}",
+            "OAuth": self.access_token,
+        }
+
+    def get(self, path: str, params: dict = None):
+        response = self._session.get(
+            f"{self.api_url}/{path}",
+            params=params,
+            headers=self._auth(),
+        )
+        if not response.ok:
+            self.logger.error(
+                "Request %s failed with error %s, message: %s",
+                path,
+                response.status_code,
+                response.text,
+            )
+            raise RuntimeError(f"Request to {path} failed with {response.status_code}")
+        return response.json()
+
+    def post(self, path: str, params: dict = None) -> requests.Response:
+        response = self._session.post(
+            f"{self.api_url}/{path}", params=params, headers=self._auth()
+        )
+        if not response.ok:
+            self.logger.error(
+                "Request %s failed with error %s, message: %s",
+                path,
+                response.status_code,
+                response.text,
+            )
+            raise RuntimeError(f"Request to {path} failed with {response.status_code}")
+        return response
+
+    def get_paginated(self, path: str, params: dict = None):
+        last = False
+        page = 0
+        params = params or {}
+        while not last:
+            # TODO: Use 'nextLink'
+            params.update({"page": page, "size": self._page_size})
+            response = self.get(path, params)
+            last = response.get("last", True)
+            for element in response.get("content", []):
+                yield element
+            page += 1
+
+    def incidents(
+        self, after: datetime = None, only_unread: bool = False, query: str = None
+    ):
+        if not query:
+            conditions = []
+            if after:
+                long_timestamp = int(after.timestamp() * 1000)
+                conditions.append(f"validationDate > {long_timestamp}")
+            if only_unread:
+                conditions.append("UNREAD")
+            query = " AND ".join(conditions)
+        # DISP rejects default encoding with + as space
+        query = urlencode({"query": query}, quote_via=quote)
+
+        return self.get_paginated(f"incident/?{query}")
+
+    def download_evidence_json(self, incident_id: str, file_id: str):
+        return self.get(f"incident/{incident_id}/file/{file_id}")
+
+    def mark_incident_read(self, incident_id: str):
+        self.post("incident/read", params={"id": incident_id, "read": True})

intelmq_extensions/bots/collectors/disp/collector.py

@@ -0,0 +1,104 @@
+"""Collector for Deloitte Intelligence Service Portal
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+Connects to the DISP portal and collects selected type of incidents
+for every monitored company. Currently, we are prepared for the credential tracing
+only.
+
+Parameters:
+api_url
+auth_token
+oauth_clientid
+oauth_clientsecret
+ouath_url
+
+mark_as_read
+wait_for_evidences
+mask_password # hardcoded to true
+"""
+
+import json
+from datetime import datetime, timedelta
+
+from intelmq.lib.bot import CollectorBot
+from intelmq.lib.utils import create_request_session, parse_relative
+
+from ._client import DISPClient
+
+
+class DISPCollectorBot(CollectorBot):
+    api_url: str = ""
+    auth_token: str = ""
+    oauth_clientid: str = ""
+    oauth_clientsecret: str = ""
+    ouath_url: str = ""
+    # refresh access token when it's less than 10 minutes to expire
+    refresh_token_before: int = 10
+
+    mark_as_read: bool = False
+    wait_for_evidences: bool = True
+    not_older_than: str = "7 days"
+    # mask_password: bool = True  # hardcoded to true
+
+    def init(self):
+        self.set_request_parameters()
+        self.session = create_request_session(self)
+        self.client = DISPClient(
+            api_url=self.api_url,
+            auth_token=self.auth_token,
+            oauth_clientid=self.oauth_clientid,
+            oauth_clientsecret=self.oauth_clientsecret,
+            oauth_url=self.ouath_url,
+            session=self.session,
+            refresh_before=self.refresh_token_before,
+            logger=self.logger,
+        )
+
+    @staticmethod
+    def _mask_passwords(data):
+        """Ensure passwords will never be processed nor stored in our systems"""
+        for credentials in data.get("credentials", []):
+            password = credentials.get("password")
+            if not password:
+                continue
+            if len(password) <= 3:
+                mask = "*" * len(password)
+            else:
+                mask = f"{password[:3]}{'*' * (len(password) - 3)}"
+            credentials["password"] = mask
+
+    def process(self):
+        after = datetime.utcnow() - timedelta(
+            minutes=parse_relative(self.not_older_than)
+        )
+        for incident in self.client.incidents(after=after, only_unread=True):
+            evidences = incident.get("evidences", [])
+            expected_file = f'{incident["id"]}.json.txt'
+            evidence_file = next(
+                filter(lambda f: f.get("name") == expected_file, evidences), None
+            )
+            if self.wait_for_evidences and not evidence_file:
+                self.logger.debug(
+                    "Incident %s doesn't have an evidence file yet.", incident.get("id")
+                )
+                continue
+
+            evidence_data = None
+            if evidence_file:
+                evidence_data = self.client.download_evidence_json(
+                    incident.get("id"), evidence_file.get("idStoredFile")
+                )
+                self._mask_passwords(evidence_data)
+
+            report = self.new_report()
+            report.add(
+                "raw", json.dumps({"incident": incident, "evidences": evidence_data})
+            )
+            self.send_message(report)
+            if self.mark_as_read:
+                self.client.mark_incident_read(incident.get("id"))
+
+
+BOT = DISPCollectorBot