regscale-cli 6.21.1.0__py3-none-any.whl → 6.21.2.0__py3-none-any.whl

regscale/integrations/commercial/microsoft_defender/defender_api.py

@@ -0,0 +1,286 @@
+"""
+Module to handle API calls to Microsoft Defender for Cloud
+"""
+
+from json import JSONDecodeError
+from logging import getLogger
+from typing import Any, Literal, Optional
+
+from requests import Response
+
+from regscale.core.app.api import Api
+from regscale.core.app.utils.app_utils import error_and_exit
+from .defender_constants import APP_JSON
+
+logger = getLogger("regscale")
+
+
+class DefenderApi:
+    """
+    Class to handle API calls to Microsoft Defender 365 or Microsoft Defender for Cloud
+
+    :param Literal["cloud", "365"] system: Which system to make API calls to, either cloud or 365
+    """
+
+    def __init__(self, system: Literal["cloud", "365"]):
+        self.api: Api = Api()
+        self.config: dict = self.api.config
+        self.system: Literal["cloud", "365"] = system
+        self.headers: dict = self.set_headers()
+        self.decode_error: str = "JSON Decode error"
+        self.skip_token_key: str = "$skipToken"
+
+    def set_headers(self) -> dict:
+        """
+        Function to set the headers for the API calls
+        """
+        token = self.check_token()
+        return {"Content-Type": APP_JSON, "Authorization": token}
+
+    def get_token(self) -> str:
+        """
+        Function to get a token from Microsoft Azure and saves it to init.yaml
+
+        :return: JWT from Azure
+        :rtype: str
+        """
+        # set the url and body for request
+        if self.system == "365":
+            url = f'https://login.windows.net/{self.config["azure365TenantId"]}/oauth2/token'
+            client_id = self.config["azure365ClientId"]
+            client_secret = self.config["azure365Secret"]
+            resource = "https://api.securitycenter.windows.com"
+            key = "azure365AccessToken"
+        elif self.system == "cloud":
+            url = f'https://login.microsoftonline.com/{self.config["azureCloudTenantId"]}/oauth2/token'
+            client_id = self.config["azureCloudClientId"]
+            client_secret = self.config["azureCloudSecret"]
+            resource = "https://management.azure.com"
+            key = "azureCloudAccessToken"
+        data = {
+            "resource": resource,
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "grant_type": "client_credentials",
+        }
+        # get the data
+        response = self.api.post(
+            url=url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            data=data,
+        )
+        try:
+            return self._parse_and_save_token(response, key)
+        except KeyError as ex:
+            # notify user we weren't able to get a token and exit
+            error_and_exit(f"Didn't receive token from Azure.\n{ex}\n{response.text}")
+        except JSONDecodeError as ex:
+            # notify user we weren't able to get a token and exit
+            error_and_exit(f"Unable to authenticate with Azure.\n{ex}\n{response.text}")
+
+    def check_token(self, url: Optional[str] = None) -> str:
+        """
+        Function to check if current Azure token from init.yaml is valid, if not replace it
+
+        :param str url: The URL to use for authentication, defaults to None
+        :return: returns JWT for Microsoft 365 Defender or Microsoft Defender for Cloud depending on system provided
+        :rtype: str
+        """
+        # set up variables for the provided system
+        if self.system == "cloud":
+            key = "azureCloudAccessToken"
+        elif self.system.lower() == "365":
+            key = "azure365AccessToken"
+        else:
+            error_and_exit(
+                f"{self.system.title()} is not supported, only Microsoft 365 Defender and Microsoft Defender for Cloud."
+            )
+        current_token = self.config[key]
+        # check the token if it isn't blank
+        if current_token and url:
+            # set the headers
+            header = {"Content-Type": APP_JSON, "Authorization": current_token}
+            # test current token by getting recommendations
+            token_pass = self.api.get(url=url, headers=header)
+            # check the status code
+            if getattr(token_pass, "status_code", 0) == 200:
+                # token still valid, return it
+                token = self.config[key]
+                logger.info(
+                    "Current token for %s is still valid and will be used for future requests.",
+                    self.system.title(),
+                )
+            elif getattr(token_pass, "status_code", 0) == 403:
+                # token doesn't have permissions, notify user and exit
+                error_and_exit(
+                    "Incorrect permissions set for application. Cannot retrieve recommendations.\n"
+                    + f"{token_pass.status_code}: {token_pass.reason}\n{token_pass.text}"
+                )
+            else:
+                # token is no longer valid, get a new one
+                token = self.get_token()
+        # token is empty, get a new token
+        else:
+            token = self.get_token()
+        return token
+
+    def _parse_and_save_token(self, response: Response, key: str) -> str:
+        """
+        Function to parse the token from the response and save it to init.yaml
+
+        :param Response response: Response from API call
+        :param str key: Key to use for init.yaml token update
+        :return: JWT from Azure for the provided system
+        :rtype: str
+        """
+        # try to read the response and parse the token
+        res = response.json()
+        token = res["access_token"]
+
+        # add the token to init.yaml
+        self.config[key] = f"Bearer {token}"
+
+        # write the changes back to file
+        self.api.app.save_config(self.api.config)  # type: ignore
+
+        # notify the user we were successful
+        logger.info(
+            f"Azure {self.system.title()} Login Successful! Init.yaml file was updated with the new access token."
+        )
+        # return the token string
+        return self.config[key]
+
+    def execute_resource_graph_query(
+        self, query: str = None, skip_token: Optional[str] = None, record_count: int = 0
+    ) -> list[dict]:
+        """
+        Function to fetch Microsoft Defender resources from Azure
+
+        :param str query: Query to use for the API call
+        :param Optional[str] skip_token: Token to skip results, used during pagination, defaults to None
+        :param int record_count: Number of records fetched, defaults to 0, used for logging during pagination
+        :return: list of Microsoft Defender resources
+        :rtype: list[dict]
+        """
+        url = "https://management.azure.com/providers/Microsoft.ResourceGraph/resources?api-version=2024-04-01"
+        if query:
+            payload: dict[str, Any] = {"query": query}
+        else:
+            payload: dict[str, Any] = {
+                "query": query,
+                "subscriptions": [self.config["azureCloudSubscriptionId"]],
+            }
+        if skip_token:
+            payload["options"] = {self.skip_token_key: skip_token}
+            logger.info("Retrieving more Microsoft Defender resources from Azure...")
+        else:
+            logger.info("Retrieving Microsoft Defender resources from Azure...")
+        response = self.api.post(url=url, headers=self.headers, json=payload)
+        if response.status_code != 200:
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}:{response.reason}"
+                + f"\n{response.text}",
+            )
+        try:
+            response_data = response.json()
+            total_records = response_data.get("totalRecords", 0)
+            count = response_data.get("count", len(response_data.get("data", [])))
+            logger.info(f"Received {count + record_count}/{total_records} items from Microsoft Defender.")
+            # try to get the values from the api response
+            defender_data = response_data["data"]
+        except JSONDecodeError:
+            # notify user if there was a json decode error from API response and exit
+            error_and_exit(self.decode_error)
+        except KeyError:
+            # notify user there was no data from API response and exit
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}: {response.reason}\n"
+                + f"{response.text}"
+            )
+        # check if pagination is required to fetch all data from Microsoft Defender
+        skip_token = response_data.get(self.skip_token_key)
+        if response.status_code == 200 and skip_token:
+            # get the rest of the data
+            defender_data.extend(
+                self.execute_resource_graph_query(query=query, skip_token=skip_token, record_count=count + record_count)
+            )
+        # return the defender recommendations
+        return defender_data
+
+    def get_items_from_azure(self, url: str) -> list:
+        """
+        Function to get data from Microsoft Defender returns the data as a list while handling pagination
+
+        :param str url: URL to use for the API call
+        :return: list of recommendations
+        :rtype: list
+        """
+        # get the data via api call
+        response = self.api.get(url=url, headers=self.headers)
+        if response.status_code != 200:
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}:{response.reason}"
+                + f"\n{response.text}",
+            )
+        # try to read the response
+        try:
+            response_data = response.json()
+            # try to get the values from the api response
+            defender_data = response_data["value"]
+        except JSONDecodeError:
+            # notify user if there was a json decode error from API response and exit
+            error_and_exit(self.decode_error)
+        except KeyError:
+            # notify user there was no data from API response and exit
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}: {response.text}"
+            )
+        # check if pagination is required to fetch all data from Microsoft Defender
+        if next_link := response_data.get("nextLink"):
+            # get the rest of the data
+            defender_data.extend(self.get_items_from_azure(url=next_link))
+        # return the defender recommendations
+        return defender_data
+
+    def fetch_queries_from_azure(self) -> list[dict]:
+        """
+        Function to fetch queries from Microsoft Defender for Cloud
+        """
+        url = (
+            f"https://management.azure.com/subscriptions/{self.config['azureCloudSubscriptionId']}/"
+            "providers/Microsoft.ResourceGraph/queries?api-version=2024-04-01"
+        )
+        logger.info("Fetching saved queries from Azure Resource Graph...")
+        response = self.api.get(url=url, headers=self.headers)
+        logger.debug(f"Azure API response status: {response.status_code}")
+        if response.raise_for_status():
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}:{response.reason}"
+                + f"\n{response.text}",
+            )
+        logger.debug("Parsing Azure API response...")
+        cloud_queries = response.json().get("value", [])
+        logger.info(f"Found {len(cloud_queries)} saved queries in Azure Resource Graph.")
+        return cloud_queries
+
+    def fetch_and_run_query(self, query: dict) -> list[dict]:
+        """
+        Function to fetch and run a query from Microsoft Defender for Cloud
+
+        :param dict query: Query to run in Azure Resource Graph
+        :return: Results from the query
+        :rtype: list[dict]
+        """
+        url = (
+            f"https://management.azure.com/subscriptions/{query['subscriptionId']}/resourceGroups/"
+            f"{query['resourceGroup']}/providers/Microsoft.ResourceGraph/queries/{query['name']}"
+            "?api-version=2024-04-01"
+        )
+        response = self.api.get(url=url, headers=self.headers)
+        if response.raise_for_status():
+            error_and_exit(
+                f"Received unexpected response from Microsoft Defender.\n{response.status_code}:{response.reason}"
+                + f"\n{response.text}",
+            )
+        query_string = response.json().get("properties", {}).get("query")
+        return self.execute_resource_graph_query(query=query_string)

regscale/integrations/commercial/microsoft_defender/defender_constants.py

@@ -0,0 +1,80 @@
+"""
+Module to store constants for Microsoft Defender for Cloud
+"""
+
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
+IDENTIFICATION_TYPE = "Vulnerability Assessment"
+CLOUD_RECS = "Microsoft Defender for Cloud Recommendation"
+APP_JSON = "application/json"
+AFD_ENDPOINTS = "microsoft.cdn/profiles/afdendpoints"
+
+
+RESOURCES_QUERY = """
+resources
+| where subscriptionId == "{SUBSCRIPTION_ID}"
+| extend resourceName = name,
+        resourceType = type,
+        resourceLocation = location,
+        resourceGroup = resourceGroup,
+        resourceId = id,
+        propertiesJson = parse_json(properties)
+| extend ipAddress =
+   case(
+       resourceType =~ "microsoft.network/networkinterfaces", tostring(propertiesJson.ipConfigurations[0].properties.privateIPAddress),
+       resourceType =~ "microsoft.network/publicipaddresses", tostring(propertiesJson.ipAddress),
+       resourceType =~ "microsoft.compute/virtualmachines", tostring(propertiesJson.networkProfile.networkInterfaces[0].id),
+       ""
+   )
+| project resourceName, resourceType, resourceLocation, resourceGroup, resourceId, ipAddress, properties
+"""
+
+CONTAINER_SCAN_QUERY = """
+securityresources
+| where type == 'microsoft.security/assessments'
+| summarize by assessmentKey=name
+| join kind=inner (
+    securityresources
+    | where type == 'microsoft.security/assessments/subassessments'
+    | extend assessmentKey = extract('.*assessments/(.+?)/.*', 1, id)
+    | where resourceGroup == '{RESOURCE_GROUP}'
+) on assessmentKey
+| project assessmentKey, subassessmentKey=name, id, parse_json(properties), resourceGroup, subscriptionId, tenantId
+| extend description = properties.description,
+    displayName = properties.displayName,
+    resourceId = properties.resourceDetails.id,
+    tag = properties.additionalData.artifactDetails.tags,
+    resourceSource = properties.resourceDetails.source,
+    category = properties.category,
+    severity = properties.status.severity,
+    code = properties.status.code,
+    timeGenerated = properties.timeGenerated,
+    remediation = properties.remediation,
+    impact = properties.impact,
+    vulnId = properties.id,
+    additionalData = properties.additionalData
+    | where resourceId startswith "/subscriptions"
+| order by ['id'] asc
+"""
+
+DB_SCAN_QUERY = """
+securityresources
+| where type =~ "microsoft.security/assessments/subassessments"
+| extend assessmentKey=extract(@"(?i)providers/Microsoft.Security/assessments/([^/]*)", 1, id), subAssessmentId=tostring(properties.id), parentResourceId= extract("(.+)/providers/Microsoft.Security", 1, id)
+| extend resourceIdTemp = iff(properties.resourceDetails.id != "", properties.resourceDetails.id, extract("(.+)/providers/Microsoft.Security", 1, id))
+| extend resourceId = iff(properties.resourceDetails.source =~ "OnPremiseSql", strcat(resourceIdTemp, "/servers/", properties.resourceDetails.serverName, "/databases/" , properties.resourceDetails.databaseName), resourceIdTemp)
+| where assessmentKey =~ "{ASSESSMENT_KEY}"
+| where subscriptionId == "{SUBSCRIPTION_ID}"
+| project assessmentKey,
+    subAssessmentId,
+    resourceId,
+    name=properties.displayName,
+    description=properties.description,
+    severity=properties.status.severity,
+    status=properties.status.code,
+    cause=properties.status.cause,
+    category=properties.category,
+    impact=properties.impact,
+    remediation=properties.remediation,
+    benchmarks=properties.additionalData.benchmarks
+| where status == "Unhealthy"
+"""

regscale/integrations/commercial/microsoft_defender/defender_scanner.py

@@ -0,0 +1,168 @@
+"""
+Microsoft Defender for Cloud Scanner Integration
+"""
+
+import logging
+from typing import Iterator
+
+from regscale.integrations.commercial.microsoft_defender.defender_api import DefenderApi
+from regscale.integrations.commercial.microsoft_defender.defender_constants import RESOURCES_QUERY, AFD_ENDPOINTS
+from regscale.integrations.scanner_integration import IntegrationAsset, IntegrationFinding, ScannerIntegration
+from regscale.models import IssueSeverity
+from regscale.utils.string import generate_html_table_from_dict
+
+logger = logging.getLogger("regscale")
+
+
+class DefenderScanner(ScannerIntegration):
+    title = "Microsoft Defender for Cloud"
+    # Required fields from ScannerIntegration
+    asset_identifier_field = "otherTrackingNumber"
+    finding_severity_map = {
+        "Critical": IssueSeverity.Critical,
+        "High": IssueSeverity.High,
+        "Medium": IssueSeverity.Moderate,
+        "Low": IssueSeverity.Low,
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.system = kwargs.pop("system", "cloud")
+        super().__init__(*args, **kwargs)
+        self.api = DefenderApi(system=self.system)  # type: ignore
+
+    def fetch_assets(self, *args, **kwargs) -> Iterator[IntegrationAsset]:
+        """
+        Fetches assets from Synqly
+
+        :yields: Iterator[IntegrationAsset]
+        """
+        cloud_resources = self.api.execute_resource_graph_query(
+            query=RESOURCES_QUERY.format(SUBSCRIPTION_ID=self.api.config["azureCloudSubscriptionId"])
+        )
+        self.num_assets_to_process = len(cloud_resources)
+        for asset in cloud_resources:
+            yield self.parse_asset(asset)
+
+    def fetch_findings(self, *args, **kwargs) -> Iterator[IntegrationFinding]:
+        """
+        Fetches findings from the Synqly
+
+        :yields: Iterator[IntegrationFinding]
+        """
+        integration_findings = kwargs.get("integration_findings")
+        for finding in integration_findings:
+            yield finding
+
+    def parse_asset(self, defender_asset: dict) -> IntegrationAsset:
+        """
+        Function to map data to an Asset object
+
+        :param defender_asset: Data from Microsoft Defender for Cloud
+        :return: IntegrationAsset object that is parsed from the defender_asset
+        :rtype: IntegrationAsset
+        """
+        asset_id = defender_asset.get("resourceId")
+        properties = defender_asset.get("properties", {})
+        resource_type = defender_asset.get("resourceType", "").lower()
+        try:
+            ip_mapping = {
+                "microsoft.network/networksecuritygroups": properties.get("securityRules", [{}])[0]
+                .get("properties", {})
+                .get("destinationAddressPrefix"),
+                "microsoft.network/virtualnetworks": properties.get("addressSpace", {}).get("addressPrefixes"),
+                "microsoft.app/managedenvironments": properties.get("staticIp"),
+                "microsoft.network/networkinterfaces": properties.get("ipConfigurations", [{}])[0]
+                .get("properties", {})
+                .get("privateIPAddress"),
+            }
+        except IndexError:
+            ip_mapping = {}
+        try:
+            fqdn_mapping = {
+                "microsoft.keyvault/vaults": properties.get("vaultUri"),
+                "microsoft.storage/storageaccounts": properties.get("primaryEndpoints", {}).get("blob"),
+                "microsoft.appconfiguration/configurationstores": properties.get("endpoint"),
+                "microsoft.dbforpostgresql/flexibleservers": properties.get("fullyQualifiedDomainName"),
+                AFD_ENDPOINTS: properties.get("hostName"),
+                "microsoft.containerregistry/registries": properties.get("loginServer"),
+                "microsoft.app/containerapps": properties.get("configuration", {}).get("ingress", {}).get("fqdn"),
+                "microsoft.network/privatednszones": defender_asset.get("name") or defender_asset.get("resourceName"),
+                "microsoft.cognitiveservices/accounts": properties.get("endpoint"),
+            }
+        except IndexError:
+            fqdn_mapping = {}
+        # pylint: disable=line-too-long
+        function_mapping = {
+            "microsoft.network/privateendpoints": "Private endpoint that links the private link and the nic together",
+            "microsoft.network/networkinterfaces": "Network Interface that connects to everything internal to the resource group",
+            "microsoft.network/privatednszones": "Dns zone that will connect to the private endpoint and network interfaces",
+            "microsoft.network/privatednszones/virtualnetworklinks": "Link for the Private DNS zone back to the vnet",
+            "microsoft.app/containerapps": "Application runner that houses the running Docker Container",
+            "microsoft.network/publicipaddresses": "Public ip address used for load balancing the container apps",
+            "microsoft.storage/storageaccounts": "Storage blob to house unstructured files uploaded to the platform",
+            "microsoft.network/networksecuritygroups": "Network protection for internal communications and load balancing",
+            "microsoft.network/networkwatchers/flowlogs": "Logs that determine the flow of traffic",
+            "microsoft.sql/servers/databases": "Database that houses application logs",
+            "microsoft.network/virtualnetworks": "Network Interface that determines what the valid IP range is for all internal resources",
+            "microsoft.portal/dashboards": "Dashboard that shows the status of the application and traffic",
+            "microsoft.dataprotection/backupvaults": "Azure Blob Storage Account backup location",
+            "microsoft.keyvault/vaults": "To securely store API keys, passwords, certificates, or cryptographic keys",
+            "microsoft.managedidentity/userassignedidentities": "Identity that connects all internal resources in the resource group",
+            "microsoft.app/managedenvironments": "Application environment to connect to the vnet",
+            "microsoft.sql/servers": "Server that will house the database for the application logs",
+            "microsoft.sql/servers/encryptionprotector": "Server encryption",
+            "microsoft.appconfiguration/configurationstores": "Configure, store, and retrieve parameters and settings. Store configuration for all system components in the environment",
+            "microsoft.insights/metricalerts": "Alerts that trigger when exceptions hit above 100",
+            "microsoft.insights/webtests": "Test to ensure the integrity of the app and alert when availability drops",
+            "microsoft.insights/components": "Insights and mapping for the data flow through the platform container application",
+            "microsoft.dbforpostgresql/flexibleservers": "Application Database for OpenAI and Automation containers",
+            "microsoft.network/loadbalancers": "Load Balancer that handles the load traffic for the containerapp",
+            "microsoft.insights/activitylogalerts": "Alert rule to send an email to the Action Group when the trigger event happens",
+            "microsoft.operationalinsights/workspaces": "Collection of Logs contained in a workspace",
+            "microsoft.insights/actiongroups": "Action Group to send Emails to when alerts trigger",
+            "microsoft.network/networkwatchers": "Monitor on the network to look for any suspecious activity",
+            "microsoft.app/managedenvironments/certificates": "Tls cert for the application environment",
+            "microsoft.authorization/roledefinitions": "Custom role definition",
+            "microsoft.alertsmanagement/actionrules": "Alert Processing Rule to show when to trigger",
+            "microsoft.network/frontdoorwebapplicationfirewallpolicies": "Waf protection policy that connects to the firewall and frontdoor",
+            "microsoft.cdn/profiles": "Monitoring and controlling inbound and outbound traffic to the environment. Functions as a Web Application Firewall (WAF) and performs Network Address Translation (NAT) connecting public networks to a series of private tenant Virtual Networks (VNets)",
+            "microsoft.resourcegraph/queries": "Query to return all resources in the SaaS subscription in the resource graph",
+            "microsoft.network/firewallpolicies": "Firewall policy that connects to frontdoor and handles our traffic coming into the system",
+            AFD_ENDPOINTS: "Endpoint that all of the routes attach to",
+            "microsoft.containerregistry/registries": "House the Docker container image for ContainerApp pull",
+            "microsoft.operationalinsights/querypacks": "Log analytics query that loads default queries for running",
+            "microsoft.alertsmanagement/smartdetectoralertrules": "Failure Anomalies notifies you of an unusual rise in the rate of failed HTTP requests or dependency calls.",
+        }
+        # pylint: enable=line-too-long
+        from regscale.models import regscale_models
+
+        mapped_asset = IntegrationAsset(
+            name=defender_asset.get("resourceName", asset_id),
+            description=generate_html_table_from_dict(defender_asset),
+            other_tracking_number=asset_id,
+            azure_identifier=asset_id,
+            external_id=asset_id,
+            identifier=asset_id,
+            asset_type=regscale_models.AssetType.Other,
+            asset_category=regscale_models.AssetCategory.Software,
+            parent_id=self.plan_id,
+            parent_module=(
+                regscale_models.Component.get_module_slug()
+                if self.is_component
+                else regscale_models.SecurityPlan.get_module_slug()
+            ),
+            status=regscale_models.AssetStatus.Active,
+            software_function=function_mapping.get(resource_type, properties.get("description")),
+            ip_address=defender_asset.get("ipAddress") or ip_mapping.get(resource_type, properties.get("ipAddress")),
+            is_public_facing=resource_type in ["microsoft.cdn/profiles", AFD_ENDPOINTS],
+            is_authenticated_scan=resource_type
+            not in ["microsoft.alertsmanagement/actionrules", "microsoft.alertsmanagement/smartdetectoralertrules"],
+            is_virtual=True,
+            baseline_configuration="Azure Hardening Guide",
+            component_names=[resource_type],
+            source_data=defender_asset,
+        )
+        if fqdn := fqdn_mapping.get(resource_type, properties.get("dnsSettings", {}).get("fqdn")):
+            mapped_asset.fqdn = fqdn
+            mapped_asset.description += f"<p>FQDN: {fqdn}</p>"
+        return mapped_asset