cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/azure/subscription.py

@@ -7,10 +7,11 @@ import neo4j
 from azure.core.exceptions import HttpResponseError
 from azure.mgmt.resource import SubscriptionClient
 
-from .util.credentials import Credentials
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
+from .util.credentials import Credentials
+
 logger = logging.getLogger(__name__)
 
 
@@ -24,26 +25,31 @@ def get_all_azure_subscriptions(credentials: Credentials) -> List[Dict]:
 
     except HttpResponseError as e:
         logger.error(
-            f'failed to fetch subscriptions for the credentials \
+            f"failed to fetch subscriptions for the credentials \
             The provided credentials do not have access to any subscriptions - \
-            {e}',
+            {e}",
         )
 
         return []
 
     subscriptions = []
     for sub in subs:
-        subscriptions.append({
-            'id': sub.id,
-            'subscriptionId': sub.subscription_id,
-            'displayName': sub.display_name,
-            'state': sub.state,
-        })
+        subscriptions.append(
+            {
+                "id": sub.id,
+                "subscriptionId": sub.subscription_id,
+                "displayName": sub.display_name,
+                "state": sub.state,
+            },
+        )
 
     return subscriptions
 
 
-def get_current_azure_subscription(credentials: Credentials, subscription_id: Optional[str]) -> List[Dict]:
+def get_current_azure_subscription(
+    credentials: Credentials,
+    subscription_id: Optional[str],
+) -> List[Dict]:
     try:
         # Create the client
         client = SubscriptionClient(credentials.arm_credentials)
@@ -53,25 +59,28 @@ def get_current_azure_subscription(credentials: Credentials, subscription_id: Op
 
     except HttpResponseError as e:
         logger.error(
-            f'failed to fetch subscription for the credentials \
+            f"failed to fetch subscription for the credentials \
             The provided credentials do not have access to this subscription: {subscription_id} - \
-            {e}',
+            {e}",
         )
 
         return []
 
     return [
         {
-            'id': sub.id,
-            'subscriptionId': sub.subscription_id,
-            'displayName': sub.display_name,
-            'state': sub.state,
+            "id": sub.id,
+            "subscriptionId": sub.subscription_id,
+            "displayName": sub.display_name,
+            "state": sub.state,
         },
     ]
 
 
 def load_azure_subscriptions(
-    neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    subscriptions: List[Dict],
+    update_tag: int,
 ) -> None:
     query = """
     MERGE (at:AzureTenant{id: $TENANT_ID})
@@ -90,21 +99,28 @@ def load_azure_subscriptions(
         neo4j_session.run(
             query,
             TENANT_ID=tenant_id,
-            SUBSCRIPTION_ID=sub['subscriptionId'],
-            SUBSCRIPTION_PATH=sub['id'],
-            SUBSCRIPTION_NAME=sub['displayName'],
-            SUBSCRIPTION_STATE=sub['state'],
+            SUBSCRIPTION_ID=sub["subscriptionId"],
+            SUBSCRIPTION_PATH=sub["id"],
+            SUBSCRIPTION_NAME=sub["displayName"],
+            SUBSCRIPTION_STATE=sub["state"],
             update_tag=update_tag,
         )
 
 
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('azure_subscriptions_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job(
+        "azure_subscriptions_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    subscriptions: List[Dict],
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     load_azure_subscriptions(neo4j_session, tenant_id, subscriptions, update_tag)

cartography/intel/azure/tenant.py

@@ -3,10 +3,11 @@ from typing import Dict
 
 import neo4j
 
-from .util.credentials import Credentials
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
+from .util.credentials import Credentials
+
 logger = logging.getLogger(__name__)
 
 
@@ -14,7 +15,12 @@ def get_tenant_id(credentials: Credentials) -> str:
     return credentials.get_tenant_id()
 
 
-def load_azure_tenant(neo4j_session: neo4j.Session, tenant_id: str, current_user: str, update_tag: int) -> None:
+def load_azure_tenant(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    current_user: str,
+    update_tag: int,
+) -> None:
     query = """
     MERGE (at:AzureTenant{id: $TENANT_ID})
     ON CREATE SET at.firstseen = timestamp()
@@ -37,12 +43,15 @@ def load_azure_tenant(neo4j_session: neo4j.Session, tenant_id: str, current_user
 
 
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('azure_tenant_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job("azure_tenant_cleanup.json", neo4j_session, common_job_parameters)
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, tenant_id: str, current_user: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    current_user: str,
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     load_azure_tenant(neo4j_session, tenant_id, current_user, update_tag)

cartography/intel/azure/util/credentials.py

@@ -13,22 +13,24 @@ from azure.identity import ClientSecretCredential
 from msrestazure.azure_active_directory import AADTokenCredentials
 
 logger = logging.getLogger(__name__)
-AUTHORITY_HOST_URI = 'https://login.microsoftonline.com'
+AUTHORITY_HOST_URI = "https://login.microsoftonline.com"
 
 
 class Credentials:
 
     def __init__(
-            self,
-            arm_credentials: Any,
-            aad_graph_credentials: Any,
-            tenant_id: Optional[str] = None,
-            subscription_id: Optional[str] = None,
-            context: Optional[adal.AuthenticationContext] = None,
-            current_user: Optional[str] = None,
+        self,
+        arm_credentials: Any,
+        aad_graph_credentials: Any,
+        tenant_id: Optional[str] = None,
+        subscription_id: Optional[str] = None,
+        context: Optional[adal.AuthenticationContext] = None,
+        current_user: Optional[str] = None,
     ) -> None:
         self.arm_credentials = arm_credentials  # Azure Resource Manager API credentials
-        self.aad_graph_credentials = aad_graph_credentials  # Azure AD Graph API credentials
+        self.aad_graph_credentials = (
+            aad_graph_credentials  # Azure AD Graph API credentials
+        )
         self.tenant_id = tenant_id
         self.subscription_id = subscription_id
         self.context = context
@@ -40,35 +42,46 @@ class Credentials:
     def get_tenant_id(self) -> Any:
         if self.tenant_id:
             return self.tenant_id
-        elif 'tenant_id' in self.aad_graph_credentials.token:
-            return self.aad_graph_credentials.token['tenant_id']
+        elif "tenant_id" in self.aad_graph_credentials.token:
+            return self.aad_graph_credentials.token["tenant_id"]
         else:
             # This is a last resort, e.g. for MSI authentication
             try:
-                h = {'Authorization': 'Bearer {}'.format(self.arm_credentials.token['access_token'])}
-                r = requests.get('https://management.azure.com/tenants?api-version=2020-01-01', headers=h)
+                h = {
+                    "Authorization": "Bearer {}".format(
+                        self.arm_credentials.token["access_token"],
+                    ),
+                }
+                r = requests.get(
+                    "https://management.azure.com/tenants?api-version=2020-01-01",
+                    headers=h,
+                )
                 r2 = r.json()
-                return r2.get('value')[0].get('tenantId')
+                return r2.get("value")[0].get("tenantId")
             except requests.ConnectionError as e:
-                logger.error(f'Unable to infer tenant ID: {e}')
+                logger.error(f"Unable to infer tenant ID: {e}")
                 return None
 
     def get_credentials(self, resource: str) -> Any:
-        if resource == 'arm':
+        if resource == "arm":
             self.arm_credentials = self.get_fresh_credentials(self.arm_credentials)
             return self.arm_credentials
-        elif resource == 'aad_graph':
-            self.aad_graph_credentials = self.get_fresh_credentials(self.aad_graph_credentials)
+        elif resource == "aad_graph":
+            self.aad_graph_credentials = self.get_fresh_credentials(
+                self.aad_graph_credentials,
+            )
             return self.aad_graph_credentials
         else:
-            raise Exception('Invalid credentials resource type')
+            raise Exception("Invalid credentials resource type")
 
     def get_fresh_credentials(self, credentials: Any) -> Any:
         """
         Check if credentials are outdated and if so refresh them.
         """
-        if self.context and hasattr(credentials, 'token'):
-            expiration_datetime = datetime.fromtimestamp(credentials.token['expires_on'])
+        if self.context and hasattr(credentials, "token"):
+            expiration_datetime = datetime.fromtimestamp(
+                credentials.token["expires_on"],
+            )
             current_datetime = datetime.now()
             expiration_delta = expiration_datetime - current_datetime
             if expiration_delta < timedelta(minutes=5):
@@ -79,8 +92,8 @@ class Credentials:
         """
         Refresh credentials
         """
-        logger.debug('Refreshing credentials')
-        authority_uri = AUTHORITY_HOST_URI + '/' + self.get_tenant_id()
+        logger.debug("Refreshing credentials")
+        authority_uri = AUTHORITY_HOST_URI + "/" + self.get_tenant_id()
         if self.context:
             existing_cache = self.context.cache
             context = adal.AuthenticationContext(authority_uri, cache=existing_cache)
@@ -89,10 +102,15 @@ class Credentials:
             context = adal.AuthenticationContext(authority_uri)
 
         new_token = context.acquire_token(
-            credentials.token['resource'], credentials.token['user_id'], credentials.token['_client_id'],
+            credentials.token["resource"],
+            credentials.token["user_id"],
+            credentials.token["_client_id"],
         )
 
-        new_credentials = AADTokenCredentials(new_token, credentials.token.get('_client_id'))
+        new_credentials = AADTokenCredentials(
+            new_token,
+            credentials.token.get("_client_id"),
+        )
         return new_credentials
 
 
@@ -105,40 +123,54 @@ class Authenticator:
         try:
 
             # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger('adal-python').setLevel(logging.ERROR)
-            logging.getLogger('msrest').setLevel(logging.ERROR)
-            logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR)
-            logging.getLogger('urllib3').setLevel(logging.ERROR)
-            logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR)
-
-            arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True)
-            aad_graph_credentials, placeholder_1, placeholder_2 = get_azure_cli_credentials(
-                with_tenant=True, resource='https://graph.windows.net',
+            logging.getLogger("adal-python").setLevel(logging.ERROR)
+            logging.getLogger("msrest").setLevel(logging.ERROR)
+            logging.getLogger("msrestazure.azure_active_directory").setLevel(
+                logging.ERROR,
+            )
+            logging.getLogger("urllib3").setLevel(logging.ERROR)
+            logging.getLogger(
+                "azure.core.pipeline.policies.http_logging_policy",
+            ).setLevel(logging.ERROR)
+
+            arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(
+                with_tenant=True,
+            )
+            aad_graph_credentials, placeholder_1, placeholder_2 = (
+                get_azure_cli_credentials(
+                    with_tenant=True,
+                    resource="https://graph.windows.net",
+                )
             )
 
             profile = get_cli_profile()
 
             return Credentials(
-                arm_credentials, aad_graph_credentials, tenant_id=tenant_id,
-                current_user=profile.get_current_account_user(), subscription_id=subscription_id,
+                arm_credentials,
+                aad_graph_credentials,
+                tenant_id=tenant_id,
+                current_user=profile.get_current_account_user(),
+                subscription_id=subscription_id,
             )
 
         except HttpResponseError as e:
-            if ', AdalError: Unsupported wstrust endpoint version. ' \
-                    'Current supported version is wstrust2005 or wstrust13.' in e.args:
+            if (
+                ", AdalError: Unsupported wstrust endpoint version. "
+                "Current supported version is wstrust2005 or wstrust13." in e.args
+            ):
                 logger.error(
-                    f'You are likely authenticating with a Microsoft Account. \
+                    f"You are likely authenticating with a Microsoft Account. \
                     This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}',
+                    {e}",
                 )
 
             raise e
 
     def authenticate_sp(
-            self,
-            tenant_id: Optional[str] = None,
-            client_id: Optional[str] = None,
-            client_secret: Optional[str] = None,
+        self,
+        tenant_id: Optional[str] = None,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
     ) -> Credentials:
         """
         Implements authentication for the Azure provider
@@ -146,11 +178,15 @@ class Authenticator:
         try:
 
             # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger('adal-python').setLevel(logging.ERROR)
-            logging.getLogger('msrest').setLevel(logging.ERROR)
-            logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR)
-            logging.getLogger('urllib3').setLevel(logging.ERROR)
-            logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR)
+            logging.getLogger("adal-python").setLevel(logging.ERROR)
+            logging.getLogger("msrest").setLevel(logging.ERROR)
+            logging.getLogger("msrestazure.azure_active_directory").setLevel(
+                logging.ERROR,
+            )
+            logging.getLogger("urllib3").setLevel(logging.ERROR)
+            logging.getLogger(
+                "azure.core.pipeline.policies.http_logging_policy",
+            ).setLevel(logging.ERROR)
 
             arm_credentials = ClientSecretCredential(
                 client_id=client_id,
@@ -162,21 +198,25 @@ class Authenticator:
                 client_id=client_id,
                 client_secret=client_secret,
                 tenant_id=tenant_id,
-                resource='https://graph.windows.net',
+                resource="https://graph.windows.net",
             )
 
             return Credentials(
-                arm_credentials, aad_graph_credentials, tenant_id=tenant_id,
+                arm_credentials,
+                aad_graph_credentials,
+                tenant_id=tenant_id,
                 current_user=client_id,
             )
 
         except HttpResponseError as e:
-            if ', AdalError: Unsupported wstrust endpoint version. ' \
-                    'Current supported version is wstrust2005 or wstrust13.' in e.args:
+            if (
+                ", AdalError: Unsupported wstrust endpoint version. "
+                "Current supported version is wstrust2005 or wstrust13." in e.args
+            ):
                 logger.error(
-                    f'You are likely authenticating with a Microsoft Account. \
+                    f"You are likely authenticating with a Microsoft Account. \
                     This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}',
+                    {e}",
                 )
 
             raise e

cartography/intel/bigfix/__init__.py

@@ -20,8 +20,8 @@ def start_bigfix_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
 
     if not config.bigfix_username or not config.bigfix_password:
         logger.info(
-            'BigFix import is not configured - skipping this module. '
-            'See docs to configure.',
+            "BigFix import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return
 

cartography/intel/bigfix/computers.py

@@ -22,49 +22,49 @@ logger = logging.getLogger(__name__)
 _TIMEOUT = (60, 60)
 
 DEFAULT_SUPPORTED_KEYS = {
-    'Active Directory Path',
-    'Agent Type',
-    'Agent Version',
-    'Average Evaluation Cycle',
-    'BES Relay Selection Method',
-    'BES Root Server',
-    'BIOS',
-    'Computer Name',
-    'Computer Type',
-    'CPU',
-    'Device Type',
-    'Distance to BES Relay',
-    'DNS Name',
-    'Enrollment Date',
-    'Free Space on System Drive',
-    'ID',
-    'IP Address',
-    'IPv6 Address',
-    'Last Report Time',
-    'Location By IP Range',
-    'Locked',
-    'Logged on User',
-    'MAC Address',
-    'OS',
-    'Provider Name',
-    'RAM',
-    'Relay',
-    'Remote Desktop Enabled',
-    'Subnet Address',
-    'Total Size of System Drive',
-    'User Name',
+    "Active Directory Path",
+    "Agent Type",
+    "Agent Version",
+    "Average Evaluation Cycle",
+    "BES Relay Selection Method",
+    "BES Root Server",
+    "BIOS",
+    "Computer Name",
+    "Computer Type",
+    "CPU",
+    "Device Type",
+    "Distance to BES Relay",
+    "DNS Name",
+    "Enrollment Date",
+    "Free Space on System Drive",
+    "ID",
+    "IP Address",
+    "IPv6 Address",
+    "Last Report Time",
+    "Location By IP Range",
+    "Locked",
+    "Logged on User",
+    "MAC Address",
+    "OS",
+    "Provider Name",
+    "RAM",
+    "Relay",
+    "Remote Desktop Enabled",
+    "Subnet Address",
+    "Total Size of System Drive",
+    "User Name",
 }
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        bigfix_root_url: str,
-        bigfix_username: str,
-        bigfix_password: str,
-        update_tag: int,
-        common_job_parameters: Dict[str, Any],
-        computer_keys: Optional[Set[str]] = None,
+    neo4j_session: neo4j.Session,
+    bigfix_root_url: str,
+    bigfix_username: str,
+    bigfix_password: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+    computer_keys: Optional[Set[str]] = None,
 ) -> None:
     if not computer_keys:
         computer_keys = DEFAULT_SUPPORTED_KEYS
@@ -75,36 +75,50 @@ def sync(
 
 
 @timeit
-def get(bigfix_api_url: str, headers: Dict[str, str], computer_keys: Set[str]) -> List[Dict[str, Any]]:
+def get(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_keys: Set[str],
+) -> List[Dict[str, Any]]:
     result = []
     computer_list = get_computer_list(bigfix_api_url, headers)
     logger.info(f"Retrieving details for {len(computer_list)} BigFix Computers")
     for computer in computer_list:
-        details = get_computer_details(bigfix_api_url, headers, computer['ID'], computer_keys)
+        details = get_computer_details(
+            bigfix_api_url,
+            headers,
+            computer["ID"],
+            computer_keys,
+        )
         processed_comp: Dict[str, Any] = computer.copy()
 
         # Property names have spaces. Neo4j properties can't have spaces so let's clean this up.
         for key, val in details.items():
-            transformed_key = key.replace(' ', '')
+            transformed_key = key.replace(" ", "")
             processed_comp[transformed_key] = val
 
         # Post-processing transforms
-        processed_comp['EnrollmentDateTime'] = datetime.strptime(
-            processed_comp['EnrollmentDate'],
+        processed_comp["EnrollmentDateTime"] = datetime.strptime(
+            processed_comp["EnrollmentDate"],
             "%a, %d %b %Y %H:%M:%S %z",
         )
-        processed_comp['LastReportDateTime'] = datetime.strptime(
-            processed_comp['LastReportTime'],
+        processed_comp["LastReportDateTime"] = datetime.strptime(
+            processed_comp["LastReportTime"],
             "%a, %d %b %Y %H:%M:%S %z",
         )
-        processed_comp['RemoteDesktopIsEnabled'] = processed_comp['RemoteDesktopEnabled'] == 'True'
-        processed_comp['IsLocked'] = processed_comp['Locked'] == 'Yes'
+        processed_comp["RemoteDesktopIsEnabled"] = (
+            processed_comp["RemoteDesktopEnabled"] == "True"
+        )
+        processed_comp["IsLocked"] = processed_comp["Locked"] == "Yes"
 
         result.append(processed_comp)
     return result
 
 
-def get_computer_list(bigfix_api_url: str, headers: Dict[str, str]) -> List[Dict[str, str]]:
+def get_computer_list(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+) -> List[Dict[str, str]]:
     """
     Returned shape: [
       {'@Resource': 'https://{URI}/api/computer/123', 'LastReportTime': 'Tue, 18 Apr 2023 21:59:44 +0000', 'ID': '123'},
@@ -112,22 +126,22 @@ def get_computer_list(bigfix_api_url: str, headers: Dict[str, str]) -> List[Dict
     """
     xml_text = _get_computer_list_raw_xml(bigfix_api_url, headers)
     as_dict = xmltodict.parse(xml_text)
-    return as_dict['BESAPI']['Computer']
+    return as_dict["BESAPI"]["Computer"]
 
 
 def get_computer_details(
-        bigfix_api_url: str,
-        headers: Dict[str, str],
-        computer_id: str,
-        computer_keys: Set[str],
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_id: str,
+    computer_keys: Set[str],
 ) -> Dict[str, Any]:
     xml_text = _get_computer_details_raw_xml(bigfix_api_url, headers, computer_id)
     as_dict = xmltodict.parse(xml_text)
     processed_computer = {}
-    for prop in as_dict['BESAPI']['Computer']['Property']:
-        prop_name = prop['@Name']
+    for prop in as_dict["BESAPI"]["Computer"]["Property"]:
+        prop_name = prop["@Name"]
         if prop_name in computer_keys:
-            processed_computer[prop_name] = prop['#text']
+            processed_computer[prop_name] = prop["#text"]
     return processed_computer
 
 
@@ -138,24 +152,33 @@ def _get_computer_list_raw_xml(bigfix_api_url: str, headers: Dict[str, str]) ->
     return resp.text
 
 
-def _get_computer_details_raw_xml(bigfix_api_url: str, headers: Dict[str, str], computer_id: str) -> str:
+def _get_computer_details_raw_xml(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_id: str,
+) -> str:
     details_endpoint = f"{bigfix_api_url}/api/computer/{computer_id}"
-    resp = requests.get(details_endpoint, headers=headers, verify=False, timeout=_TIMEOUT)
+    resp = requests.get(
+        details_endpoint,
+        headers=headers,
+        verify=False,
+        timeout=_TIMEOUT,
+    )
     resp.raise_for_status()
     return resp.text
 
 
 @timeit
 def load_computers(
-        neo4j_session: neo4j.Session,
-        data: List[Dict[str, Any]],
-        bigfix_root_url: str,
-        update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    bigfix_root_url: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
         BigfixRootSchema(),
-        [{'id': bigfix_root_url}],
+        [{"id": bigfix_root_url}],
         lastupdated=update_tag,
     )
 
@@ -170,8 +193,13 @@ def load_computers(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    GraphJob.from_node_schema(BigfixComputerSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(BigfixComputerSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 def _get_headers(bigfix_username: str, bigfix_password: str) -> Dict[str, str]: