cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/digitalocean/platform.py

@@ -1,67 +1,72 @@
 import logging
+from typing import Any
+from typing import Dict
+from typing import List
 
 import neo4j
 from digitalocean import Account
+from digitalocean import Manager
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.digitalocean.account import DOAccountSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        account: Account,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
-) -> None:
-    sync_account(neo4j_session, account, digitalocean_update_tag, common_job_parameters)
+    neo4j_session: neo4j.Session,
+    manager: Manager,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> str:
+    logger.info("Syncing Account")
+    account = get_account(manager)
+    account_transformed = transform_account(account)
+    load_account(
+        neo4j_session,
+        [
+            account_transformed,
+        ],
+        update_tag,
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+    return account_transformed["id"]
 
 
 @timeit
-def sync_account(
-        neo4j_session: neo4j.Session,
-        account: Account,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
-) -> None:
-    logger.info("Syncing Account")
-    account_transformed = transform_account(account)
-    load_account(neo4j_session, account_transformed, digitalocean_update_tag)
-    return
+def get_account(manager: Manager) -> Account:
+    return manager.get_account()
 
 
 @timeit
 def transform_account(account_res: Account) -> dict:
-    account = {
-        'id': account_res.uuid,
-        'uuid': account_res.uuid,
-        'droplet_limit': account_res.droplet_limit,
-        'floating_ip_limit': account_res.floating_ip_limit,
-        'status': account_res.status,
+    return {
+        "id": account_res.uuid,
+        "uuid": account_res.uuid,
+        "droplet_limit": account_res.droplet_limit,
+        "floating_ip_limit": account_res.floating_ip_limit,
+        "status": account_res.status,
     }
-    return account
 
 
 @timeit
-def load_account(neo4j_session: neo4j.Session, account: dict, digitalocean_update_tag: int) -> None:
-    query = """
-            MERGE (a:DOAccount{id:$AccountId})
-            ON CREATE SET a.firstseen = timestamp()
-            SET a.uuid = $Uuid,
-            a.droplet_limit = $DropletLimit,
-            a.floating_ip_limit = $FloatingIpLimit,
-            a.status = $Status,
-            a.lastupdated = $digitalocean_update_tag
-            """
-    neo4j_session.run(
-        query,
-        AccountId=account['id'],
-        Uuid=account['uuid'],
-        DropletLimit=account['droplet_limit'],
-        FloatingIpLimit=account['floating_ip_limit'],
-        Status=account['status'],
-        digitalocean_update_tag=digitalocean_update_tag,
+def load_account(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(neo4j_session, DOAccountSchema(), data, lastupdated=update_tag)
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(DOAccountSchema(), common_job_parameters).run(
+        neo4j_session,
     )
-    return

cartography/intel/dns.py

@@ -15,7 +15,11 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def ingest_dns_record_by_fqdn(
-    neo4j_session: neo4j.Session, update_tag: int, fqdn: str, points_to_record: str, record_label: str,
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    fqdn: str,
+    points_to_record: str,
+    record_label: str,
     dns_node_additional_label: Optional[str] = None,
 ) -> None:
     """
@@ -43,7 +47,7 @@ def ingest_dns_record_by_fqdn(
     fqdn_data = get_dns_resolution_by_fqdn(fqdn)
     record_type = get_dns_record_type(fqdn_data)
 
-    if record_type == 'A':
+    if record_type == "A":
         ip_list = []
         for result in fqdn_data:
             ip = str(result)
@@ -51,8 +55,14 @@ def ingest_dns_record_by_fqdn(
 
         value = ",".join(ip_list)
         record_id = ingest_dns_record(
-            neo4j_session, fqdn, value, record_type, update_tag, points_to_record,
-            record_label, dns_node_additional_label,  # type: ignore
+            neo4j_session,
+            fqdn,
+            value,
+            record_type,
+            update_tag,
+            points_to_record,
+            record_label,
+            dns_node_additional_label,  # type: ignore
         )
         _link_ip_to_A_record(neo4j_session, update_tag, ip_list, record_id)
 
@@ -67,7 +77,12 @@ def ingest_dns_record_by_fqdn(
 
 
 @timeit
-def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list: List[str], parent_record: str) -> None:
+def _link_ip_to_A_record(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    ip_list: List[str],
+    parent_record: str,
+) -> None:
     """
     Link A record to to its IP
 
@@ -99,8 +114,14 @@ def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list:
 
 @timeit
 def ingest_dns_record(
-    neo4j_session: neo4j.Session, name: neo4j.Session, value: str, type: str, update_tag: int, points_to_record: str,
-    record_label: str, dns_node_additional_label: str,
+    neo4j_session: neo4j.Session,
+    name: neo4j.Session,
+    value: str,
+    type: str,
+    update_tag: int,
+    points_to_record: str,
+    record_label: str,
+    dns_node_additional_label: str,
 ) -> str:
     """
     Ingest a new DNS record
@@ -115,7 +136,8 @@ def ingest_dns_record(
     :param dns_node_additional_label: The specific label of the DNSRecord, e.g. AWSDNSRecord.
     :return: the intel graph node id for the new/merged record
     """
-    template = Template("""
+    template = Template(
+        """
     MERGE (record:DNSRecord:$dns_node_additional_label{id: $Id})
     ON CREATE SET record.firstseen = timestamp(), record.name = $Name, record.type = $Type
     SET record.lastupdated = $update_tag, record.value = $Value
@@ -124,12 +146,16 @@ def ingest_dns_record(
     MERGE (record)-[r:DNS_POINTS_TO]->(n)
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $update_tag
-    """)
+    """,
+    )
 
     record_id = f"{name}+{type}"
 
     neo4j_session.run(
-        template.safe_substitute(record_label=record_label, dns_node_additional_label=dns_node_additional_label),
+        template.safe_substitute(
+            record_label=record_label,
+            dns_node_additional_label=dns_node_additional_label,
+        ),
         Id=record_id,
         Name=name,
         Type=type,

cartography/intel/duo/__init__.py

@@ -13,31 +13,34 @@ from cartography.intel.duo.groups import sync_duo_groups
 from cartography.intel.duo.phones import sync as sync_duo_phones
 from cartography.intel.duo.tokens import sync as sync_duo_tokens
 from cartography.intel.duo.users import sync_duo_users
-from cartography.intel.duo.web_authn_credentials import sync as sync_duo_web_authn_credentials
+from cartography.intel.duo.web_authn_credentials import (
+    sync as sync_duo_web_authn_credentials,
+)
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def get_client(config: Config) -> duo_client.Admin:
-    '''
+    """
     Return a duo Admin client with the creds in the config object
-    '''
+    """
     client = duo_client.Admin(
         ikey=config.duo_api_key,
         skey=config.duo_api_secret,
         host=config.duo_api_hostname,
     )
     # Duo's library does not automatically respect the HTTP_PROXY env variable
-    proxy_url = os.environ.get('HTTP_PROXY')
+    proxy_url = os.environ.get("HTTP_PROXY")
     if proxy_url:
         proxy_config = urlparse(proxy_url)
         headers = {}
         if proxy_config.username:
-            proxy_auth_token = b64encode(f"{proxy_config.username}:{proxy_config.password}".encode()).decode('ascii')
-            headers['Proxy-Authorization'] = f'Basic {proxy_auth_token}'
+            proxy_auth_token = b64encode(
+                f"{proxy_config.username}:{proxy_config.password}".encode(),
+            ).decode("ascii")
+            headers["Proxy-Authorization"] = f"Basic {proxy_auth_token}"
         client.set_proxy(
             host=proxy_config.hostname,
             port=proxy_config.port,
@@ -48,20 +51,22 @@ def get_client(config: Config) -> duo_client.Admin:
 
 @timeit
 def start_duo_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
-    '''
+    """
     If this module is configured, perform ingestion of duo data. Otherwise warn and exit
     :param neo4j_session: Neo4J session for database interface
     :param config: A cartography.config object
     :return: None
-    '''
-    if not all([
-        config.duo_api_key,
-        config.duo_api_secret,
-        config.duo_api_hostname,
-    ]):
+    """
+    if not all(
+        [
+            config.duo_api_key,
+            config.duo_api_secret,
+            config.duo_api_hostname,
+        ],
+    ):
         logger.info(
-            'Duo import is not configured - skipping this module. '
-            'See docs to configure.',
+            "Duo import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return
 

cartography/intel/duo/api_host.py

@@ -8,31 +8,36 @@ from cartography.client.core.tx import load
 from cartography.models.duo.api_host import DuoApiHostSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_duo_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def sync_duo_api_host(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Add the DuoApiHost subresource
-    '''
+    """
     _load_api_host(neo4j_session, common_job_parameters)
 
 
 @timeit
-def _load_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _load_api_host(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Load the host node into the graph
-    '''
+    """
     data = [
         {
-            'id': common_job_parameters['DUO_API_HOSTNAME'],
+            "id": common_job_parameters["DUO_API_HOSTNAME"],
         },
     ]
     load(
         neo4j_session,
         DuoApiHostSchema(),
         data,
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )

cartography/intel/duo/endpoints.py

@@ -21,9 +21,9 @@ def sync_duo_endpoints(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync Duo Endpoints
-    '''
+    """
     endpoints = _get_endpoints(client)
     transformed_endpoints = _transform_endpoints(endpoints)
     _load_endpoints(neo4j_session, transformed_endpoints, common_job_parameters)
@@ -32,52 +32,52 @@ def sync_duo_endpoints(
 
 @timeit
 def _get_endpoints(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all endpoint data
     https://duo.com/docs/adminapi#endpoints
-    '''
+    """
     logger.info("Fetching Duo endpoints")
     return client.get_endpoints()
 
 
 @timeit
 def _transform_endpoints(endpoints: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(endpoints)} duo endpoints')
+    """
+    logger.info(f"Transforming {len(endpoints)} duo endpoints")
     transformed_endpoints = []
     for endpoint in endpoints:
         transformed_endpoint = {
-            'browsers': [dumps(browser) for browser in endpoint['browsers']],
-            'computer_sid': endpoint['computer_sid'],
-            'cpu_id': endpoint['cpu_id'],
-            'device_id': endpoint['device_id'],
-            'device_identifier': endpoint['device_identifier'],
-            'device_identifier_type': endpoint['device_identifier_type'],
-            'device_name': endpoint['device_name'],
-            'device_udid': endpoint['device_udid'],
-            'device_username': endpoint['device_username'],
-            'device_username_type': endpoint['device_username_type'],
-            'disk_encryption_status': endpoint['disk_encryption_status'],
-            'domain_sid': endpoint['domain_sid'],
-            'email': endpoint['email'],
-            'epkey': endpoint['epkey'],
-            'firewall_status': endpoint['firewall_status'],
-            'hardware_uuid': endpoint['hardware_uuid'],
-            'health_app_client_version': endpoint['health_app_client_version'],
-            'health_data_last_collected': endpoint['health_data_last_collected'],
-            'last_updated': endpoint['last_updated'],
-            'machine_guid': endpoint['machine_guid'],
-            'model': endpoint['model'],
-            'os_build': endpoint['os_build'],
-            'os_family': endpoint['os_family'],
-            'os_version': endpoint['os_version'],
-            'password_status': endpoint['password_status'],
-            'security_agents': [dumps(agent) for agent in endpoint['security_agents']],
-            'trusted_endpoint': endpoint['trusted_endpoint'],
-            'type': endpoint['type'],
-            'username': endpoint['username'],
+            "browsers": [dumps(browser) for browser in endpoint["browsers"]],
+            "computer_sid": endpoint["computer_sid"],
+            "cpu_id": endpoint["cpu_id"],
+            "device_id": endpoint["device_id"],
+            "device_identifier": endpoint["device_identifier"],
+            "device_identifier_type": endpoint["device_identifier_type"],
+            "device_name": endpoint["device_name"],
+            "device_udid": endpoint["device_udid"],
+            "device_username": endpoint["device_username"],
+            "device_username_type": endpoint["device_username_type"],
+            "disk_encryption_status": endpoint["disk_encryption_status"],
+            "domain_sid": endpoint["domain_sid"],
+            "email": endpoint["email"],
+            "epkey": endpoint["epkey"],
+            "firewall_status": endpoint["firewall_status"],
+            "hardware_uuid": endpoint["hardware_uuid"],
+            "health_app_client_version": endpoint["health_app_client_version"],
+            "health_data_last_collected": endpoint["health_data_last_collected"],
+            "last_updated": endpoint["last_updated"],
+            "machine_guid": endpoint["machine_guid"],
+            "model": endpoint["model"],
+            "os_build": endpoint["os_build"],
+            "os_family": endpoint["os_family"],
+            "os_version": endpoint["os_version"],
+            "password_status": endpoint["password_status"],
+            "security_agents": [dumps(agent) for agent in endpoint["security_agents"]],
+            "trusted_endpoint": endpoint["trusted_endpoint"],
+            "type": endpoint["type"],
+            "username": endpoint["username"],
         }
         transformed_endpoints.append(transformed_endpoint)
     return transformed_endpoints
@@ -89,22 +89,27 @@ def _load_endpoints(
     endpoints: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the endpoints into the database
-    '''
-    logger.info(f'Loading {len(endpoints)} duo endpoints')
+    """
+    logger.info(f"Loading {len(endpoints)} duo endpoints")
     load(
         neo4j_session,
         DuoEndpointSchema(),
         endpoints,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup_endpoints(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup_endpoints(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup endpoints
-    '''
-    GraphJob.from_node_schema(DuoEndpointSchema(), common_job_parameters).run(neo4j_session)
+    """
+    GraphJob.from_node_schema(DuoEndpointSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/duo/groups.py

@@ -11,7 +11,6 @@ from cartography.graph.job import GraphJob
 from cartography.models.duo.group import DuoGroupSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -21,9 +20,9 @@ def sync_duo_groups(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync Duo groups
-    '''
+    """
     groups = _get_groups(client)
     _load_groups(neo4j_session, groups, common_job_parameters)
     _cleanup_groups(neo4j_session, common_job_parameters)
@@ -31,10 +30,10 @@ def sync_duo_groups(
 
 @timeit
 def _get_groups(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all group data
     https://duo.com/docs/adminapi#users
-    '''
+    """
     logger.info("Fetching Duo groups")
     return client.get_groups()
 
@@ -45,22 +44,27 @@ def _load_groups(
     groups: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the groups into the graph
-    '''
-    logger.info(f'Loading {len(groups)} duo groups')
+    """
+    logger.info(f"Loading {len(groups)} duo groups")
     load(
         neo4j_session,
         DuoGroupSchema(),
         groups,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup_groups(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup endpoints
-    '''
-    GraphJob.from_node_schema(DuoGroupSchema(), common_job_parameters).run(neo4j_session)
+    """
+    GraphJob.from_node_schema(DuoGroupSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/duo/phones.py

@@ -20,9 +20,9 @@ def sync(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync
-    '''
+    """
     data = _get(client)
     transformed_data = _transform(data)
     _load(neo4j_session, transformed_data, common_job_parameters)
@@ -31,45 +31,45 @@ def sync(
 
 @timeit
 def _get(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all data
     https://duo.com/docs/adminapi#endpoints
-    '''
-    logger.info(f'Fetching data for {Schema.label}')
+    """
+    logger.info(f"Fetching data for {Schema.label}")
     return client.get_phones()
 
 
 @timeit
 def _transform(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Transforming {len(data)} items for {Schema.label}")
     transformed_data = []
     for datum in data:
         transformed_datum = {
-            'activated': datum['activated'],
-            'capabilities': datum['capabilities'],
-            'encrypted': datum['encrypted'],
-            'extension': datum['extension'],
-            'fingerprint': datum['fingerprint'],
-            'last_seen': datum['last_seen'],
-            'model': datum['model'],
-            'name': datum['name'],
-            'phone_id': datum['phone_id'],
-            'platform': datum['platform'],
-            'postdelay': datum['postdelay'],
-            'predelay': datum['predelay'],
-            'screenlock': datum['screenlock'],
-            'sms_passcodes_sent': datum['sms_passcodes_sent'],
-            'tampered': datum['tampered'],
-            'type': datum['type'],
+            "activated": datum["activated"],
+            "capabilities": datum["capabilities"],
+            "encrypted": datum["encrypted"],
+            "extension": datum["extension"],
+            "fingerprint": datum["fingerprint"],
+            "last_seen": datum["last_seen"],
+            "model": datum["model"],
+            "name": datum["name"],
+            "phone_id": datum["phone_id"],
+            "platform": datum["platform"],
+            "postdelay": datum["postdelay"],
+            "predelay": datum["predelay"],
+            "screenlock": datum["screenlock"],
+            "sms_passcodes_sent": datum["sms_passcodes_sent"],
+            "tampered": datum["tampered"],
+            "type": datum["type"],
         }
         transformed_data.append(transformed_datum)
-        for user in datum['users']:
+        for user in datum["users"]:
             match_datum = {
                 **transformed_datum,
-                'user_id': user['user_id'],
+                "user_id": user["user_id"],
             }
             transformed_data.append(match_datum)
     return transformed_data
@@ -81,22 +81,25 @@ def _load(
     data: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the data into the database
-    '''
-    logger.info(f'Loading {len(data)} items for {Schema.label}')
+    """
+    logger.info(f"Loading {len(data)} items for {Schema.label}")
     load(
         neo4j_session,
         Schema(),
         data,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup nodes
-    '''
+    """
     GraphJob.from_node_schema(Schema(), common_job_parameters).run(neo4j_session)