cartography 0.102.0rc1__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/intel/digitalocean/management.py

@@ -1,9 +1,14 @@
 import logging
+from typing import Any
+from typing import Dict
+from typing import List
 
 import neo4j
 from digitalocean import Manager
 
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.digitalocean.project import DOProjectSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -11,31 +16,19 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        manager: Manager,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
-) -> dict:
-    projects_resources = sync_projects(neo4j_session, manager, digitalocean_update_tag, common_job_parameters)
-    return projects_resources
-
-
-@timeit
-def sync_projects(
-        neo4j_session: neo4j.Session,
-        manager: Manager,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
+    neo4j_session: neo4j.Session,
+    manager: Manager,
+    account_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
 ) -> dict:
     logger.info("Syncing Projects")
-    account_id = common_job_parameters['DO_ACCOUNT_ID']
     projects_res = get_projects(manager)
-    projects_resources = get_projects_resources(projects_res)
-    projects = transform_projects(projects_res, account_id)
-    load_projects(neo4j_session, projects, digitalocean_update_tag)
-    cleanup_projects(neo4j_session, common_job_parameters)
+    projects = transform_projects(projects_res)
+    load_projects(neo4j_session, projects, account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
 
-    return projects_resources
+    return get_projects_resources(projects_res)
 
 
 @timeit
@@ -49,77 +42,48 @@ def get_projects_resources(projects_res: list) -> dict:
     for p in projects_res:
         resources = p.get_all_resources()
         result[p.id] = resources
-
     return result
 
 
 @timeit
-def transform_projects(project_res: list, account_id: str) -> list:
+def transform_projects(project_res: list) -> list:
     result = list()
     for p in project_res:
         project = {
-            'id': p.id,
-            'name': p.name,
-            'owner_uuid': p.owner_uuid,
-            'description': p.description,
-            'environment': p.environment,
-            'is_default': p.is_default,
-            'created_at': p.created_at,
-            'updated_at': p.updated_at,
-            'account_id': account_id,
+            "id": p.id,
+            "name": p.name,
+            "owner_uuid": p.owner_uuid,
+            "description": p.description,
+            "environment": p.environment,
+            "is_default": p.is_default,
+            "created_at": p.created_at,
+            "updated_at": p.updated_at,
         }
         result.append(project)
     return result
 
 
 @timeit
-def load_projects(neo4j_session: neo4j.Session, data: list, digitalocean_update_tag: int) -> None:
-    query = """
-        MERGE (a:DOAccount{id:$AccountId})
-        ON CREATE SET a.firstseen = timestamp()
-        SET a.lastupdated = $digitalocean_update_tag
-
-        MERGE (p:DOProject{id:$ProjectId})
-        ON CREATE SET p.firstseen = timestamp()
-        SET p.account_id = $AccountId,
-        p.name = $Name,
-        p.owner_uuid = $OwnerUuid,
-        p.description = $Description,
-        p.environment = $Environment,
-        p.is_default = $IsDefault,
-        p.created_at = $CreatedAt,
-        p.updated_at = $UpdatedAt,
-        p.lastupdated = $digitalocean_update_tag
-        WITH p, a
-
-        MERGE (a)-[r:RESOURCE]->(p)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $digitalocean_update_tag
-        """
-    for project in data:
-        neo4j_session.run(
-            query,
-            AccountId=project['account_id'],
-            ProjectId=project['id'],
-            Name=project['name'],
-            OwnerUuid=project['owner_uuid'],
-            Description=project['description'],
-            Environment=project['environment'],
-            IsDefault=project['is_default'],
-            CreatedAt=project['created_at'],
-            UpdatedAt=project['updated_at'],
-            digitalocean_update_tag=digitalocean_update_tag,
-        )
-    return
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        DOProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ACCOUNT_ID=str(account_id),
+    )
 
 
 @timeit
-def cleanup_projects(neo4j_session: neo4j.Session, common_job_parameters: dict) -> None:
-    """
-        Delete out-of-date DigitalOcean projects and relationships
-        :param neo4j_session: The Neo4j session
-        :param common_job_parameters: dict of other job parameters to pass to Neo4j
-        :return: Nothing
-    """
-    run_cleanup_job('digitalocean_project_cleanup.json', neo4j_session, common_job_parameters)
-    return
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(DOProjectSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/digitalocean/platform.py

@@ -1,67 +1,72 @@
 import logging
+from typing import Any
+from typing import Dict
+from typing import List
 
 import neo4j
 from digitalocean import Account
+from digitalocean import Manager
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.digitalocean.account import DOAccountSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        account: Account,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
-) -> None:
-    sync_account(neo4j_session, account, digitalocean_update_tag, common_job_parameters)
+    neo4j_session: neo4j.Session,
+    manager: Manager,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> str:
+    logger.info("Syncing Account")
+    account = get_account(manager)
+    account_transformed = transform_account(account)
+    load_account(
+        neo4j_session,
+        [
+            account_transformed,
+        ],
+        update_tag,
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+    return account_transformed["id"]
 
 
 @timeit
-def sync_account(
-        neo4j_session: neo4j.Session,
-        account: Account,
-        digitalocean_update_tag: int,
-        common_job_parameters: dict,
-) -> None:
-    logger.info("Syncing Account")
-    account_transformed = transform_account(account)
-    load_account(neo4j_session, account_transformed, digitalocean_update_tag)
-    return
+def get_account(manager: Manager) -> Account:
+    return manager.get_account()
 
 
 @timeit
 def transform_account(account_res: Account) -> dict:
-    account = {
-        'id': account_res.uuid,
-        'uuid': account_res.uuid,
-        'droplet_limit': account_res.droplet_limit,
-        'floating_ip_limit': account_res.floating_ip_limit,
-        'status': account_res.status,
+    return {
+        "id": account_res.uuid,
+        "uuid": account_res.uuid,
+        "droplet_limit": account_res.droplet_limit,
+        "floating_ip_limit": account_res.floating_ip_limit,
+        "status": account_res.status,
     }
-    return account
 
 
 @timeit
-def load_account(neo4j_session: neo4j.Session, account: dict, digitalocean_update_tag: int) -> None:
-    query = """
-            MERGE (a:DOAccount{id:$AccountId})
-            ON CREATE SET a.firstseen = timestamp()
-            SET a.uuid = $Uuid,
-            a.droplet_limit = $DropletLimit,
-            a.floating_ip_limit = $FloatingIpLimit,
-            a.status = $Status,
-            a.lastupdated = $digitalocean_update_tag
-            """
-    neo4j_session.run(
-        query,
-        AccountId=account['id'],
-        Uuid=account['uuid'],
-        DropletLimit=account['droplet_limit'],
-        FloatingIpLimit=account['floating_ip_limit'],
-        Status=account['status'],
-        digitalocean_update_tag=digitalocean_update_tag,
+def load_account(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(neo4j_session, DOAccountSchema(), data, lastupdated=update_tag)
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(DOAccountSchema(), common_job_parameters).run(
+        neo4j_session,
     )
-    return

cartography/intel/dns.py

@@ -15,7 +15,11 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def ingest_dns_record_by_fqdn(
-    neo4j_session: neo4j.Session, update_tag: int, fqdn: str, points_to_record: str, record_label: str,
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    fqdn: str,
+    points_to_record: str,
+    record_label: str,
     dns_node_additional_label: Optional[str] = None,
 ) -> None:
     """
@@ -43,7 +47,7 @@ def ingest_dns_record_by_fqdn(
     fqdn_data = get_dns_resolution_by_fqdn(fqdn)
     record_type = get_dns_record_type(fqdn_data)
 
-    if record_type == 'A':
+    if record_type == "A":
         ip_list = []
         for result in fqdn_data:
             ip = str(result)
@@ -51,8 +55,14 @@ def ingest_dns_record_by_fqdn(
 
         value = ",".join(ip_list)
         record_id = ingest_dns_record(
-            neo4j_session, fqdn, value, record_type, update_tag, points_to_record,
-            record_label, dns_node_additional_label,  # type: ignore
+            neo4j_session,
+            fqdn,
+            value,
+            record_type,
+            update_tag,
+            points_to_record,
+            record_label,
+            dns_node_additional_label,  # type: ignore
         )
         _link_ip_to_A_record(neo4j_session, update_tag, ip_list, record_id)
 
@@ -67,7 +77,12 @@ def ingest_dns_record_by_fqdn(
 
 
 @timeit
-def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list: List[str], parent_record: str) -> None:
+def _link_ip_to_A_record(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    ip_list: List[str],
+    parent_record: str,
+) -> None:
     """
     Link A record to to its IP
 
@@ -99,8 +114,14 @@ def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list:
 
 @timeit
 def ingest_dns_record(
-    neo4j_session: neo4j.Session, name: neo4j.Session, value: str, type: str, update_tag: int, points_to_record: str,
-    record_label: str, dns_node_additional_label: str,
+    neo4j_session: neo4j.Session,
+    name: neo4j.Session,
+    value: str,
+    type: str,
+    update_tag: int,
+    points_to_record: str,
+    record_label: str,
+    dns_node_additional_label: str,
 ) -> str:
     """
     Ingest a new DNS record
@@ -115,7 +136,8 @@ def ingest_dns_record(
     :param dns_node_additional_label: The specific label of the DNSRecord, e.g. AWSDNSRecord.
     :return: the intel graph node id for the new/merged record
     """
-    template = Template("""
+    template = Template(
+        """
     MERGE (record:DNSRecord:$dns_node_additional_label{id: $Id})
     ON CREATE SET record.firstseen = timestamp(), record.name = $Name, record.type = $Type
     SET record.lastupdated = $update_tag, record.value = $Value
@@ -124,12 +146,16 @@ def ingest_dns_record(
     MERGE (record)-[r:DNS_POINTS_TO]->(n)
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $update_tag
-    """)
+    """,
+    )
 
     record_id = f"{name}+{type}"
 
     neo4j_session.run(
-        template.safe_substitute(record_label=record_label, dns_node_additional_label=dns_node_additional_label),
+        template.safe_substitute(
+            record_label=record_label,
+            dns_node_additional_label=dns_node_additional_label,
+        ),
         Id=record_id,
         Name=name,
         Type=type,

cartography/intel/duo/__init__.py

@@ -13,31 +13,34 @@ from cartography.intel.duo.groups import sync_duo_groups
 from cartography.intel.duo.phones import sync as sync_duo_phones
 from cartography.intel.duo.tokens import sync as sync_duo_tokens
 from cartography.intel.duo.users import sync_duo_users
-from cartography.intel.duo.web_authn_credentials import sync as sync_duo_web_authn_credentials
+from cartography.intel.duo.web_authn_credentials import (
+    sync as sync_duo_web_authn_credentials,
+)
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def get_client(config: Config) -> duo_client.Admin:
-    '''
+    """
     Return a duo Admin client with the creds in the config object
-    '''
+    """
     client = duo_client.Admin(
         ikey=config.duo_api_key,
         skey=config.duo_api_secret,
         host=config.duo_api_hostname,
     )
     # Duo's library does not automatically respect the HTTP_PROXY env variable
-    proxy_url = os.environ.get('HTTP_PROXY')
+    proxy_url = os.environ.get("HTTP_PROXY")
     if proxy_url:
         proxy_config = urlparse(proxy_url)
         headers = {}
         if proxy_config.username:
-            proxy_auth_token = b64encode(f"{proxy_config.username}:{proxy_config.password}".encode()).decode('ascii')
-            headers['Proxy-Authorization'] = f'Basic {proxy_auth_token}'
+            proxy_auth_token = b64encode(
+                f"{proxy_config.username}:{proxy_config.password}".encode(),
+            ).decode("ascii")
+            headers["Proxy-Authorization"] = f"Basic {proxy_auth_token}"
         client.set_proxy(
             host=proxy_config.hostname,
             port=proxy_config.port,
@@ -48,20 +51,22 @@ def get_client(config: Config) -> duo_client.Admin:
 
 @timeit
 def start_duo_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
-    '''
+    """
     If this module is configured, perform ingestion of duo data. Otherwise warn and exit
     :param neo4j_session: Neo4J session for database interface
     :param config: A cartography.config object
     :return: None
-    '''
-    if not all([
-        config.duo_api_key,
-        config.duo_api_secret,
-        config.duo_api_hostname,
-    ]):
+    """
+    if not all(
+        [
+            config.duo_api_key,
+            config.duo_api_secret,
+            config.duo_api_hostname,
+        ],
+    ):
         logger.info(
-            'Duo import is not configured - skipping this module. '
-            'See docs to configure.',
+            "Duo import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return
 

cartography/intel/duo/api_host.py

@@ -8,31 +8,36 @@ from cartography.client.core.tx import load
 from cartography.models.duo.api_host import DuoApiHostSchema
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_duo_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def sync_duo_api_host(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Add the DuoApiHost subresource
-    '''
+    """
     _load_api_host(neo4j_session, common_job_parameters)
 
 
 @timeit
-def _load_api_host(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _load_api_host(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Load the host node into the graph
-    '''
+    """
     data = [
         {
-            'id': common_job_parameters['DUO_API_HOSTNAME'],
+            "id": common_job_parameters["DUO_API_HOSTNAME"],
         },
     ]
     load(
         neo4j_session,
         DuoApiHostSchema(),
         data,
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )

cartography/intel/duo/endpoints.py

@@ -21,9 +21,9 @@ def sync_duo_endpoints(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Sync Duo Endpoints
-    '''
+    """
     endpoints = _get_endpoints(client)
     transformed_endpoints = _transform_endpoints(endpoints)
     _load_endpoints(neo4j_session, transformed_endpoints, common_job_parameters)
@@ -32,52 +32,52 @@ def sync_duo_endpoints(
 
 @timeit
 def _get_endpoints(client: duo_client.Admin) -> List[Dict[str, Any]]:
-    '''
+    """
     Fetch all endpoint data
     https://duo.com/docs/adminapi#endpoints
-    '''
+    """
     logger.info("Fetching Duo endpoints")
     return client.get_endpoints()
 
 
 @timeit
 def _transform_endpoints(endpoints: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    '''
+    """
     Reformat the data before loading
-    '''
-    logger.info(f'Transforming {len(endpoints)} duo endpoints')
+    """
+    logger.info(f"Transforming {len(endpoints)} duo endpoints")
     transformed_endpoints = []
     for endpoint in endpoints:
         transformed_endpoint = {
-            'browsers': [dumps(browser) for browser in endpoint['browsers']],
-            'computer_sid': endpoint['computer_sid'],
-            'cpu_id': endpoint['cpu_id'],
-            'device_id': endpoint['device_id'],
-            'device_identifier': endpoint['device_identifier'],
-            'device_identifier_type': endpoint['device_identifier_type'],
-            'device_name': endpoint['device_name'],
-            'device_udid': endpoint['device_udid'],
-            'device_username': endpoint['device_username'],
-            'device_username_type': endpoint['device_username_type'],
-            'disk_encryption_status': endpoint['disk_encryption_status'],
-            'domain_sid': endpoint['domain_sid'],
-            'email': endpoint['email'],
-            'epkey': endpoint['epkey'],
-            'firewall_status': endpoint['firewall_status'],
-            'hardware_uuid': endpoint['hardware_uuid'],
-            'health_app_client_version': endpoint['health_app_client_version'],
-            'health_data_last_collected': endpoint['health_data_last_collected'],
-            'last_updated': endpoint['last_updated'],
-            'machine_guid': endpoint['machine_guid'],
-            'model': endpoint['model'],
-            'os_build': endpoint['os_build'],
-            'os_family': endpoint['os_family'],
-            'os_version': endpoint['os_version'],
-            'password_status': endpoint['password_status'],
-            'security_agents': [dumps(agent) for agent in endpoint['security_agents']],
-            'trusted_endpoint': endpoint['trusted_endpoint'],
-            'type': endpoint['type'],
-            'username': endpoint['username'],
+            "browsers": [dumps(browser) for browser in endpoint["browsers"]],
+            "computer_sid": endpoint["computer_sid"],
+            "cpu_id": endpoint["cpu_id"],
+            "device_id": endpoint["device_id"],
+            "device_identifier": endpoint["device_identifier"],
+            "device_identifier_type": endpoint["device_identifier_type"],
+            "device_name": endpoint["device_name"],
+            "device_udid": endpoint["device_udid"],
+            "device_username": endpoint["device_username"],
+            "device_username_type": endpoint["device_username_type"],
+            "disk_encryption_status": endpoint["disk_encryption_status"],
+            "domain_sid": endpoint["domain_sid"],
+            "email": endpoint["email"],
+            "epkey": endpoint["epkey"],
+            "firewall_status": endpoint["firewall_status"],
+            "hardware_uuid": endpoint["hardware_uuid"],
+            "health_app_client_version": endpoint["health_app_client_version"],
+            "health_data_last_collected": endpoint["health_data_last_collected"],
+            "last_updated": endpoint["last_updated"],
+            "machine_guid": endpoint["machine_guid"],
+            "model": endpoint["model"],
+            "os_build": endpoint["os_build"],
+            "os_family": endpoint["os_family"],
+            "os_version": endpoint["os_version"],
+            "password_status": endpoint["password_status"],
+            "security_agents": [dumps(agent) for agent in endpoint["security_agents"]],
+            "trusted_endpoint": endpoint["trusted_endpoint"],
+            "type": endpoint["type"],
+            "username": endpoint["username"],
         }
         transformed_endpoints.append(transformed_endpoint)
     return transformed_endpoints
@@ -89,22 +89,27 @@ def _load_endpoints(
     endpoints: List[Dict[str, Any]],
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    '''
+    """
     Load the endpoints into the database
-    '''
-    logger.info(f'Loading {len(endpoints)} duo endpoints')
+    """
+    logger.info(f"Loading {len(endpoints)} duo endpoints")
     load(
         neo4j_session,
         DuoEndpointSchema(),
         endpoints,
-        DUO_API_HOSTNAME=common_job_parameters['DUO_API_HOSTNAME'],
-        lastupdated=common_job_parameters['UPDATE_TAG'],
+        DUO_API_HOSTNAME=common_job_parameters["DUO_API_HOSTNAME"],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
     )
 
 
 @timeit
-def _cleanup_endpoints(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    '''
+def _cleanup_endpoints(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
     Cleanup endpoints
-    '''
-    GraphJob.from_node_schema(DuoEndpointSchema(), common_job_parameters).run(neo4j_session)
+    """
+    GraphJob.from_node_schema(DuoEndpointSchema(), common_job_parameters).run(
+        neo4j_session,
+    )