cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/cloudflare/__init__.py

@@ -0,0 +1,74 @@
+import logging
+
+import neo4j
+from cloudflare import Cloudflare
+
+import cartography.intel.cloudflare.accounts
+import cartography.intel.cloudflare.dnsrecords
+import cartography.intel.cloudflare.members
+import cartography.intel.cloudflare.roles
+import cartography.intel.cloudflare.zones
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_cloudflare_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Cloudflare data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.cloudflare_token:
+        logger.info(
+            "Cloudflare import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = Cloudflare(api_token=config.cloudflare_token)
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    for account in cartography.intel.cloudflare.accounts.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+    ):
+        account_job_parameters = common_job_parameters.copy()
+        account_job_parameters["account_id"] = account["id"]
+        cartography.intel.cloudflare.roles.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        cartography.intel.cloudflare.members.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        for zone in cartography.intel.cloudflare.zones.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        ):
+            zone_job_parameters = account_job_parameters.copy()
+            zone_job_parameters["zone_id"] = zone["id"]
+            cartography.intel.cloudflare.dnsrecords.sync(
+                neo4j_session,
+                client,
+                zone_job_parameters,
+                zone_id=zone["id"],
+            )

cartography/intel/cloudflare/accounts.py

@@ -0,0 +1,57 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.account import CloudflareAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict]:
+    accounts = get(client)
+    load_accounts(
+        neo4j_session,
+        accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return accounts
+
+
+@timeit
+def get(client: Cloudflare) -> List[Dict[str, Any]]:
+    return [account.to_dict() for account in client.accounts.list()]
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare accounts into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareAccountSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/dnsrecords.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.dnsrecord import CloudflareDNSRecordSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    zone_id: str,
+) -> None:
+    dnsrecords = get(client, zone_id)
+    load_dnsrecords(
+        neo4j_session,
+        dnsrecords,
+        zone_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, zone_id: str) -> List[Dict[str, Any]]:
+    return [
+        dnsrecord.to_dict() for dnsrecord in client.dns.records.list(zone_id=zone_id)
+    ]
+
+
+def load_dnsrecords(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    zone_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare DNSRecords into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareDNSRecordSchema(),
+        data,
+        lastupdated=update_tag,
+        zone_id=zone_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareDNSRecordSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/members.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.member import CloudflareMemberSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    members = get(client, account_id)
+    transformed_members = transform_members(members)
+    load_members(
+        neo4j_session,
+        transformed_members,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, account_id: str) -> List[Dict[str, Any]]:
+    return [
+        member.to_dict()
+        for member in client.accounts.members.list(account_id=account_id)
+    ]
+
+
+def load_members(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare members into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareMemberSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def transform_members(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for member in data:
+        member["roles_ids"] = [role["id"] for role in member.get("roles", [])]
+        member["policies_ids"] = [policy["id"] for policy in member.get("policies", [])]
+        result.append(member)
+    return result
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareMemberSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/roles.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.role import CloudflareRoleSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    roles = get(client, account_id)
+    load_roles(
+        neo4j_session,
+        roles,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [
+        role.to_dict() for role in client.accounts.roles.list(account_id=account_id)
+    ]
+
+
+def load_roles(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare roles into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareRoleSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareRoleSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/zones.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.zone import CloudflareZoneSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> List[Dict]:
+    zones = get(client, account_id)
+    load_zones(
+        neo4j_session,
+        zones,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return zones
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [zone.to_dict() for zone in client.zones.list(account=account_id)]
+
+
+def load_zones(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare zones into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareZoneSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareZoneSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/create_indexes.py

@@ -5,15 +5,16 @@ import neo4j
 
 from cartography.config import Config
 from cartography.util import load_resource_binary
+
 logger = logging.getLogger(__name__)
 
 
 def get_index_statements() -> List[str]:
     statements = []
-    with load_resource_binary('cartography.data', 'indexes.cypher') as f:
+    with load_resource_binary("cartography.data", "indexes.cypher") as f:
         for line in f.readlines():
             statements.append(
-                line.decode('UTF-8').rstrip('\r\n'),
+                line.decode("UTF-8").rstrip("\r\n"),
             )
     return statements
 

cartography/intel/crowdstrike/__init__.py

@@ -20,7 +20,8 @@ stat_handler = get_stats_client(__name__)
 
 @timeit
 def start_crowdstrike_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
+    neo4j_session: neo4j.Session,
+    config: Config,
 ) -> None:
     """
     Perform ingestion of crowdstrike data.
@@ -31,10 +32,7 @@ def start_crowdstrike_ingestion(
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
     }
-    if (
-        not config.crowdstrike_client_id or
-        not config.crowdstrike_client_secret
-    ):
+    if not config.crowdstrike_client_id or not config.crowdstrike_client_secret:
         logger.error("crowdstrike config not found")
         return
 
@@ -60,18 +58,22 @@ def start_crowdstrike_ingestion(
         group_id = config.crowdstrike_api_url
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='crowdstrike',
+        group_type="crowdstrike",
         group_id=group_id,
-        synced_type='crowdstrike',
+        synced_type="crowdstrike",
         update_tag=config.update_tag,
         stat_handler=stat_handler,
     )
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
     logger.info("Running Crowdstrike cleanup")
-    GraphJob.from_node_schema(CrowdstrikeHostSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(CrowdstrikeHostSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
     # Cleanup other crowdstrike assets not handled by the data model
     run_cleanup_job(

cartography/intel/crowdstrike/endpoints.py

@@ -44,7 +44,11 @@ def load_host_data(
     )
 
 
-def get_host_ids(client: Hosts, crowdstrikeapi_filter: str = '', crowdstrikeapi_limit: int = 5000) -> List[List[str]]:
+def get_host_ids(
+    client: Hosts,
+    crowdstrikeapi_filter: str = "",
+    crowdstrikeapi_limit: int = 5000,
+) -> List[List[str]]:
     ids = []
     parameters = {"filter": crowdstrikeapi_filter, "limit": crowdstrikeapi_limit}
     response = client.QueryDevicesByFilter(parameters=parameters)

cartography/intel/crowdstrike/spotlight.py

@@ -25,7 +25,9 @@ def sync_vulnerabilities(
 
 
 def load_vulnerability_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load scan information
@@ -111,7 +113,9 @@ def _load_cves(neo4j_session: neo4j.Session, data: List[Dict], update_tag: int)
     )
 
 
-def get_spotlight_vulnerability_ids(client: Spotlight_Vulnerabilities) -> List[List[str]]:
+def get_spotlight_vulnerability_ids(
+    client: Spotlight_Vulnerabilities,
+) -> List[List[str]]:
     ids = []
     parameters = {"filter": 'status:!"closed"', "limit": 400}
     response = client.queryVulnerabilities(parameters=parameters)
@@ -135,7 +139,8 @@ def get_spotlight_vulnerability_ids(client: Spotlight_Vulnerabilities) -> List[L
 
 
 def get_spotlight_vulnerabilities(
-    client: Spotlight_Vulnerabilities, ids: List[str],
+    client: Spotlight_Vulnerabilities,
+    ids: List[str],
 ) -> List[Dict]:
     response = client.getVulnerabilities(ids=",".join(ids))
     body = response.get("body", {})

cartography/intel/cve/__init__.py

@@ -38,21 +38,33 @@ def _sync_year_archives(
 ) -> None:
     existing_years = feed.get_cve_sync_metadata(neo4j_session)
     current_year = datetime.now().year
-    logger.info(f"Syncing CVE data for year archives. Existing years: {existing_years}. Current year: {current_year}")
+    logger.info(
+        f"Syncing CVE data for year archives. Existing years: {existing_years}. Current year: {current_year}",
+    )
     for year in range(1999, current_year + 1):
         if year in existing_years:
             continue
         logger.info(f"Syncing CVE data for year {year}")
-        cves = feed.get_published_cves_per_year(http_session, config.nist_cve_url, str(year), cve_api_key)
+        cves = feed.get_published_cves_per_year(
+            http_session,
+            config.nist_cve_url,
+            str(year),
+            cve_api_key,
+        )
         feed_metadata = feed.transform_cve_feed(cves)
         feed.load_cve_feed(neo4j_session, [feed_metadata], config.update_tag)
         published_cves = feed.transform_cves(cves)
-        feed.load_cves(neo4j_session, published_cves, feed_metadata['FEED_ID'], config.update_tag)
+        feed.load_cves(
+            neo4j_session,
+            published_cves,
+            feed_metadata["FEED_ID"],
+            config.update_tag,
+        )
         merge_module_sync_metadata(
             neo4j_session,
-            group_type='CVE',
+            group_type="CVE",
             group_id=year,
-            synced_type='year',
+            synced_type="year",
             update_tag=config.update_tag,
             stat_handler=stat_handler,
         )
@@ -66,16 +78,26 @@ def _sync_modified_data(
 ) -> None:
     logger.info("Syncing CVE data for modified data")
     last_modified_date = feed.get_last_modified_cve_date(neo4j_session)
-    cves = feed.get_modified_cves(http_session, config.nist_cve_url, last_modified_date, cve_api_key)
+    cves = feed.get_modified_cves(
+        http_session,
+        config.nist_cve_url,
+        last_modified_date,
+        cve_api_key,
+    )
     feed_metadata = feed.transform_cve_feed(cves)
     feed.load_cve_feed(neo4j_session, [feed_metadata], config.update_tag)
     modified_cves = feed.transform_cves(cves)
-    feed.load_cves(neo4j_session, modified_cves, feed_metadata['FEED_ID'], config.update_tag)
+    feed.load_cves(
+        neo4j_session,
+        modified_cves,
+        feed_metadata["FEED_ID"],
+        config.update_tag,
+    )
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='CVE',
-        group_id=feed_metadata['timestamp'][:4],
-        synced_type='modified',
+        group_type="CVE",
+        group_id=feed_metadata["timestamp"][:4],
+        synced_type="modified",
         update_tag=config.update_tag,
         stat_handler=stat_handler,
     )
@@ -83,7 +105,8 @@ def _sync_modified_data(
 
 @timeit
 def start_cve_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
+    neo4j_session: neo4j.Session,
+    config: Config,
 ) -> None:
     """
     Perform ingestion of CVE data from NIST APIs.
@@ -95,6 +118,16 @@ def start_cve_ingestion(
         return
     cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
     with _retryable_session() as http_session:
-        _sync_year_archives(http_session, neo4j_session=neo4j_session, config=config, cve_api_key=cve_api_key)
-        _sync_modified_data(http_session, neo4j_session=neo4j_session, config=config, cve_api_key=cve_api_key)
+        _sync_year_archives(
+            http_session,
+            neo4j_session=neo4j_session,
+            config=config,
+            cve_api_key=cve_api_key,
+        )
+        _sync_modified_data(
+            http_session,
+            neo4j_session=neo4j_session,
+            config=config,
+            cve_api_key=cve_api_key,
+        )
         # CVEs are never deleted, so we don't need to run a cleanup job