cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/bigfix/computers.py

@@ -22,49 +22,49 @@ logger = logging.getLogger(__name__)
 _TIMEOUT = (60, 60)
 
 DEFAULT_SUPPORTED_KEYS = {
-    'Active Directory Path',
-    'Agent Type',
-    'Agent Version',
-    'Average Evaluation Cycle',
-    'BES Relay Selection Method',
-    'BES Root Server',
-    'BIOS',
-    'Computer Name',
-    'Computer Type',
-    'CPU',
-    'Device Type',
-    'Distance to BES Relay',
-    'DNS Name',
-    'Enrollment Date',
-    'Free Space on System Drive',
-    'ID',
-    'IP Address',
-    'IPv6 Address',
-    'Last Report Time',
-    'Location By IP Range',
-    'Locked',
-    'Logged on User',
-    'MAC Address',
-    'OS',
-    'Provider Name',
-    'RAM',
-    'Relay',
-    'Remote Desktop Enabled',
-    'Subnet Address',
-    'Total Size of System Drive',
-    'User Name',
+    "Active Directory Path",
+    "Agent Type",
+    "Agent Version",
+    "Average Evaluation Cycle",
+    "BES Relay Selection Method",
+    "BES Root Server",
+    "BIOS",
+    "Computer Name",
+    "Computer Type",
+    "CPU",
+    "Device Type",
+    "Distance to BES Relay",
+    "DNS Name",
+    "Enrollment Date",
+    "Free Space on System Drive",
+    "ID",
+    "IP Address",
+    "IPv6 Address",
+    "Last Report Time",
+    "Location By IP Range",
+    "Locked",
+    "Logged on User",
+    "MAC Address",
+    "OS",
+    "Provider Name",
+    "RAM",
+    "Relay",
+    "Remote Desktop Enabled",
+    "Subnet Address",
+    "Total Size of System Drive",
+    "User Name",
 }
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        bigfix_root_url: str,
-        bigfix_username: str,
-        bigfix_password: str,
-        update_tag: int,
-        common_job_parameters: Dict[str, Any],
-        computer_keys: Optional[Set[str]] = None,
+    neo4j_session: neo4j.Session,
+    bigfix_root_url: str,
+    bigfix_username: str,
+    bigfix_password: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+    computer_keys: Optional[Set[str]] = None,
 ) -> None:
     if not computer_keys:
         computer_keys = DEFAULT_SUPPORTED_KEYS
@@ -75,36 +75,50 @@ def sync(
 
 
 @timeit
-def get(bigfix_api_url: str, headers: Dict[str, str], computer_keys: Set[str]) -> List[Dict[str, Any]]:
+def get(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_keys: Set[str],
+) -> List[Dict[str, Any]]:
     result = []
     computer_list = get_computer_list(bigfix_api_url, headers)
     logger.info(f"Retrieving details for {len(computer_list)} BigFix Computers")
     for computer in computer_list:
-        details = get_computer_details(bigfix_api_url, headers, computer['ID'], computer_keys)
+        details = get_computer_details(
+            bigfix_api_url,
+            headers,
+            computer["ID"],
+            computer_keys,
+        )
         processed_comp: Dict[str, Any] = computer.copy()
 
         # Property names have spaces. Neo4j properties can't have spaces so let's clean this up.
         for key, val in details.items():
-            transformed_key = key.replace(' ', '')
+            transformed_key = key.replace(" ", "")
             processed_comp[transformed_key] = val
 
         # Post-processing transforms
-        processed_comp['EnrollmentDateTime'] = datetime.strptime(
-            processed_comp['EnrollmentDate'],
+        processed_comp["EnrollmentDateTime"] = datetime.strptime(
+            processed_comp["EnrollmentDate"],
             "%a, %d %b %Y %H:%M:%S %z",
         )
-        processed_comp['LastReportDateTime'] = datetime.strptime(
-            processed_comp['LastReportTime'],
+        processed_comp["LastReportDateTime"] = datetime.strptime(
+            processed_comp["LastReportTime"],
             "%a, %d %b %Y %H:%M:%S %z",
         )
-        processed_comp['RemoteDesktopIsEnabled'] = processed_comp['RemoteDesktopEnabled'] == 'True'
-        processed_comp['IsLocked'] = processed_comp['Locked'] == 'Yes'
+        processed_comp["RemoteDesktopIsEnabled"] = (
+            processed_comp["RemoteDesktopEnabled"] == "True"
+        )
+        processed_comp["IsLocked"] = processed_comp["Locked"] == "Yes"
 
         result.append(processed_comp)
     return result
 
 
-def get_computer_list(bigfix_api_url: str, headers: Dict[str, str]) -> List[Dict[str, str]]:
+def get_computer_list(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+) -> List[Dict[str, str]]:
     """
     Returned shape: [
       {'@Resource': 'https://{URI}/api/computer/123', 'LastReportTime': 'Tue, 18 Apr 2023 21:59:44 +0000', 'ID': '123'},
@@ -112,22 +126,22 @@ def get_computer_list(bigfix_api_url: str, headers: Dict[str, str]) -> List[Dict
     """
     xml_text = _get_computer_list_raw_xml(bigfix_api_url, headers)
     as_dict = xmltodict.parse(xml_text)
-    return as_dict['BESAPI']['Computer']
+    return as_dict["BESAPI"]["Computer"]
 
 
 def get_computer_details(
-        bigfix_api_url: str,
-        headers: Dict[str, str],
-        computer_id: str,
-        computer_keys: Set[str],
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_id: str,
+    computer_keys: Set[str],
 ) -> Dict[str, Any]:
     xml_text = _get_computer_details_raw_xml(bigfix_api_url, headers, computer_id)
     as_dict = xmltodict.parse(xml_text)
     processed_computer = {}
-    for prop in as_dict['BESAPI']['Computer']['Property']:
-        prop_name = prop['@Name']
+    for prop in as_dict["BESAPI"]["Computer"]["Property"]:
+        prop_name = prop["@Name"]
         if prop_name in computer_keys:
-            processed_computer[prop_name] = prop['#text']
+            processed_computer[prop_name] = prop["#text"]
     return processed_computer
 
 
@@ -138,24 +152,33 @@ def _get_computer_list_raw_xml(bigfix_api_url: str, headers: Dict[str, str]) ->
     return resp.text
 
 
-def _get_computer_details_raw_xml(bigfix_api_url: str, headers: Dict[str, str], computer_id: str) -> str:
+def _get_computer_details_raw_xml(
+    bigfix_api_url: str,
+    headers: Dict[str, str],
+    computer_id: str,
+) -> str:
     details_endpoint = f"{bigfix_api_url}/api/computer/{computer_id}"
-    resp = requests.get(details_endpoint, headers=headers, verify=False, timeout=_TIMEOUT)
+    resp = requests.get(
+        details_endpoint,
+        headers=headers,
+        verify=False,
+        timeout=_TIMEOUT,
+    )
     resp.raise_for_status()
     return resp.text
 
 
 @timeit
 def load_computers(
-        neo4j_session: neo4j.Session,
-        data: List[Dict[str, Any]],
-        bigfix_root_url: str,
-        update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    bigfix_root_url: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
         BigfixRootSchema(),
-        [{'id': bigfix_root_url}],
+        [{"id": bigfix_root_url}],
         lastupdated=update_tag,
     )
 
@@ -170,8 +193,13 @@ def load_computers(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    GraphJob.from_node_schema(BigfixComputerSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(BigfixComputerSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 def _get_headers(bigfix_username: str, bigfix_password: str) -> Dict[str, str]:

cartography/intel/cloudflare/__init__.py

@@ -0,0 +1,74 @@
+import logging
+
+import neo4j
+from cloudflare import Cloudflare
+
+import cartography.intel.cloudflare.accounts
+import cartography.intel.cloudflare.dnsrecords
+import cartography.intel.cloudflare.members
+import cartography.intel.cloudflare.roles
+import cartography.intel.cloudflare.zones
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_cloudflare_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Cloudflare data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.cloudflare_token:
+        logger.info(
+            "Cloudflare import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = Cloudflare(api_token=config.cloudflare_token)
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    for account in cartography.intel.cloudflare.accounts.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+    ):
+        account_job_parameters = common_job_parameters.copy()
+        account_job_parameters["account_id"] = account["id"]
+        cartography.intel.cloudflare.roles.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        cartography.intel.cloudflare.members.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        )
+
+        for zone in cartography.intel.cloudflare.zones.sync(
+            neo4j_session,
+            client,
+            account_job_parameters,
+            account_id=account["id"],
+        ):
+            zone_job_parameters = account_job_parameters.copy()
+            zone_job_parameters["zone_id"] = zone["id"]
+            cartography.intel.cloudflare.dnsrecords.sync(
+                neo4j_session,
+                client,
+                zone_job_parameters,
+                zone_id=zone["id"],
+            )

cartography/intel/cloudflare/accounts.py

@@ -0,0 +1,57 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.account import CloudflareAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict]:
+    accounts = get(client)
+    load_accounts(
+        neo4j_session,
+        accounts,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return accounts
+
+
+@timeit
+def get(client: Cloudflare) -> List[Dict[str, Any]]:
+    return [account.to_dict() for account in client.accounts.list()]
+
+
+def load_accounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare accounts into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareAccountSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/dnsrecords.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.dnsrecord import CloudflareDNSRecordSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    zone_id: str,
+) -> None:
+    dnsrecords = get(client, zone_id)
+    load_dnsrecords(
+        neo4j_session,
+        dnsrecords,
+        zone_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, zone_id: str) -> List[Dict[str, Any]]:
+    return [
+        dnsrecord.to_dict() for dnsrecord in client.dns.records.list(zone_id=zone_id)
+    ]
+
+
+def load_dnsrecords(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    zone_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare DNSRecords into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareDNSRecordSchema(),
+        data,
+        lastupdated=update_tag,
+        zone_id=zone_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareDNSRecordSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/members.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.member import CloudflareMemberSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    members = get(client, account_id)
+    transformed_members = transform_members(members)
+    load_members(
+        neo4j_session,
+        transformed_members,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, account_id: str) -> List[Dict[str, Any]]:
+    return [
+        member.to_dict()
+        for member in client.accounts.members.list(account_id=account_id)
+    ]
+
+
+def load_members(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare members into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareMemberSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def transform_members(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for member in data:
+        member["roles_ids"] = [role["id"] for role in member.get("roles", [])]
+        member["policies_ids"] = [policy["id"] for policy in member.get("policies", [])]
+        result.append(member)
+    return result
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareMemberSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/roles.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.role import CloudflareRoleSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    roles = get(client, account_id)
+    load_roles(
+        neo4j_session,
+        roles,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [
+        role.to_dict() for role in client.accounts.roles.list(account_id=account_id)
+    ]
+
+
+def load_roles(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare roles into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareRoleSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareRoleSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/zones.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.zone import CloudflareZoneSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> List[Dict]:
+    zones = get(client, account_id)
+    load_zones(
+        neo4j_session,
+        zones,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return zones
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [zone.to_dict() for zone in client.zones.list(account=account_id)]
+
+
+def load_zones(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare zones into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareZoneSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareZoneSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/create_indexes.py

@@ -3,17 +3,19 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.config import Config
 from cartography.util import load_resource_binary
+
 logger = logging.getLogger(__name__)
 
 
 def get_index_statements() -> List[str]:
     statements = []
-    with load_resource_binary('cartography.data', 'indexes.cypher') as f:
+    with load_resource_binary("cartography.data", "indexes.cypher") as f:
         for line in f.readlines():
             statements.append(
-                line.decode('UTF-8').rstrip('\r\n'),
+                line.decode("UTF-8").rstrip("\r\n"),
             )
     return statements
 
@@ -22,4 +24,4 @@ def run(neo4j_session: neo4j.Session, config: Config) -> None:
     logger.info("Creating indexes for cartography node types.")
     for statement in get_index_statements():
         logger.debug("Executing statement: %s", statement)
-        neo4j_session.run(statement)
+        run_write_query(neo4j_session, statement)