cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/oci/__init__.py

@@ -11,17 +11,19 @@ from oci.exceptions import ConfigFileNotFound
 from oci.exceptions import InvalidConfig
 from oci.exceptions import ProfileNotFound
 
+from cartography.config import Config
+
 from . import iam
 from . import organizations
 from . import utils
-from cartography.config import Config
+
 # from cartography.util import run_analysis_job
 # from cartography.util import run_cleanup_job
 # from . import network
 # from . import compute
 
 logger = logging.getLogger(__name__)
-Resources = namedtuple('Resources', 'compute iam network')
+Resources = namedtuple("Resources", "compute iam network")
 
 
 def _sync_one_account(
@@ -32,11 +34,21 @@ def _sync_one_account(
     common_job_parameters: Dict[str, Any],
 ) -> None:
     logger.info("Syncing OCI IAM client for OCI Tenancy with ID '%s'.", tenancy_id)
-    iam.sync(neo4j_session, resources.iam, tenancy_id, oci_sync_tag, common_job_parameters)
+    iam.sync(
+        neo4j_session,
+        resources.iam,
+        tenancy_id,
+        oci_sync_tag,
+        common_job_parameters,
+    )
 
     regions = utils.get_regions_in_tenancy(neo4j_session, tenancy_id)
     for region in regions:
-        logger.info("Syncing OCI region '%s' for OCI Tenancy with ID '%s'.", region["name"], tenancy_id)
+        logger.info(
+            "Syncing OCI region '%s' for OCI Tenancy with ID '%s'.",
+            region["name"],
+            tenancy_id,
+        )
         _change_resources_region(resources, region["name"])
         # compute.sync(neo4j_session, resources.compute,
         #   tenancy_id, region["name"], oci_sync_tag, common_job_parameters
@@ -56,15 +68,25 @@ def _sync_multiple_accounts(
     sync_tag: int,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    logger.debug("Syncing OCI accounts: %s", ', '.join(accounts.keys()))
+    logger.debug("Syncing OCI accounts: %s", ", ".join(accounts.keys()))
     organizations.sync(neo4j_session, accounts, sync_tag, common_job_parameters)
 
     for name in accounts:
-        logger.info("Syncing OCI Tenancy with ID '%s' using configured profile '%s'.", accounts[name]["tenancy"], name)
+        logger.info(
+            "Syncing OCI Tenancy with ID '%s' using configured profile '%s'.",
+            accounts[name]["tenancy"],
+            name,
+        )
         resources = _initialize_resources(accounts[name])
         tenancy_id = accounts[name]["tenancy"]
         common_job_parameters["OCI_TENANCY_ID"] = tenancy_id
-        _sync_one_account(neo4j_session, resources, tenancy_id, sync_tag, common_job_parameters)
+        _sync_one_account(
+            neo4j_session,
+            resources,
+            tenancy_id,
+            sync_tag,
+            common_job_parameters,
+        )
 
     del common_job_parameters["OCI_TENANCY_ID"]
 
@@ -82,7 +104,9 @@ def _change_resources_region(resources: NamedTuple, region: str) -> None:
         resource.base_client.set_region(region)
 
 
-def _get_network_resource(credentials: Dict[str, Any]) -> oci.core.virtual_network_client.VirtualNetworkClient:
+def _get_network_resource(
+    credentials: Dict[str, Any],
+) -> oci.core.virtual_network_client.VirtualNetworkClient:
     """
     Instantiates a OCI VirtualNetworkClient resource object to call the Network API.
      See https://docs.cloud.oracle.com/en-us/iaas/Content/Network/Concepts/overview.htm.
@@ -92,7 +116,9 @@ def _get_network_resource(credentials: Dict[str, Any]) -> oci.core.virtual_netwo
     return oci.core.VirtualNetworkClient(credentials)
 
 
-def _get_iam_resource(credentials: Dict[str, Any]) -> oci.identity.identity_client.IdentityClient:
+def _get_iam_resource(
+    credentials: Dict[str, Any],
+) -> oci.identity.identity_client.IdentityClient:
     """
     Instantiates a OCI IdentityCleint resource object to call the Identity API. This is used to users,
      ..., ... and ... data. See https://docs.cloud.oracle.com/iaas/Content/Compute/Concepts/computeoverview.htm.
@@ -102,7 +128,9 @@ def _get_iam_resource(credentials: Dict[str, Any]) -> oci.identity.identity_clie
     return oci.identity.IdentityClient(credentials)
 
 
-def _get_compute_resource(credentials: Dict[str, Any]) -> oci.core.compute_client.ComputeClient:
+def _get_compute_resource(
+    credentials: Dict[str, Any],
+) -> oci.core.compute_client.ComputeClient:
     """
     Instantiates a OCI ComputeClient resource object to call the Compute API. This is used to pull zone, instance, and
     networking data. https://docs.cloud.oracle.com/iaas/Content/Compute/Concepts/computeoverview.htm.
@@ -180,7 +208,12 @@ def start_oci_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
         )
         return
 
-    _sync_multiple_accounts(neo4j_session, oci_accounts, config.update_tag, common_job_parameters)
+    _sync_multiple_accounts(
+        neo4j_session,
+        oci_accounts,
+        config.update_tag,
+        common_job_parameters,
+    )
 
     # Look into adding analysis job once compute is implemented.
     # run_analysis_job(

cartography/intel/oci/iam.py

@@ -10,9 +10,12 @@ from typing import List
 import neo4j
 import oci
 
-from . import utils
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 
+from . import utils
+
 logger = logging.getLogger(__name__)
 
 
@@ -25,8 +28,17 @@ def sync_compartments(
 ) -> None:
     logger.debug("Syncing IAM compartments for account '%s'.", current_tenancy_id)
     data = get_compartment_list_data(iam, current_tenancy_id)
-    load_compartments(neo4j_session, data['Compartments'], current_tenancy_id, oci_update_tag)
-    run_cleanup_job('oci_import_compartments_cleanup.json', neo4j_session, common_job_parameters)
+    load_compartments(
+        neo4j_session,
+        data["Compartments"],
+        current_tenancy_id,
+        oci_update_tag,
+    )
+    run_cleanup_job(
+        "oci_import_compartments_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 def get_compartment_list_data_recurse(
@@ -35,11 +47,17 @@ def get_compartment_list_data_recurse(
     compartment_id: str,
 ) -> None:
 
-    response = oci.pagination.list_call_get_all_results(iam.list_compartments, compartment_id)
+    response = oci.pagination.list_call_get_all_results(
+        iam.list_compartments,
+        compartment_id,
+    )
     if not response.data:
         return
     compartment_list.update(
-        {"Compartments": list(compartment_list["Compartments"]) + utils.oci_object_to_json(response.data)},
+        {
+            "Compartments": list(compartment_list["Compartments"])
+            + utils.oci_object_to_json(response.data),
+        },
     )
     for compartment in response.data:
         get_compartment_list_data_recurse(iam, compartment_list, compartment.id)
@@ -73,7 +91,8 @@ def load_compartments(
     """
 
     for compartment in compartments:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_compartment,
             OCID=compartment["id"],
             COMPARTMENT_ID=compartment["compartment-id"],
@@ -110,7 +129,8 @@ def load_users(
     """
 
     for user in users:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_user,
             OCID=user["id"],
             CREATE_DATE=str(user["time-created"]),
@@ -122,7 +142,9 @@ def load_users(
             CAN_USE_API_KEYS=user["capabilities"]["can-use-api-keys"],
             CAN_USE_AUTH_TOKENS=user["capabilities"]["can-use-auth-tokens"],
             CAN_USE_CONSOLE_PASSWORD=user["capabilities"]["can-use-console-password"],
-            CAN_USE_CUSTOMER_SECRET_KEYS=user["capabilities"]["can-use-customer-secret-keys"],
+            CAN_USE_CUSTOMER_SECRET_KEYS=user["capabilities"][
+                "can-use-customer-secret-keys"
+            ],
             CAN_USE_SMTP_CREDENTIALS=user["capabilities"]["can-use-smtp-credentials"],
             COMPARTMENT_ID=user["compartment-id"],
             OCI_TENANCY_ID=current_oci_tenancy_id,
@@ -134,8 +156,11 @@ def get_user_list_data(
     iam: oci.identity.identity_client.IdentityClient,
     current_tenancy_id: str,
 ) -> Dict[str, List[Dict[str, Any]]]:
-    response = oci.pagination.list_call_get_all_results(iam.list_users, current_tenancy_id)
-    return {'Users': utils.oci_object_to_json(response.data)}
+    response = oci.pagination.list_call_get_all_results(
+        iam.list_users,
+        current_tenancy_id,
+    )
+    return {"Users": utils.oci_object_to_json(response.data)}
 
 
 def sync_users(
@@ -147,16 +172,23 @@ def sync_users(
 ) -> None:
     logger.debug("Syncing IAM users for account '%s'.", current_tenancy_id)
     data = get_user_list_data(iam, current_tenancy_id)
-    load_users(neo4j_session, data['Users'], current_tenancy_id, oci_update_tag)
-    run_cleanup_job('oci_import_users_cleanup.json', neo4j_session, common_job_parameters)
+    load_users(neo4j_session, data["Users"], current_tenancy_id, oci_update_tag)
+    run_cleanup_job(
+        "oci_import_users_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 def get_group_list_data(
     iam: oci.identity.identity_client.IdentityClient,
     current_tenancy_id: str,
 ) -> Dict[str, List[Dict[str, Any]]]:
-    response = oci.pagination.list_call_get_all_results(iam.list_groups, current_tenancy_id)
-    return {'Groups': utils.oci_object_to_json(response.data)}
+    response = oci.pagination.list_call_get_all_results(
+        iam.list_groups,
+        current_tenancy_id,
+    )
+    return {"Groups": utils.oci_object_to_json(response.data)}
 
 
 def load_groups(
@@ -178,7 +210,8 @@ def load_groups(
     """
 
     for group in groups:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_group,
             OCID=group["id"],
             CREATE_DATE=str(group["time-created"]),
@@ -200,7 +233,11 @@ def sync_groups(
     logger.debug("Syncing IAM groups for account '%s'.", current_tenancy_id)
     data = get_group_list_data(iam, current_tenancy_id)
     load_groups(neo4j_session, data["Groups"], current_tenancy_id, oci_update_tag)
-    run_cleanup_job('oci_import_groups_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job(
+        "oci_import_groups_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 def get_group_membership_data(
@@ -209,9 +246,11 @@ def get_group_membership_data(
     current_tenancy_id: str,
 ) -> Dict[str, List[Dict[str, Any]]]:
     response = oci.pagination.list_call_get_all_results(
-        iam.list_user_group_memberships, compartment_id=current_tenancy_id, group_id=group_id,
+        iam.list_user_group_memberships,
+        compartment_id=current_tenancy_id,
+        group_id=group_id,
     )
-    return {'GroupMemberships': utils.oci_object_to_json(response.data)}
+    return {"GroupMemberships": utils.oci_object_to_json(response.data)}
 
 
 def sync_group_memberships(
@@ -222,15 +261,22 @@ def sync_group_memberships(
     common_job_parameters: Dict[str, Any],
 ) -> None:
     logger.debug("Syncing IAM group membership for account '%s'.", current_tenancy_id)
-    query = "MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) " \
-            "return group.name as name, group.ocid as ocid;"
-    groups = neo4j_session.run(query, OCI_TENANCY_ID=current_tenancy_id)
+    query = (
+        "MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) "
+        "return group.name as name, group.ocid as ocid;"
+    )
+    groups = neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=current_tenancy_id,
+    )
     groups_membership = {
-        group["ocid"]: get_group_membership_data(iam, group['ocid'], current_tenancy_id) for group in groups
+        group["ocid"]: get_group_membership_data(iam, group["ocid"], current_tenancy_id)
+        for group in groups
     }
     load_group_memberships(neo4j_session, groups_membership, oci_update_tag)
     run_cleanup_job(
-        'oci_import_groups_membership_cleanup.json',
+        "oci_import_groups_membership_cleanup.json",
         neo4j_session,
         common_job_parameters,
     )
@@ -251,7 +297,8 @@ def load_group_memberships(
     """
     for group_ocid, membership_data in group_memberships.items():
         for info in membership_data["GroupMemberships"]:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_membership,
                 COMPARTMENT_ID=info["compartment-id"],
                 GROUP_OCID=info["group-id"],
@@ -280,7 +327,8 @@ def load_policies(
     """
 
     for policy in policies:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_policy,
             OCID=policy["id"],
             POLICY_NAME=policy["name"],
@@ -298,8 +346,11 @@ def get_policy_list_data(
     iam: oci.identity.identity_client.IdentityClient,
     current_tenancy_id: str,
 ) -> Dict[str, List[Dict[str, Any]]]:
-    response = oci.pagination.list_call_get_all_results(iam.list_policies, compartment_id=current_tenancy_id)
-    return {'Policies': utils.oci_object_to_json(response.data)}
+    response = oci.pagination.list_call_get_all_results(
+        iam.list_policies,
+        compartment_id=current_tenancy_id,
+    )
+    return {"Policies": utils.oci_object_to_json(response.data)}
 
 
 def sync_policies(
@@ -313,12 +364,23 @@ def sync_policies(
     compartments = utils.get_compartments_in_tenancy(neo4j_session, current_tenancy_id)
     for compartment in compartments:
         logger.debug(
-            "Syncing OCI policies for compartment '%s' in account '%s'.", compartment['ocid'], current_tenancy_id,
+            "Syncing OCI policies for compartment '%s' in account '%s'.",
+            compartment["ocid"],
+            current_tenancy_id,
         )
         data = get_policy_list_data(iam, compartment["ocid"])
-        if (data["Policies"]):
-            load_policies(neo4j_session, data["Policies"], current_tenancy_id, oci_update_tag)
-    run_cleanup_job('oci_import_policies_cleanup.json', neo4j_session, common_job_parameters)
+        if data["Policies"]:
+            load_policies(
+                neo4j_session,
+                data["Policies"],
+                current_tenancy_id,
+                oci_update_tag,
+            )
+    run_cleanup_job(
+        "oci_import_policies_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 def load_oci_policy_group_reference(
@@ -335,7 +397,8 @@ def load_oci_policy_group_reference(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $oci_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_group_reference,
         POLICY_ID=policy_id,
         GROUP_ID=group_id,
@@ -357,7 +420,8 @@ def load_oci_policy_compartment_reference(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $oci_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_compartment_reference,
         POLICY_ID=policy_id,
         COMPARTMENT_ID=compartment_id,
@@ -378,22 +442,33 @@ def sync_oci_policy_references(
     for policy in policies:
         check_compart = policy["compartmentid"]
         for statement in policy["statements"]:
-            m = re.search('(?<=group\\s)[^ ]*(?=\\s)', statement)
+            m = re.search("(?<=group\\s)[^ ]*(?=\\s)", statement)
             if m:
                 for group in groups:
                     if group["name"].lower() == m.group(0).lower():
                         load_oci_policy_group_reference(
-                            neo4j_session, policy["ocid"], group["ocid"], tenancy_id, oci_update_tag,
+                            neo4j_session,
+                            policy["ocid"],
+                            group["ocid"],
+                            tenancy_id,
+                            oci_update_tag,
                         )
-            m = re.search('(?<=compartment\\s)[^ ]*(?=$)', statement)
+            m = re.search("(?<=compartment\\s)[^ ]*(?=$)", statement)
             if m:
                 for compartment in compartments:
                     # Only look at the compartment or subcompartment name referenced in the policy statement
                     # in which the policy is a member of.
-                    if compartment["ocid"] == check_compart or compartment["compartmentid"] == check_compart:
+                    if (
+                        compartment["ocid"] == check_compart
+                        or compartment["compartmentid"] == check_compart
+                    ):
                         if compartment["name"].lower() == m.group(0).lower():
                             load_oci_policy_compartment_reference(
-                                neo4j_session, policy["ocid"], compartment['ocid'], tenancy_id, oci_update_tag,
+                                neo4j_session,
+                                policy["ocid"],
+                                compartment["ocid"],
+                                tenancy_id,
+                                oci_update_tag,
                             )
 
 
@@ -401,8 +476,11 @@ def get_region_subscriptions_list_data(
     iam: oci.identity.identity_client.IdentityClient,
     current_tenancy_id: str,
 ) -> Dict[str, List[Dict[str, Any]]]:
-    response = oci.pagination.list_call_get_all_results(iam.list_region_subscriptions, current_tenancy_id)
-    return {'RegionSubscriptions': utils.oci_object_to_json(response.data)}
+    response = oci.pagination.list_call_get_all_results(
+        iam.list_region_subscriptions,
+        current_tenancy_id,
+    )
+    return {"RegionSubscriptions": utils.oci_object_to_json(response.data)}
 
 
 def load_region_subscriptions(
@@ -422,7 +500,8 @@ def load_region_subscriptions(
     SET r.lastupdated = $oci_update_tag
     """
     for region in regions:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             REGION_KEY=region["region-key"],
             REGION_NAME=region["region-name"],
@@ -438,9 +517,17 @@ def sync_region_subscriptions(
     oci_update_tag: int,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    logger.debug("Syncing IAM region subscriptions for account '%s'.", current_tenancy_id)
+    logger.debug(
+        "Syncing IAM region subscriptions for account '%s'.",
+        current_tenancy_id,
+    )
     data = get_region_subscriptions_list_data(iam, current_tenancy_id)
-    load_region_subscriptions(neo4j_session, data["RegionSubscriptions"], current_tenancy_id, oci_update_tag)
+    load_region_subscriptions(
+        neo4j_session,
+        data["RegionSubscriptions"],
+        current_tenancy_id,
+        oci_update_tag,
+    )
     # run_cleanup_job('oci_import_region_subscriptions_cleanup.json', neo4j_session, common_job_parameters)
 
 
@@ -454,8 +541,31 @@ def sync(
     logger.info("Syncing IAM for account '%s'.", tenancy_id)
     sync_users(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
     sync_groups(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
-    sync_group_memberships(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
-    sync_compartments(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
+    sync_group_memberships(
+        neo4j_session,
+        iam,
+        tenancy_id,
+        oci_update_tag,
+        common_job_parameters,
+    )
+    sync_compartments(
+        neo4j_session,
+        iam,
+        tenancy_id,
+        oci_update_tag,
+        common_job_parameters,
+    )
     sync_policies(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
-    sync_oci_policy_references(neo4j_session, tenancy_id, oci_update_tag, common_job_parameters)
-    sync_region_subscriptions(neo4j_session, iam, tenancy_id, oci_update_tag, common_job_parameters)
+    sync_oci_policy_references(
+        neo4j_session,
+        tenancy_id,
+        oci_update_tag,
+        common_job_parameters,
+    )
+    sync_region_subscriptions(
+        neo4j_session,
+        iam,
+        tenancy_id,
+        oci_update_tag,
+        common_job_parameters,
+    )

cartography/intel/oci/organizations.py

@@ -11,6 +11,7 @@ from oci.exceptions import ConfigFileNotFound
 from oci.exceptions import InvalidConfig
 from oci.exceptions import ProfileNotFound
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 
 logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ def get_oci_account_default() -> Dict[str, Any]:
 def get_oci_profile_names_from_config() -> List[Any]:
     config_path = oci.config._get_config_path_with_fallback("~/.oci/config")
     config = open(config_path).read()
-    pattern = r'\[(.*)\]'
+    pattern = r"\[(.*)\]"
     m = re.findall(pattern, config)
     return m
 
@@ -52,16 +53,20 @@ def get_oci_accounts_from_config() -> Dict[str, Any]:
 
     d = {}
     for profile_name in available_profiles:
-        if profile_name == 'DEFAULT':
+        if profile_name == "DEFAULT":
             logger.debug("Skipping OCI profile 'DEFAULT'.")
             continue
         try:
-            profile_oci_credentials = oci.config.from_file("~/.oci/config", profile_name)
+            profile_oci_credentials = oci.config.from_file(
+                "~/.oci/config",
+                profile_name,
+            )
             oci.config.validate_config(profile_oci_credentials)
         except (ConfigFileNotFound, ProfileNotFound, InvalidConfig) as e:
             logger.debug(
                 "Error occurred calling oci.config.from_file with profile_name '%s'.",
-                profile_name, exc_info=True,
+                profile_name,
+                exc_info=True,
             )
             logger.error(
                 (
@@ -96,7 +101,8 @@ def load_oci_accounts(
     SET aa.lastupdated = $oci_update_tag, aa.name = $ACCOUNT_NAME
     """
     for name in oci_accounts:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             TENANCY_ID=oci_accounts[name]["tenancy"],
             ACCOUNT_NAME=name,
@@ -104,8 +110,11 @@ def load_oci_accounts(
         )
 
 
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    run_cleanup_job('oci_tenancy_cleanup.json', neo4j_session, common_job_parameters)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    run_cleanup_job("oci_tenancy_cleanup.json", neo4j_session, common_job_parameters)
 
 
 def sync(