cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/oci/utils.py

@@ -7,6 +7,8 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import read_list_of_dicts_tx
+
 
 # Generic way to turn a OCI python object into the json response that you would see from calling the REST API.
 def oci_object_to_json(in_obj: Any) -> List[Dict[str, Any]]:
@@ -22,47 +24,91 @@ def replace_char_in_dict(in_dict: Dict[str, Any]) -> Dict[str, Any]:
     for dict_key, dict_val in in_dict.items():
         if isinstance(dict_val, dict):
             dict_val = replace_char_in_dict(dict_val)
-        out_dict[dict_key.replace('_', '-')] = dict_val
+        out_dict[dict_key.replace("_", "-")] = dict_val
     return out_dict
 
 
 # Grab list of all compartments and sub-compartments in neo4j already populated by iam.
-def get_compartments_in_tenancy(neo4j_session: neo4j.Session, tenancy_id: str) -> neo4j.Result:
-    query = "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(compartment:OCICompartment) " \
-            "return DISTINCT compartment.name as name, compartment.ocid as ocid, " \
-            "compartment.compartmentid as compartmentid;"
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+def get_compartments_in_tenancy(
+    neo4j_session: neo4j.Session,
+    tenancy_id: str,
+) -> neo4j.Result:
+    query = (
+        "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(compartment:OCICompartment) "
+        "return DISTINCT compartment.name as name, compartment.ocid as ocid, "
+        "compartment.compartmentid as compartmentid;"
+    )
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all groups in neo4j already populated by iam.
-def get_groups_in_tenancy(neo4j_session: neo4j.Session, tenancy_id: str) -> neo4j.Result:
-    query = "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)" \
-            "return DISTINCT group.name as name, group.ocid as ocid;"
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+def get_groups_in_tenancy(
+    neo4j_session: neo4j.Session,
+    tenancy_id: str,
+) -> neo4j.Result:
+    query = (
+        "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)"
+        "return DISTINCT group.name as name, group.ocid as ocid;"
+    )
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all policies in neo4j already populated by iam.
-def get_policies_in_tenancy(neo4j_session: neo4j.Session, tenancy_id: str) -> neo4j.Result:
-    query = "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(policy:OCIPolicy)" \
-            "return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, " \
-            "policy.compartmentid as compartmentid;"
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+def get_policies_in_tenancy(
+    neo4j_session: neo4j.Session,
+    tenancy_id: str,
+) -> neo4j.Result:
+    query = (
+        "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(policy:OCIPolicy)"
+        "return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, "
+        "policy.compartmentid as compartmentid;"
+    )
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all regions in neo4j already populated by iam.
-def get_regions_in_tenancy(neo4j_session: neo4j.Session, tenancy_id: str) -> neo4j.Result:
-    query = "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)" \
-            "return DISTINCT region.name as name, region.key as key;"
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+def get_regions_in_tenancy(
+    neo4j_session: neo4j.Session,
+    tenancy_id: str,
+) -> neo4j.Result:
+    query = (
+        "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)"
+        "return DISTINCT region.name as name, region.key as key;"
+    )
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all security groups in neo4j already populated by network. Need to handle regions for this one.
 def get_security_groups_in_tenancy(
     neo4j_session: neo4j.Session,
-    tenancy_id: str, region: str,
+    tenancy_id: str,
+    region: str,
 ) -> neo4j.Result:
-    query = "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(security_group:OCINetworkSecurityGroup)-[OCI_REGION]->" \
-            "(region:OCIRegion{name: $OCI_REGION})" \
-            "return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid " \
-            "as compartmentid;"
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id, OCI_REGION=region)
+    query = (
+        "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(security_group:OCINetworkSecurityGroup)-[OCI_REGION]->"
+        "(region:OCIRegion{name: $OCI_REGION})"
+        "return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid "
+        "as compartmentid;"
+    )
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+        OCI_REGION=region,
+    )

cartography/intel/okta/__init__.py

@@ -24,19 +24,25 @@ stat_handler = get_stats_client(__name__)
 
 
 @timeit
-def _cleanup_okta_organizations(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+def _cleanup_okta_organizations(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
     """
     Remove stale Okta organization
     :param neo4j_session: The Neo4j session
     :param common_job_parameters: Parameters to carry to the cleanup job
     :return: Nothing
     """
-    run_cleanup_job('okta_import_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job("okta_import_cleanup.json", neo4j_session, common_job_parameters)
     cleanup_okta_groups(neo4j_session, common_job_parameters)
 
 
-def cleanup_okta_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('okta_groups_cleanup.json', neo4j_session, common_job_parameters)
+def cleanup_okta_groups(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
+    run_cleanup_job("okta_groups_cleanup.json", neo4j_session, common_job_parameters)
 
 
 @timeit
@@ -62,34 +68,79 @@ def start_okta_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
 
     state = OktaSyncState()
 
-    organization.create_okta_organization(neo4j_session, config.okta_org_id, config.update_tag)
-    users.sync_okta_users(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
-    groups.sync_okta_groups(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
-    applications.sync_okta_applications(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
-    factors.sync_users_factors(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
-    origins.sync_trusted_origins(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
-    awssaml.sync_okta_aws_saml(neo4j_session, config.okta_saml_role_regex, config.update_tag, config.okta_org_id)
+    organization.create_okta_organization(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+    )
+    users.sync_okta_users(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+        config.okta_api_key,
+        state,
+    )
+    groups.sync_okta_groups(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+        config.okta_api_key,
+        state,
+    )
+    applications.sync_okta_applications(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+        config.okta_api_key,
+    )
+    factors.sync_users_factors(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+        config.okta_api_key,
+        state,
+    )
+    origins.sync_trusted_origins(
+        neo4j_session,
+        config.okta_org_id,
+        config.update_tag,
+        config.okta_api_key,
+    )
+    awssaml.sync_okta_aws_saml(
+        neo4j_session,
+        config.okta_saml_role_regex,
+        config.update_tag,
+        config.okta_org_id,
+    )
 
     # need creds with permission
     # soft fail as some won't be able to get such high priv token
     # when we get the E0000006 error
     # see https://developer.okta.com/docs/reference/error-codes/
     try:
-        roles.sync_roles(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
+        roles.sync_roles(
+            neo4j_session,
+            config.okta_org_id,
+            config.update_tag,
+            config.okta_api_key,
+            state,
+        )
     except OktaError as okta_error:
         logger.warning(f"Unable to pull admin roles got {okta_error}")
 
         # Getting roles requires super admin which most won't be able to get easily
         if okta_error.error_code == "E0000006":
-            logger.warning("Unable to sync admin roles - api token needs admin rights to pull admin roles data")
+            logger.warning(
+                "Unable to sync admin roles - api token needs admin rights to pull admin roles data",
+            )
 
     _cleanup_okta_organizations(neo4j_session, common_job_parameters)
 
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='OktaOrganization',
+        group_type="OktaOrganization",
         group_id=config.okta_org_id,
-        synced_type='OktaOrganization',
+        synced_type="OktaOrganization",
         update_tag=config.update_tag,
         stat_handler=stat_handler,
     )

cartography/intel/okta/applications.py

@@ -10,12 +10,12 @@ import neo4j
 from okta.framework.ApiClient import ApiClient
 from okta.framework.OktaError import OktaError
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.intel.okta.utils import create_api_client
 from cartography.intel.okta.utils import is_last_page
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,9 +36,9 @@ def _get_okta_applications(api_client: ApiClient) -> List[Dict]:
                 paged_response = api_client.get(next_url)
             else:
                 params = {
-                    'limit': 500,
+                    "limit": 500,
                 }
-                paged_response = api_client.get_path('/', params)
+                paged_response = api_client.get_path("/", params)
         except OktaError as okta_error:
             logger.debug(f"Got error while listing applications {okta_error}")
             break
@@ -73,11 +73,13 @@ def _get_application_assigned_users(api_client: ApiClient, app_id: str) -> List[
                 paged_response = api_client.get(next_url)
             else:
                 params = {
-                    'limit': 500,
+                    "limit": 500,
                 }
-                paged_response = api_client.get_path(f'/{app_id}/users', params)
+                paged_response = api_client.get_path(f"/{app_id}/users", params)
         except OktaError as okta_error:
-            logger.debug(f"Got error while going through list application assigned users {okta_error}")
+            logger.debug(
+                f"Got error while going through list application assigned users {okta_error}",
+            )
             break
 
         app_users.append(paged_response.text)
@@ -110,11 +112,13 @@ def _get_application_assigned_groups(api_client: ApiClient, app_id: str) -> List
                 paged_response = api_client.get(next_url)
             else:
                 params = {
-                    'limit': 500,
+                    "limit": 500,
                 }
-                paged_response = api_client.get_path(f'/{app_id}/groups', params)
+                paged_response = api_client.get_path(f"/{app_id}/groups", params)
         except OktaError as okta_error:
-            logger.debug(f"Got error while going through list application assigned groups {okta_error}")
+            logger.debug(
+                f"Got error while going through list application assigned groups {okta_error}",
+            )
             break
 
         app_groups.append(paged_response.text)
@@ -130,7 +134,9 @@ def _get_application_assigned_groups(api_client: ApiClient, app_id: str) -> List
 
 
 @timeit
-def transform_application_assigned_users_list(assigned_user_list: List[str]) -> List[str]:
+def transform_application_assigned_users_list(
+    assigned_user_list: List[str],
+) -> List[str]:
     """
     Transform application users Okta data
     :param assigned_user_list: Okta data on assigned users
@@ -161,7 +167,9 @@ def transform_application_assigned_users(json_app_data: str) -> List[str]:
 
 
 @timeit
-def transform_application_assigned_groups_list(assigned_group_list: List[str]) -> List[Dict]:
+def transform_application_assigned_groups_list(
+    assigned_group_list: List[str],
+) -> List[Dict]:
     group_list: List[Dict] = []
 
     for current in assigned_group_list:
@@ -206,14 +214,16 @@ def transform_okta_application(okta_application: Dict) -> Dict:
     app_props["label"] = okta_application["label"]
     if "created" in okta_application and okta_application["created"]:
         app_props["created"] = datetime.strptime(
-            okta_application["created"], "%Y-%m-%dT%H:%M:%S.%fZ",
+            okta_application["created"],
+            "%Y-%m-%dT%H:%M:%S.%fZ",
         ).strftime("%m/%d/%Y, %H:%M:%S")
     else:
         app_props["created"] = None
 
     if "lastUpdated" in okta_application and okta_application["lastUpdated"]:
         app_props["okta_last_updated"] = datetime.strptime(
-            okta_application["lastUpdated"], "%Y-%m-%dT%H:%M:%S.%fZ",
+            okta_application["lastUpdated"],
+            "%Y-%m-%dT%H:%M:%S.%fZ",
         ).strftime("%m/%d/%Y, %H:%M:%S")
     else:
         app_props["okta_last_updated"] = None
@@ -222,7 +232,8 @@ def transform_okta_application(okta_application: Dict) -> Dict:
 
     if "activated" in okta_application and okta_application["activated"]:
         app_props["activated"] = datetime.strptime(
-            okta_application["activated"], "%Y-%m-%dT%H:%M:%S.%fZ",
+            okta_application["activated"],
+            "%Y-%m-%dT%H:%M:%S.%fZ",
         ).strftime("%m/%d/%Y, %H:%M:%S")
     else:
         app_props["activated"] = None
@@ -234,7 +245,9 @@ def transform_okta_application(okta_application: Dict) -> Dict:
 
 
 @timeit
-def transform_okta_application_extract_replyurls(okta_application: Dict) -> Optional[str]:
+def transform_okta_application_extract_replyurls(
+    okta_application: Dict,
+) -> Optional[str]:
     """
     Extracts the reply uri information from an okta app
     """
@@ -247,7 +260,9 @@ def transform_okta_application_extract_replyurls(okta_application: Dict) -> Opti
 
 @timeit
 def _load_okta_applications(
-    neo4j_session: neo4j.Session, okta_org_id: str, app_list: List[Dict],
+    neo4j_session: neo4j.Session,
+    okta_org_id: str,
+    app_list: List[Dict],
     okta_update_tag: int,
 ) -> None:
     """
@@ -264,7 +279,12 @@ def _load_okta_applications(
     UNWIND $APP_LIST as app_data
     MERGE (new_app:OktaApplication{id: app_data.id})
     ON CREATE SET new_app.firstseen = timestamp()
-    SET new_app.name = app_data.name,
+    SET new_app:ThirdPartyApp,
+    new_app.name = app_data.name,
+    new_app._ont_client_id = app_data.id,
+    new_app._ont_name = app_data.label,
+    new_app._ont_enabled = (app_data.status IN ['ACTIVE']),
+    new_app._ont_protocol = app_data.sign_on_mode,
     new_app.label = app_data.label,
     new_app.created = app_data.created,
     new_app.okta_last_updated = app_data.okta_last_updated,
@@ -279,7 +299,8 @@ def _load_okta_applications(
     SET org_r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         ORG_ID=okta_org_id,
         APP_LIST=app_list,
@@ -289,7 +310,9 @@ def _load_okta_applications(
 
 @timeit
 def _load_application_user(
-    neo4j_session: neo4j.Session, app_id: str, user_list: List[str],
+    neo4j_session: neo4j.Session,
+    app_id: str,
+    user_list: List[str],
     okta_update_tag: int,
 ) -> None:
     """
@@ -311,7 +334,8 @@ def _load_application_user(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         USER_LIST=user_list,
@@ -321,7 +345,9 @@ def _load_application_user(
 
 @timeit
 def _load_application_group(
-    neo4j_session: neo4j.Session, app_id: str, group_list: List[str],
+    neo4j_session: neo4j.Session,
+    app_id: str,
+    group_list: List[str],
     okta_update_tag: int,
 ) -> None:
     """
@@ -343,7 +369,8 @@ def _load_application_group(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         GROUP_LIST=group_list,
@@ -353,7 +380,9 @@ def _load_application_group(
 
 @timeit
 def _load_application_reply_urls(
-    neo4j_session: neo4j.Session, app_id: str, reply_urls: List[str],
+    neo4j_session: neo4j.Session,
+    app_id: str,
+    reply_urls: List[str],
     okta_update_tag: int,
 ) -> None:
     """
@@ -380,7 +409,8 @@ def _load_application_reply_urls(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         URL_LIST=reply_urls,
@@ -390,7 +420,9 @@ def _load_application_reply_urls(
 
 @timeit
 def sync_okta_applications(
-    neo4j_session: neo4j.Session, okta_org_id: str, okta_update_tag: int,
+    neo4j_session: neo4j.Session,
+    okta_org_id: str,
+    okta_update_tag: int,
     okta_api_key: str,
 ) -> None:
     """