cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/oci/organizations.py

@@ -11,6 +11,7 @@ from oci.exceptions import ConfigFileNotFound
 from oci.exceptions import InvalidConfig
 from oci.exceptions import ProfileNotFound
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 
 logger = logging.getLogger(__name__)
@@ -100,7 +101,8 @@ def load_oci_accounts(
     SET aa.lastupdated = $oci_update_tag, aa.name = $ACCOUNT_NAME
     """
     for name in oci_accounts:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             TENANCY_ID=oci_accounts[name]["tenancy"],
             ACCOUNT_NAME=name,

cartography/intel/oci/utils.py

@@ -7,6 +7,8 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import read_list_of_dicts_tx
+
 
 # Generic way to turn a OCI python object into the json response that you would see from calling the REST API.
 def oci_object_to_json(in_obj: Any) -> List[Dict[str, Any]]:
@@ -36,7 +38,11 @@ def get_compartments_in_tenancy(
         "return DISTINCT compartment.name as name, compartment.ocid as ocid, "
         "compartment.compartmentid as compartmentid;"
     )
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all groups in neo4j already populated by iam.
@@ -48,7 +54,11 @@ def get_groups_in_tenancy(
         "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)"
         "return DISTINCT group.name as name, group.ocid as ocid;"
     )
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all policies in neo4j already populated by iam.
@@ -61,7 +71,11 @@ def get_policies_in_tenancy(
         "return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, "
         "policy.compartmentid as compartmentid;"
     )
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all regions in neo4j already populated by iam.
@@ -73,7 +87,11 @@ def get_regions_in_tenancy(
         "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)"
         "return DISTINCT region.name as name, region.key as key;"
     )
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+    )
 
 
 # Grab list of all security groups in neo4j already populated by network. Need to handle regions for this one.
@@ -88,4 +106,9 @@ def get_security_groups_in_tenancy(
         "return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid "
         "as compartmentid;"
     )
-    return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id, OCI_REGION=region)
+    return neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=tenancy_id,
+        OCI_REGION=region,
+    )

cartography/intel/okta/applications.py

@@ -10,6 +10,7 @@ import neo4j
 from okta.framework.ApiClient import ApiClient
 from okta.framework.OktaError import OktaError
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.intel.okta.utils import create_api_client
 from cartography.intel.okta.utils import is_last_page
@@ -278,7 +279,12 @@ def _load_okta_applications(
     UNWIND $APP_LIST as app_data
     MERGE (new_app:OktaApplication{id: app_data.id})
     ON CREATE SET new_app.firstseen = timestamp()
-    SET new_app.name = app_data.name,
+    SET new_app:ThirdPartyApp,
+    new_app.name = app_data.name,
+    new_app._ont_client_id = app_data.id,
+    new_app._ont_name = app_data.label,
+    new_app._ont_enabled = (app_data.status IN ['ACTIVE']),
+    new_app._ont_protocol = app_data.sign_on_mode,
     new_app.label = app_data.label,
     new_app.created = app_data.created,
     new_app.okta_last_updated = app_data.okta_last_updated,
@@ -293,7 +299,8 @@ def _load_okta_applications(
     SET org_r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         ORG_ID=okta_org_id,
         APP_LIST=app_list,
@@ -327,7 +334,8 @@ def _load_application_user(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         USER_LIST=user_list,
@@ -361,7 +369,8 @@ def _load_application_group(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         GROUP_LIST=group_list,
@@ -400,7 +409,8 @@ def _load_application_reply_urls(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         APP_ID=app_id,
         URL_LIST=reply_urls,

cartography/intel/okta/awssaml.py

@@ -10,6 +10,7 @@ import neo4j
 
 from cartography.client.core.tx import read_list_of_dicts_tx
 from cartography.client.core.tx import read_single_value_tx
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 AccountRole = namedtuple("AccountRole", ["account_id", "role_name"])
@@ -67,24 +68,25 @@ def query_for_okta_to_aws_role_mapping(
     :param neo4j_session: session from the Neo4j server
     :param mapping_regex: the regex used by the organization to map groups to aws roles
     """
-    query = "MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) return group.id, group.name"
+    query = (
+        "MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) "
+        "RETURN group.id AS group_id, group.name AS group_name"
+    )
 
     group_to_role_mapping: List[Dict] = []
-    has_results = False
-    results = neo4j_session.run(query)
+    results = neo4j_session.execute_read(read_list_of_dicts_tx, query)
 
     for res in results:
-        has_results = True
         # input: okta group id, okta group name. output: aws role arn.
         mapping = transform_okta_group_to_aws_role(
-            res["group.id"],
-            res["group.name"],
+            res["group_id"],
+            res["group_name"],
             mapping_regex,
         )
         if mapping:
             group_to_role_mapping.append(mapping)
 
-    if has_results and not group_to_role_mapping:
+    if results and not group_to_role_mapping:
         logger.warning(
             "AWS Okta Application present, but no mappings were found. "
             "Please verify the mapping regex is correct",
@@ -116,7 +118,8 @@ def _load_okta_group_to_aws_roles(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         GROUP_TO_ROLE=group_to_role,
         okta_update_tag=okta_update_tag,
@@ -140,7 +143,8 @@ def _load_human_can_assume_role(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         okta_update_tag=okta_update_tag,
     )
@@ -244,7 +248,7 @@ def _load_awssso_tx(
         ingest_statement,
         GROUP_TO_ROLE=[g._asdict() for g in group_to_role],
         okta_update_tag=okta_update_tag,
-    )
+    ).consume()
 
 
 def _load_okta_group_to_awssso_roles(

cartography/intel/okta/factors.py

@@ -8,6 +8,7 @@ from okta import FactorsClient
 from okta.framework.OktaError import OktaError
 from okta.models.factor.Factor import Factor
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.sync_state import OktaSyncState
 from cartography.util import timeit
 
@@ -130,7 +131,8 @@ def _load_user_factors(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         USER_ID=user_id,
         FACTOR_LIST=factors,

cartography/intel/okta/groups.py

@@ -11,6 +11,7 @@ from okta.framework.OktaError import OktaError
 from okta.framework.PagedResults import PagedResults
 from okta.models.usergroup import UserGroup
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.sync_state import OktaSyncState
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.intel.okta.utils import create_api_client
@@ -204,7 +205,8 @@ def _load_okta_groups(
     SET org_r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         ORG_ID=okta_org_id,
         GROUP_LIST=group_list,
@@ -251,7 +253,8 @@ def load_okta_group_members(
         SET r.lastupdated = $okta_update_tag
     """
     logging.info(f"Loading {len(member_list)} members of group {group_id}")
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         GROUP_ID=group_id,
         MEMBER_LIST=member_list,

cartography/intel/okta/organization.py

@@ -3,6 +3,7 @@ import logging
 
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -27,7 +28,8 @@ def create_okta_organization(
     SET org.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         ORG_NAME=organization,
         okta_update_tag=okta_update_tag,

cartography/intel/okta/origins.py

@@ -7,6 +7,7 @@ from typing import List
 import neo4j
 from okta.framework.ApiClient import ApiClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.utils import create_api_client
 from cartography.util import timeit
 
@@ -96,7 +97,8 @@ def _load_trusted_origins(
     SET r.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         ORG_ID=okta_org_id,
         TRUSTED_LIST=trusted_list,

cartography/intel/okta/roles.py

@@ -7,6 +7,7 @@ from typing import List
 import neo4j
 from okta.framework.ApiClient import ApiClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.sync_state import OktaSyncState
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.intel.okta.utils import create_api_client
@@ -117,7 +118,8 @@ def _load_user_role(
     SET r2.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         USER_ID=user_id,
         ROLES_DATA=roles_data,
@@ -149,7 +151,8 @@ def _load_group_role(
     SET r2.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         GROUP_ID=group_id,
         ROLES_DATA=roles_data,

cartography/intel/okta/users.py

@@ -8,6 +8,7 @@ import neo4j
 from okta import UsersClient
 from okta.models.user import User
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.okta.sync_state import OktaSyncState
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.util import timeit
@@ -160,7 +161,13 @@ def _load_okta_users(
     new_user.password_changed = user_data.password_changed,
     new_user.transition_to_status = user_data.transition_to_status,
     new_user.lastupdated = $okta_update_tag,
-    new_user :UserAccount
+    new_user:UserAccount,
+    new_user._module_name = "cartography:okta",
+    new_user._ont_email = user_data.email,
+    new_user._ont_firstname = user_data.first_name,
+    new_user._ont_lastname = user_data.last_name,
+    new_user._ont_lastactivity = user_data.last_login,
+    new_user._ont_source = "okta"
     WITH new_user, org
     MERGE (org)-[org_r:RESOURCE]->(new_user)
     ON CREATE SET org_r.firstseen = timestamp()
@@ -174,7 +181,8 @@ def _load_okta_users(
     SET h.lastupdated = $okta_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_statement,
         ORG_ID=okta_org_id,
         USER_LIST=user_list,

cartography/intel/ontology/__init__.py

@@ -0,0 +1,44 @@
+import logging
+
+import neo4j
+
+import cartography.intel.ontology.devices
+import cartography.intel.ontology.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def run(neo4j_session: neo4j.Session, config: Config) -> None:
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    # Get source of truth from config
+    if config.ontology_users_source:
+        users_source_of_truth = [
+            source.strip() for source in config.ontology_users_source.split(",")
+        ]
+    else:
+        users_source_of_truth = []
+    if config.ontology_devices_source:
+        computers_source_of_truth = [
+            source.strip() for source in config.ontology_devices_source.split(",")
+        ]
+    else:
+        computers_source_of_truth = []
+
+    cartography.intel.ontology.users.sync(
+        neo4j_session,
+        users_source_of_truth,
+        config.update_tag,
+        common_job_parameters,
+    )
+    cartography.intel.ontology.devices.sync(
+        neo4j_session,
+        computers_source_of_truth,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/ontology/devices.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.ontology.utils import get_source_nodes_from_graph
+from cartography.intel.ontology.utils import link_ontology_nodes
+from cartography.models.ontology.device import DeviceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "devices")
+    load_devices(
+        neo4j_session,
+        data,
+        update_tag,
+    )
+    link_ontology_nodes(neo4j_session, "devices", update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        DeviceSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(DeviceSchema(), common_job_parameters).run(
+        neo4j_session,
+    )

cartography/intel/ontology/users.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Any
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.ontology.utils import get_source_nodes_from_graph
+from cartography.intel.ontology.utils import link_ontology_nodes
+from cartography.models.ontology.user import UserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    source_of_truth: list[str],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "users")
+    load_users(
+        neo4j_session,
+        data,
+        update_tag,
+    )
+    link_ontology_nodes(neo4j_session, "users", update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        UserSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(UserSchema(), common_job_parameters).run(
+        neo4j_session,
+    )