cartography 0.117.0__py3-none-any.whl → 0.118.0__py3-none-any.whl

cartography/intel/aws/elasticsearch.py

@@ -8,6 +8,7 @@ import botocore.config
 import neo4j
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.dns import ingest_dns_record_by_fqdn
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
@@ -95,7 +96,8 @@ def _load_es_domains(
     for d in domain_list:
         del d["ServiceSoftwareOptions"]
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_records,
         Records=domain_list,
         AWS_ACCOUNT_ID=aws_account_id,
@@ -179,7 +181,8 @@ def _link_es_domain_vpc(
         groupList = vpc_data.get("SecurityGroupIds", [])
 
         if len(subnetList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_subnet,
                 DomainId=domain_id,
                 SubnetList=subnetList,
@@ -187,7 +190,8 @@ def _link_es_domain_vpc(
             )
 
         if len(groupList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_sec_groups,
                 DomainId=domain_id,
                 SecGroupList=groupList,
@@ -220,7 +224,12 @@ def _process_access_policy(
         if policy.is_internet_accessible():
             exposed_internet = True
 
-    neo4j_session.run(tag_es, DomainId=domain_id, InternetExposed=exposed_internet)
+    run_write_query(
+        neo4j_session,
+        tag_es,
+        DomainId=domain_id,
+        InternetExposed=exposed_internet,
+    )
 
 
 @timeit

cartography/intel/aws/identitycenter.py

@@ -6,10 +6,12 @@ from typing import Optional
 from typing import Union
 
 import boto3
+import botocore.exceptions
 import neo4j
 
 from cartography.client.core.tx import load
 from cartography.client.core.tx import load_matchlinks
+from cartography.client.core.tx import read_list_of_dicts_tx
 from cartography.graph.job import GraphJob
 from cartography.models.aws.identitycenter.awsidentitycenter import (
     AWSIdentityCenterInstanceSchema,
@@ -31,6 +33,18 @@ from cartography.util import timeit
 logger = logging.getLogger(__name__)
 
 
+def _is_permission_set_sync_unsupported_error(
+    error: botocore.exceptions.ClientError,
+) -> bool:
+    """Return True when the Identity Center instance does not support permission sets."""
+    error_info = error.response.get("Error", {})
+    if error_info.get("Code") != "ValidationException":
+        return False
+
+    message = error_info.get("Message", "").lower()
+    return "not supported for this identity center instance" in message
+
+
 @timeit
 @aws_handle_regions
 def get_identity_center_instances(
@@ -394,8 +408,11 @@ def get_permset_roles(
     WHERE permset.arn IN $PermSetIds
     RETURN permset.arn AS PermissionSetArn, role.arn AS RoleArn
     """
-    result = neo4j_session.run(query, PermSetIds=permset_ids)
-    permset_to_role = [record.data() for record in result]
+    permset_to_role = neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        PermSetIds=permset_ids,
+    )
 
     # Create mapping from permission set ARN to role ARN
     permset_to_role_map = {
@@ -504,32 +521,51 @@ def sync_identity_center_instances(
             instance_arn = instance["InstanceArn"]
             identity_store_id = instance["IdentityStoreId"]
 
-            permission_sets = get_permission_sets(boto3_session, instance_arn, region)
-
-            load_permission_sets(
-                neo4j_session,
-                permission_sets,
-                instance_arn,
-                region,
-                current_aws_account_id,
-                update_tag,
-            )
+            permission_set_sync_supported = True
+            try:
+                permission_sets = get_permission_sets(
+                    boto3_session, instance_arn, region
+                )
+            except botocore.exceptions.ClientError as error:
+                if _is_permission_set_sync_unsupported_error(error):
+                    permission_set_sync_supported = False
+                    permission_sets = []
+                    logger.warning(
+                        "Skipping permission set sync for Identity Center instance %s in region %s "
+                        "because the instance does not support permission sets.",
+                        instance_arn,
+                        region,
+                    )
+                else:
+                    raise
+
+            if permission_set_sync_supported:
+                load_permission_sets(
+                    neo4j_session,
+                    permission_sets,
+                    instance_arn,
+                    region,
+                    current_aws_account_id,
+                    update_tag,
+                )
 
             # Fetch groups first to avoid interleaving between groups and users
             groups = get_sso_groups(boto3_session, identity_store_id, region)
 
             # Get permission set assignments for groups
             group_permission_sets: Dict[str, List[str]] = {}
-            group_role_assignments_raw = get_group_role_assignments(
-                boto3_session,
-                groups,
-                instance_arn,
-                region,
-            )
-            for assignment in group_role_assignments_raw:
-                group_id = assignment["GroupId"]
-                perm_set = assignment["PermissionSetArn"]
-                group_permission_sets.setdefault(group_id, []).append(perm_set)
+            group_role_assignments_raw: List[Dict[str, Any]] = []
+            if permission_set_sync_supported:
+                group_role_assignments_raw = get_group_role_assignments(
+                    boto3_session,
+                    groups,
+                    instance_arn,
+                    region,
+                )
+                for assignment in group_role_assignments_raw:
+                    group_id = assignment["GroupId"]
+                    perm_set = assignment["PermissionSetArn"]
+                    group_permission_sets.setdefault(group_id, []).append(perm_set)
 
             # Transform and load groups with their permission set assignments FIRST
             # so that user->group membership edges can attach in the same run.
@@ -554,16 +590,18 @@ def sync_identity_center_instances(
 
             # Get direct permission set assignments for users
             user_permission_sets: Dict[str, List[str]] = {}
-            user_role_assignments_raw = get_role_assignments(
-                boto3_session,
-                users,
-                instance_arn,
-                region,
-            )
-            for assignment in user_role_assignments_raw:
-                uid = assignment["UserId"]
-                perm_set = assignment["PermissionSetArn"]
-                user_permission_sets.setdefault(uid, []).append(perm_set)
+            user_role_assignments_raw: List[Dict[str, Any]] = []
+            if permission_set_sync_supported:
+                user_role_assignments_raw = get_role_assignments(
+                    boto3_session,
+                    users,
+                    instance_arn,
+                    region,
+                )
+                for assignment in user_role_assignments_raw:
+                    uid = assignment["UserId"]
+                    perm_set = assignment["PermissionSetArn"]
+                    user_permission_sets.setdefault(uid, []).append(perm_set)
 
             # Transform and load users with their group memberships AFTER groups exist
             transformed_users = transform_sso_users(
@@ -584,28 +622,29 @@ def sync_identity_center_instances(
             # Note: we do this after groups and users are loaded so that
             # load_role_assignments calls can MATCH existing AWSSSOUser/AWSSSOGroup
             # nodes when drawing the ALLOWED_BY edges.
-            enriched_role_assignments = get_permset_roles(
-                neo4j_session,
-                user_role_assignments_raw,
-            )
-            load_role_assignments(
-                neo4j_session,
-                enriched_role_assignments,
-                current_aws_account_id,
-                update_tag,
-                RoleAssignmentAllowedByMatchLink(),
-            )
+            if permission_set_sync_supported:
+                enriched_role_assignments = get_permset_roles(
+                    neo4j_session,
+                    user_role_assignments_raw,
+                )
+                load_role_assignments(
+                    neo4j_session,
+                    enriched_role_assignments,
+                    current_aws_account_id,
+                    update_tag,
+                    RoleAssignmentAllowedByMatchLink(),
+                )
 
-            enriched_group_role_assignments = get_permset_roles(
-                neo4j_session,
-                group_role_assignments_raw,
-            )
-            load_role_assignments(
-                neo4j_session,
-                enriched_group_role_assignments,
-                current_aws_account_id,
-                update_tag,
-                RoleAssignmentAllowedByGroupMatchLink(),
-            )
+                enriched_group_role_assignments = get_permset_roles(
+                    neo4j_session,
+                    group_role_assignments_raw,
+                )
+                load_role_assignments(
+                    neo4j_session,
+                    enriched_group_role_assignments,
+                    current_aws_account_id,
+                    update_tag,
+                    RoleAssignmentAllowedByGroupMatchLink(),
+                )
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/inspector.py

@@ -75,16 +75,17 @@ def get_inspector_findings(
     session: boto3.session.Session,
     region: str,
     account_id: str,
+    batch_size: int,
 ) -> Iterator[List[Dict[str, Any]]]:
     """
     Query inspector2.list_findings by filtering the request, otherwise the request could timeout.
     First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away,
     only fetch those in ACTIVE or SUPPRESSED statuses.
-    Run the query in batches of 1000 findings and return an iterator to fetch the results.
+    Run the query in batches and return an iterator to fetch the results.
     """
     client = session.client("inspector2", region_name=region)
     logger.info(
-        f"Getting findings in batches of {BATCH_SIZE} for account {account_id} in region {region}"
+        f"Getting findings in batches of {batch_size} for account {account_id} in region {region}"
     )
     aws_args: Dict[str, Any] = {
         "filterCriteria": {
@@ -103,7 +104,7 @@ def get_inspector_findings(
         }
     }
     findings_batches = batch(
-        aws_paginate(client, "list_findings", "findings", None, **aws_args), BATCH_SIZE
+        aws_paginate(client, "list_findings", "findings", None, **aws_args), batch_size
     )
     yield from findings_batches
 
@@ -271,19 +272,25 @@ def load_inspector_finding_to_package_match_links(
 def cleanup(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict[str, Any],
+    batch_size: int = BATCH_SIZE,
 ) -> None:
     logger.info("Running AWS Inspector cleanup")
-    GraphJob.from_node_schema(AWSInspectorFindingSchema(), common_job_parameters).run(
-        neo4j_session,
-    )
-    GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
-        neo4j_session,
-    )
     GraphJob.from_matchlink(
         InspectorFindingToPackageMatchLink(),
         "AWSAccount",
         common_job_parameters["ACCOUNT_ID"],
         common_job_parameters["UPDATE_TAG"],
+        iterationsize=batch_size,
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AWSInspectorPackageSchema(), common_job_parameters, iterationsize=batch_size
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AWSInspectorFindingSchema(), common_job_parameters, iterationsize=batch_size
     ).run(
         neo4j_session,
     )
@@ -296,11 +303,12 @@ def _sync_findings_for_account(
     account_id: str,
     update_tag: int,
     current_aws_account_id: str,
+    batch_size: int = BATCH_SIZE,
 ) -> None:
     """
     Syncs the findings for a given account in a given region.
     """
-    findings = get_inspector_findings(boto3_session, region, account_id)
+    findings = get_inspector_findings(boto3_session, region, account_id, batch_size)
     if not findings:
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
@@ -343,6 +351,10 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict[str, Any],
 ) -> None:
+    batch_size = common_job_parameters.get(
+        "experimental_aws_inspector_batch", BATCH_SIZE
+    )
+
     inspector_regions = [
         region for region in regions if region in AWS_INSPECTOR_REGIONS
     ]
@@ -363,8 +375,8 @@ def sync(
                 account_id,
                 update_tag,
                 current_aws_account_id,
+                batch_size,
             )
-        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
-        common_job_parameters["UPDATE_TAG"] = update_tag
-
-    cleanup(neo4j_session, common_job_parameters)
+    common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+    common_job_parameters["UPDATE_TAG"] = update_tag
+    cleanup(neo4j_session, common_job_parameters, batch_size)

cartography/intel/aws/permission_relationships.py

@@ -13,6 +13,7 @@ import neo4j
 import yaml
 
 from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.client.core.tx import read_list_of_values_tx
 from cartography.client.core.tx import run_write_query
 from cartography.graph.statement import GraphStatement
 from cartography.util import timeit
@@ -300,12 +301,11 @@ def get_resource_arns(
     get_resource_query_template = get_resource_query.safe_substitute(
         node_label=node_label,
     )
-    results = neo4j_session.run(
+    return neo4j_session.execute_read(
+        read_list_of_values_tx,
         get_resource_query_template,
         AccountId=account_id,
     )
-    arns = [r["arn"] for r in results]
-    return arns
 
 
 def load_principal_mappings(

cartography/intel/aws/s3.py

@@ -16,6 +16,7 @@ from botocore.exceptions import ClientError
 from botocore.exceptions import EndpointConnectionError
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import run_write_query
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
@@ -334,7 +335,8 @@ def _load_s3_acls(
     SET r.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_acls,
         acls=acls,
         UpdateTag=update_tag,
@@ -368,7 +370,8 @@ def _load_s3_policies(
     s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policies,
         policies=policies,
         UpdateTag=update_tag,
@@ -401,11 +404,12 @@ def _load_s3_policy_statements(
         MERGE (bucket)-[r:POLICY_STATEMENT]->(statement)
         SET r.lastupdated = $UpdateTag
         """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_statement,
         Statements=statements,
         UpdateTag=update_tag,
-    ).consume()
+    )
 
 
 @timeit
@@ -427,7 +431,8 @@ def _load_s3_encryption(
     s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_encryption,
         encryption_configs=encryption_configs,
         UpdateTag=update_tag,
@@ -451,7 +456,8 @@ def _load_s3_versioning(
         s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_versioning,
         versioning_configs=versioning_configs,
         UpdateTag=update_tag,
@@ -477,7 +483,8 @@ def _load_s3_public_access_block(
         s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_public_access_block,
         public_access_block_configs=public_access_block_configs,
         UpdateTag=update_tag,
@@ -500,7 +507,8 @@ def _load_bucket_ownership_controls(
         s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_bucket_ownership_controls,
         bucket_ownership_controls_configs=bucket_ownership_controls_configs,
         UpdateTag=update_tag,
@@ -524,7 +532,8 @@ def _load_bucket_logging(
         bucket.logging_target_bucket = bucket_logging.target_bucket,
         bucket.lastupdated = $update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_bucket_logging,
         bucket_logging_configs=bucket_logging_configs,
         update_tag=update_tag,
@@ -536,7 +545,8 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
     SET s.anonymous_access = false, s.anonymous_actions = []
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         set_defaults,
         AWS_ID=aws_account_id,
     )
@@ -545,7 +555,8 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.default_encryption IS NULL
     SET s.default_encryption = false
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         set_encryption_defaults,
         AWS_ID=aws_account_id,
     )
@@ -993,7 +1004,8 @@ def _load_s3_notifications(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $UpdateTag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_notifications,
         notifications=notifications,
         UpdateTag=update_tag,
@@ -1025,7 +1037,8 @@ def load_s3_buckets(
 
     for bucket in data["Buckets"]:
         arn = "arn:aws:s3:::" + bucket["Name"]
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_bucket,
             BucketName=bucket["Name"],
             BucketRegion=bucket["Region"],

cartography/intel/aws/ssm.py

@@ -9,6 +9,7 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import read_list_of_values_tx
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
@@ -31,15 +32,12 @@ def get_instance_ids(
     WHERE i.region = $Region
     RETURN i.id
     """
-    results = neo4j_session.run(
+    return neo4j_session.execute_read(
+        read_list_of_values_tx,
         get_instances_query,
         AWS_ACCOUNT_ID=current_aws_account_id,
         Region=region,
     )
-    instance_ids = []
-    for r in results:
-        instance_ids.append(r["i.id"])
-    return instance_ids
 
 
 @timeit

cartography/intel/azure/compute.py

@@ -6,6 +6,7 @@ import neo4j
 from azure.core.exceptions import HttpResponseError
 from azure.mgmt.compute import ComputeManagementClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -63,7 +64,8 @@ def load_vms(
     SET r.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_vm,
         vms=vm_list,
         SUBSCRIPTION_ID=subscription_id,
@@ -103,7 +105,8 @@ def load_vm_data_disks(
     """
 
     # for disk in data_disks:
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_data_disk,
         disks=data_disks,
         VM_ID=vm_id,
@@ -162,7 +165,8 @@ def load_disks(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $update_tag"""
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_disks,
         disks=disk_list,
         SUBSCRIPTION_ID=subscription_id,
@@ -217,7 +221,8 @@ def load_snapshots(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $update_tag"""
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_snapshots,
         snapshots=snapshots,
         SUBSCRIPTION_ID=subscription_id,