cartography 0.105.0__py3-none-any.whl → 0.106.0rc1__py3-none-any.whl

cartography/intel/aws/efs.py

@@ -0,0 +1,181 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.file_system import EfsFileSystemSchema
+from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_efs_file_systems(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_file_systems")
+    fileSystems = []
+    for page in paginator.paginate():
+        fileSystems.extend(page.get("FileSystems", []))
+
+    return fileSystems
+
+
+def transform_efs_file_systems(
+    fileSystems: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform SNS topic data for ingestion
+    """
+    transformed_file_systems = []
+    for file_system in fileSystems:
+        transformed_file_system = {
+            "FileSystemId": file_system["FileSystemId"],
+            "FileSystemArn": file_system["FileSystemArn"],
+            "OwnerId": file_system.get("OwnerId"),
+            "CreationToken": file_system.get("CreationToken"),
+            "CreationTime": file_system.get("CreationTime"),
+            "LifeCycleState": file_system.get("LifeCycleState"),
+            "Name": file_system.get("Name"),
+            "NumberOfMountTargets": file_system.get("NumberOfMountTargets"),
+            "SizeInBytesValue": file_system.get("SizeInBytes", {}).get("Value"),
+            "SizeInBytesTimestamp": file_system.get("SizeInBytes", {}).get("Timestamp"),
+            "PerformanceMode": file_system.get("PerformanceMode"),
+            "Encrypted": file_system.get("Encrypted"),
+            "KmsKeyId": file_system.get("KmsKeyId"),
+            "ThroughputMode": file_system.get("ThroughputMode"),
+            "AvailabilityZoneName": file_system.get("AvailabilityZoneName"),
+            "AvailabilityZoneId": file_system.get("AvailabilityZoneId"),
+            "FileSystemProtection": file_system.get("FileSystemProtection", {}).get(
+                "ReplicationOverwriteProtection"
+            ),
+        }
+        transformed_file_systems.append(transformed_file_system)
+
+    return transformed_file_systems
+
+
+@timeit
+@aws_handle_regions
+def get_efs_mount_targets(
+    fileSystems: List[Dict[str, Any]], boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    file_system_ids = []
+    for file_system in fileSystems:
+        file_system_ids.append(file_system["FileSystemId"])
+    paginator = client.get_paginator("describe_mount_targets")
+    mountTargets = []
+    for fs_id in file_system_ids:
+        for page in paginator.paginate(FileSystemId=fs_id):
+            mountTargets.extend(page.get("MountTargets", []))
+
+    return mountTargets
+
+
+@timeit
+def load_efs_mount_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} mount targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsMountTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_efs_file_systems(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} file systems for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsFileSystemSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(EfsMountTargetSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(EfsFileSystemSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Efs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        fileSystems = get_efs_file_systems(boto3_session, region)
+        tranformed_file_systems = transform_efs_file_systems(fileSystems, region)
+
+        load_efs_file_systems(
+            neo4j_session,
+            tranformed_file_systems,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        mountTargets = get_efs_mount_targets(fileSystems, boto3_session, region)
+
+        load_efs_mount_targets(
+            neo4j_session,
+            mountTargets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/identitycenter.py

@@ -140,13 +140,23 @@ def get_sso_users(
     for page in paginator.paginate(IdentityStoreId=identity_store_id):
         user_page = page.get("Users", [])
         for user in user_page:
-            if user.get("ExternalIds", None):
-                user["ExternalId"] = user.get("ExternalIds")[0].get("Id")
             users.append(user)
 
     return users
 
 
+def transform_sso_users(users: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Transform SSO users to match the expected schema
+    """
+    transformed_users = []
+    for user in users:
+        if user.get("ExternalIds") is not None:
+            user["ExternalId"] = user["ExternalIds"][0].get("Id")
+        transformed_users.append(user)
+    return transformed_users
+
+
 @timeit
 def load_sso_users(
     neo4j_session: neo4j.Session,
@@ -300,9 +310,10 @@ def sync_identity_center_instances(
             )
 
             users = get_sso_users(boto3_session, identity_store_id, region)
+            transformed_users = transform_sso_users(users)
             load_sso_users(
                 neo4j_session,
-                users,
+                transformed_users,
                 identity_store_id,
                 region,
                 current_aws_account_id,

cartography/intel/aws/inspector.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any
 from typing import Dict
+from typing import Iterator
 from typing import List
 from typing import Tuple
 
@@ -13,11 +14,14 @@ from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
+from cartography.util import batch
 from cartography.util import timeit
+from cartography.util import to_asynchronous
+from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
-# As of 7/22/24, Inspector is only available in the below regions. We will need to update this hardcoded list here over
+# As of 7/1/25, Inspector is only available in the below regions. We will need to update this hardcoded list here over
 # time. :\ https://docs.aws.amazon.com/general/latest/gr/inspector2.html
 AWS_INSPECTOR_REGIONS = {
     "us-east-2",
@@ -43,54 +47,64 @@ AWS_INSPECTOR_REGIONS = {
     "eu-central-2",
     "me-south-1",
     "sa-east-1",
+    "us-gov-east-1",
+    "us-gov-west-1",
 }
 
+BATCH_SIZE = 1000
+
+
+@aws_handle_regions
+def get_member_accounts(
+    session: boto3.session.Session,
+    region: str,
+) -> List[str]:
+    """
+    List all the accounts that have delegated access to the account specified by current_aws_account_id.
+    """
+    client = session.client("inspector2", region_name=region)
+    members = list(aws_paginate(client, "list_members", "members"))
+    accounts = [m["accountId"] for m in members]
+    return accounts
+
 
 @timeit
 @aws_handle_regions
 def get_inspector_findings(
     session: boto3.session.Session,
     region: str,
-    current_aws_account_id: str,
-) -> List[Dict[str, Any]]:
+    account_id: str,
+) -> Iterator[List[Dict[str, Any]]]:
     """
-    We must list_findings by filtering the request, otherwise the request could tiemout.
+    Query inspector2.list_findings by filtering the request, otherwise the request could timeout.
     First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away,
-    we will only fetch those in ACTIVE or SUPPRESSED statuses.
-    list_members will get us all the accounts that
-    have delegated access to the account specified by current_aws_account_id.
+    only fetch those in ACTIVE or SUPPRESSED statuses.
+    Run the query in batches of 1000 findings and return an iterator to fetch the results.
     """
     client = session.client("inspector2", region_name=region)
-
-    members = aws_paginate(client, "list_members", "members")
-    # the current host account may not be considered a "member", but we still fetch its findings
-    accounts = [current_aws_account_id] + [m["accountId"] for m in members]
-
-    findings = []
-    for account in accounts:
-        logger.info(f"Getting findings for member account {account} in region {region}")
-        findings.extend(
-            aws_paginate(
-                client,
-                "list_findings",
-                "findings",
-                filterCriteria={
-                    "awsAccountId": [
-                        {
-                            "comparison": "EQUALS",
-                            "value": account,
-                        },
-                    ],
-                    "findingStatus": [
-                        {
-                            "comparison": "NOT_EQUALS",
-                            "value": "CLOSED",
-                        },
-                    ],
+    logger.info(
+        f"Getting findings in batches of {BATCH_SIZE} for account {account_id} in region {region}"
+    )
+    aws_args: Dict[str, Any] = {
+        "filterCriteria": {
+            "awsAccountId": [
+                {
+                    "comparison": "EQUALS",
+                    "value": account_id,
                 },
-            ),
-        )
-    return findings
+            ],
+            "findingStatus": [
+                {
+                    "comparison": "NOT_EQUALS",
+                    "value": "CLOSED",
+                },
+            ],
+        }
+    }
+    findings_batches = batch(
+        aws_paginate(client, "list_findings", "findings", None, **aws_args), BATCH_SIZE
+    )
+    yield from findings_batches
 
 
 def transform_inspector_findings(
@@ -260,26 +274,24 @@ def cleanup(
     )
 
 
-@timeit
-def sync(
+def _sync_findings_for_account(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
-    current_aws_account_id: str,
+    region: str,
+    account_id: str,
     update_tag: int,
-    common_job_parameters: Dict[str, Any],
+    current_aws_account_id: str,
 ) -> None:
-    inspector_regions = [
-        region for region in regions if region in AWS_INSPECTOR_REGIONS
-    ]
-
-    for region in inspector_regions:
-        logger.info(
-            f"Syncing AWS Inspector findings for account {current_aws_account_id} and region {region}",
-        )
-        findings = get_inspector_findings(boto3_session, region, current_aws_account_id)
-        finding_data, package_data = transform_inspector_findings(findings)
-        logger.info(f"Loading {len(finding_data)} findings")
+    """
+    Syncs the findings for a given account in a given region.
+    """
+    findings = get_inspector_findings(boto3_session, region, account_id)
+    if not findings:
+        logger.info(f"No findings to sync for account {account_id} in region {region}")
+        return
+    for f_batch in findings:
+        finding_data, package_data = transform_inspector_findings(f_batch)
+        logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
             finding_data,
@@ -295,4 +307,45 @@ def sync(
             update_tag,
             current_aws_account_id,
         )
-        cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    inspector_regions = [
+        region for region in regions if region in AWS_INSPECTOR_REGIONS
+    ]
+
+    for region in inspector_regions:
+        logger.info(
+            f"Syncing AWS Inspector findings delegated to account {current_aws_account_id} and region {region}",
+        )
+        member_accounts = get_member_accounts(boto3_session, region)
+        # the current host account may not be considered a "member", but we still fetch its findings
+        member_accounts.append(current_aws_account_id)
+
+        async def async_ingest_findings_for_account(account_id: str) -> None:
+            await to_asynchronous(
+                _sync_findings_for_account,
+                neo4j_session,
+                boto3_session,
+                region,
+                account_id,
+                update_tag,
+                current_aws_account_id,
+            )
+
+        to_synchronous(
+            *[
+                async_ingest_findings_for_account(account_id)
+                for account_id in member_accounts
+            ]
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/rds.py

@@ -263,7 +263,8 @@ def get_rds_snapshot_data(
     Create an RDS boto3 client and grab all the DBSnapshots.
     """
     client = boto3_session.client("rds", region_name=region)
-    return aws_paginate(client, "describe_db_snapshots", "DBSnapshots")
+    snapshots = list(aws_paginate(client, "describe_db_snapshots", "DBSnapshots"))
+    return snapshots
 
 
 @timeit

cartography/intel/aws/resources.py

@@ -11,6 +11,7 @@ from . import config
 from . import dynamodb
 from . import ecr
 from . import ecs
+from . import efs
 from . import eks
 from . import elasticache
 from . import elasticsearch
@@ -106,4 +107,5 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
     "cloudwatch": cloudwatch.sync,
+    "efs": efs.sync,
 }

cartography/intel/entra/__init__.py

@@ -4,6 +4,7 @@ import logging
 import neo4j
 
 from cartography.config import Config
+from cartography.intel.entra.applications import sync_entra_applications
 from cartography.intel.entra.groups import sync_entra_groups
 from cartography.intel.entra.ou import sync_entra_ous
 from cartography.intel.entra.users import sync_entra_users
@@ -68,5 +69,15 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             common_job_parameters,
         )
 
+        # Run application sync
+        await sync_entra_applications(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
+
     # Execute both syncs in sequence
     asyncio.run(main())