cartography 0.109.0rc1__py3-none-any.whl → 0.110.0rc1__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.109.0rc1'
-__version_tuple__ = version_tuple = (0, 109, 0, 'rc1')
+__version__ = version = '0.110.0rc1'
+__version_tuple__ = version_tuple = (0, 110, 0, 'rc1')

cartography/cli.py

@@ -254,6 +254,14 @@ class CLI:
                 "The name of environment variable containing Entra Client Secret for Service Principal Authentication."
             ),
         )
+        parser.add_argument(
+            "--entra-best-effort-mode",
+            action="store_true",
+            help=(
+                "Enable Entra ID sync best effort mode. This will allow cartography to continue "
+                "syncing other Entra ID entities and delay raising an exception until the end of the sync."
+            ),
+        )
         parser.add_argument(
             "--aws-requested-syncs",
             type=str,
@@ -700,6 +708,15 @@ class CLI:
                 "Required if you are using the Trivy module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--trivy-results-dir",
+            type=str,
+            default=None,
+            help=(
+                "Path to a directory containing Trivy JSON results on disk. "
+                "Required if you are using the Trivy module with local results."
+            ),
+        )
         parser.add_argument(
             "--scaleway-org",
             type=str,
@@ -1089,6 +1106,9 @@ class CLI:
         if config.trivy_s3_prefix:
             logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
 
+        if config.trivy_results_dir:
+            logger.debug(f"Trivy results dir: {config.trivy_results_dir}")
+
         # Scaleway config
         if config.scaleway_secret_key_env_var:
             logger.debug(
@@ -1118,6 +1138,8 @@ class CLI:
             config.sentinelone_api_token = os.environ.get(
                 config.sentinelone_api_token_env_var
             )
+        else:
+            config.sentinelone_api_token = None
 
         # Run cartography
         try:

cartography/config.py

@@ -51,6 +51,9 @@ class Config:
     :param entra_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
     :type entra_client_secret: str
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
+    :type entra_best_effort_mode: bool
+    :param entra_best_effort_mode: If True, Entra ID sync will continue on errors and raise an aggregated
+        exception at the end of the sync. If False (default), exceptions will be raised immediately.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
     :type aws_guardduty_severity_threshold: str
@@ -152,6 +155,8 @@ class Config:
     :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
     :type trivy_s3_prefix: str
     :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type trivy_results_dir: str
+    :param trivy_results_dir: Local directory containing Trivy scan results. Optional.
     :type scaleway_access_key: str
     :param scaleway_access_key: Scaleway access key. Optional.
     :type scaleway_secret_key: str
@@ -162,6 +167,8 @@ class Config:
     :param sentinelone_api_url: SentinelOne API URL. Optional.
     :type sentinelone_api_token: string
     :param sentinelone_api_token: SentinelOne API token for authentication. Optional.
+    :type sentinelone_api_token_env_var: string
+    :param sentinelone_api_token_env_var: The name of an environment variable containing the SentinelOne API token. Optional.
     :type sentinelone_account_ids: list[str]
     :param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
     """
@@ -187,6 +194,7 @@ class Config:
         entra_tenant_id=None,
         entra_client_id=None,
         entra_client_secret=None,
+        entra_best_effort_mode=False,
         aws_requested_syncs=None,
         aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
@@ -243,11 +251,13 @@ class Config:
         airbyte_api_url=None,
         trivy_s3_bucket=None,
         trivy_s3_prefix=None,
+        trivy_results_dir=None,
         scaleway_access_key=None,
         scaleway_secret_key=None,
         scaleway_org=None,
         sentinelone_api_url=None,
         sentinelone_api_token=None,
+        sentinelone_api_token_env_var=None,
         sentinelone_account_ids=None,
     ):
         self.neo4j_uri = neo4j_uri
@@ -271,6 +281,7 @@ class Config:
         self.entra_tenant_id = entra_tenant_id
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
+        self.entra_best_effort_mode = entra_best_effort_mode
         self.aws_requested_syncs = aws_requested_syncs
         self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
@@ -327,9 +338,11 @@ class Config:
         self.airbyte_api_url = airbyte_api_url
         self.trivy_s3_bucket = trivy_s3_bucket
         self.trivy_s3_prefix = trivy_s3_prefix
+        self.trivy_results_dir = trivy_results_dir
         self.scaleway_access_key = scaleway_access_key
         self.scaleway_secret_key = scaleway_secret_key
         self.scaleway_org = scaleway_org
         self.sentinelone_api_url = sentinelone_api_url
         self.sentinelone_api_token = sentinelone_api_token
+        self.sentinelone_api_token_env_var = sentinelone_api_token_env_var
         self.sentinelone_account_ids = sentinelone_account_ids

cartography/data/indexes.cypher

@@ -29,14 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
@@ -158,13 +150,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);

cartography/intel/aws/cloudtrail_management_events.py

@@ -223,6 +223,13 @@ def transform_assume_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRole event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         if cloudtrail_event.get("userIdentity", {}).get("arn"):
             source_principal = cloudtrail_event["userIdentity"]["arn"]
             destination_principal = cloudtrail_event["requestParameters"]["roleArn"]
@@ -298,6 +305,13 @@ def transform_saml_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRoleWithSAML event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         response_elements = cloudtrail_event.get("responseElements", {})
         assumed_role_user = response_elements.get("assumedRoleUser", {})
 
@@ -370,6 +384,13 @@ def transform_web_identity_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRoleWithWebIdentity event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         user_identity = cloudtrail_event.get("userIdentity", {})
 
         if user_identity.get("type") == "WebIdentityUser" and user_identity.get(

cartography/intel/aws/eventbridge.py

@@ -0,0 +1,91 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_eventbridge_rules(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_rules")
+    rules = []
+
+    for page in paginator.paginate():
+        rules.extend(page.get("Rules", []))
+
+    return rules
+
+
+@timeit
+def load_eventbridge_rules(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} rules for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeRuleSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running EventBridge cleanup job.")
+    GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing EventBridge for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        rules = get_eventbridge_rules(boto3_session, region)
+        load_eventbridge_rules(
+            neo4j_session,
+            rules,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/glue.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_glue_connections(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_connections")
+    connections = []
+    for page in paginator.paginate():
+        connections.extend(page.get("ConnectionList", []))
+
+    return connections
+
+
+def transform_glue_connections(
+    connections: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Glue connection data for ingestion
+    """
+    transformed_connections = []
+    for connection in connections:
+        transformed_connection = {
+            "Name": connection["Name"],
+            "Description": connection.get("Description"),
+            "ConnectionType": connection.get("ConnectionType"),
+            "Status": connection.get("Status"),
+            "StatusReason": connection.get("StatusReason"),
+            "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
+                "AuthenticationType"
+            ),
+            "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
+                "SecretArn"
+            ),
+            "Region": region,
+        }
+        transformed_connections.append(transformed_connection)
+    return transformed_connections
+
+
+@timeit
+def load_glue_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} connections for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueConnectionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Glue cleanup job.")
+    GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        connections = get_glue_connections(boto3_session, region)
+        transformed_connections = transform_glue_connections(connections, region)
+        load_glue_connections(
+            neo4j_session,
+            transformed_connections,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/identitycenter.py

@@ -7,6 +7,7 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.identitycenter.awsidentitycenter import (
     AWSIdentityCenterInstanceSchema,
@@ -14,9 +15,11 @@ from cartography.models.aws.identitycenter.awsidentitycenter import (
 from cartography.models.aws.identitycenter.awspermissionset import (
     AWSPermissionSetSchema,
 )
+from cartography.models.aws.identitycenter.awspermissionset import (
+    RoleAssignmentAllowedByMatchLink,
+)
 from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -120,6 +123,8 @@ def load_permission_sets(
         InstanceArn=instance_arn,
         Region=region,
         AWS_ID=aws_account_id,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
     )
 
 
@@ -220,31 +225,64 @@ def get_role_assignments(
     return role_assignments
 
 
+@timeit
+def get_permset_roles(
+    neo4j_session: neo4j.Session,
+    role_assignments: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """
+    Enrich role assignments with exact role ARNs by querying existing permission set relationships.
+    Uses the ASSIGNED_TO_ROLE relationships created when permission sets were loaded.
+    """
+    # Get unique permission set ARNs from role assignments
+    permset_ids = list({ra["PermissionSetArn"] for ra in role_assignments})
+
+    query = """
+    MATCH (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-(permset:AWSPermissionSet)
+    WHERE permset.arn IN $PermSetIds
+    RETURN permset.arn AS PermissionSetArn, role.arn AS RoleArn
+    """
+    result = neo4j_session.run(query, PermSetIds=permset_ids)
+    permset_to_role = [record.data() for record in result]
+
+    # Create mapping from permission set ARN to role ARN
+    permset_to_role_map = {
+        entry["PermissionSetArn"]: entry["RoleArn"] for entry in permset_to_role
+    }
+
+    # Enrich role assignments with exact role ARNs
+    enriched_assignments = []
+    for assignment in role_assignments:
+        role_arn = permset_to_role_map.get(assignment["PermissionSetArn"])
+        enriched_assignments.append(
+            {
+                **assignment,
+                "RoleArn": role_arn,
+            }
+        )
+
+    return enriched_assignments
+
+
 @timeit
 def load_role_assignments(
     neo4j_session: neo4j.Session,
     role_assignments: List[Dict],
+    aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     """
-    Load role assignments into the graph
+    Load role assignments into the graph using MatchLink schema
     """
     logger.info(f"Loading {len(role_assignments)} role assignments")
-    if role_assignments:
-        neo4j_session.run(
-            """
-            UNWIND $role_assignments AS ra
-            MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
-            (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
-            (permset:AWSPermissionSet {id: ra.PermissionSetArn})
-            MATCH (sso:AWSSSOUser {id: ra.UserId})
-            MERGE (role)-[r:ALLOWED_BY]->(sso)
-            SET r.lastupdated = $aws_update_tag,
-            r.permission_set_arn = ra.PermissionSetArn
-            """,
-            role_assignments=role_assignments,
-            aws_update_tag=aws_update_tag,
-        )
+    load_matchlinks(
+        neo4j_session,
+        RoleAssignmentAllowedByMatchLink(),
+        role_assignments,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
+    )
 
 
 @timeit
@@ -262,11 +300,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(
         neo4j_session,
     )
-    run_cleanup_job(
-        "aws_import_identity_center_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
-    )
+
+    # Clean up role assignment MatchLinks
+    GraphJob.from_matchlink(
+        RoleAssignmentAllowedByMatchLink(),
+        "AWSAccount",
+        common_job_parameters["AWS_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)
 
 
 @timeit
@@ -327,9 +368,16 @@ def sync_identity_center_instances(
                 instance_arn,
                 region,
             )
-            load_role_assignments(
+
+            # Enrich role assignments with exact role ARNs using permission set relationships
+            enriched_role_assignments = get_permset_roles(
                 neo4j_session,
                 role_assignments,
+            )
+            load_role_assignments(
+                neo4j_session,
+                enriched_role_assignments,
+                current_aws_account_id,
                 update_tag,
             )