cartography 0.109.0rc1__py3-none-any.whl → 0.110.0__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.109.0rc1'
-__version_tuple__ = version_tuple = (0, 109, 0, 'rc1')
+__version__ = version = '0.110.0'
+__version_tuple__ = version_tuple = (0, 110, 0)

cartography/cli.py

@@ -700,6 +700,15 @@ class CLI:
                 "Required if you are using the Trivy module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--trivy-results-dir",
+            type=str,
+            default=None,
+            help=(
+                "Path to a directory containing Trivy JSON results on disk. "
+                "Required if you are using the Trivy module with local results."
+            ),
+        )
         parser.add_argument(
             "--scaleway-org",
             type=str,
@@ -1089,6 +1098,9 @@ class CLI:
         if config.trivy_s3_prefix:
             logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
 
+        if config.trivy_results_dir:
+            logger.debug(f"Trivy results dir: {config.trivy_results_dir}")
+
         # Scaleway config
         if config.scaleway_secret_key_env_var:
             logger.debug(
@@ -1118,6 +1130,8 @@ class CLI:
             config.sentinelone_api_token = os.environ.get(
                 config.sentinelone_api_token_env_var
             )
+        else:
+            config.sentinelone_api_token = None
 
         # Run cartography
         try:

cartography/config.py

@@ -152,6 +152,8 @@ class Config:
     :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
     :type trivy_s3_prefix: str
     :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type trivy_results_dir: str
+    :param trivy_results_dir: Local directory containing Trivy scan results. Optional.
     :type scaleway_access_key: str
     :param scaleway_access_key: Scaleway access key. Optional.
     :type scaleway_secret_key: str
@@ -243,6 +245,7 @@ class Config:
         airbyte_api_url=None,
         trivy_s3_bucket=None,
         trivy_s3_prefix=None,
+        trivy_results_dir=None,
         scaleway_access_key=None,
         scaleway_secret_key=None,
         scaleway_org=None,
@@ -327,6 +330,7 @@ class Config:
         self.airbyte_api_url = airbyte_api_url
         self.trivy_s3_bucket = trivy_s3_bucket
         self.trivy_s3_prefix = trivy_s3_prefix
+        self.trivy_results_dir = trivy_results_dir
         self.scaleway_access_key = scaleway_access_key
         self.scaleway_secret_key = scaleway_secret_key
         self.scaleway_org = scaleway_org

cartography/data/indexes.cypher

@@ -29,14 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
@@ -158,13 +150,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);

cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json

@@ -22,8 +22,8 @@
             "iterative": false
         },
         {
-            "__comment__": "Attach EC2KeyPairs with matching fingerprints to eachother and set duplicate_keyfingerprint = True",
-            "query": "MATCH (k1:EC2KeyPair), (k2:EC2KeyPair) WHERE k1.id <> k2.id AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG return COUNT(*) as TotalCompleted",
+            "__comment__": "Attach EC2KeyPairs with matching fingerprints to each other and set duplicate_keyfingerprint = True. Use id(k1) < id(k2) to avoid Cartesian product warning and ensure O(1) comparison.",
+            "query": "MATCH (k1:EC2KeyPair) MATCH (k2:EC2KeyPair) WHERE id(k1) < id(k2) AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG RETURN COUNT(*) as TotalCompleted",
             "iterative": false
         }
     ]

cartography/intel/aws/cloudtrail_management_events.py

@@ -223,6 +223,13 @@ def transform_assume_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRole event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         if cloudtrail_event.get("userIdentity", {}).get("arn"):
             source_principal = cloudtrail_event["userIdentity"]["arn"]
             destination_principal = cloudtrail_event["requestParameters"]["roleArn"]
@@ -298,6 +305,13 @@ def transform_saml_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRoleWithSAML event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         response_elements = cloudtrail_event.get("responseElements", {})
         assumed_role_user = response_elements.get("assumedRoleUser", {})
 
@@ -370,6 +384,13 @@ def transform_web_identity_role_events_to_role_assumptions(
 
         cloudtrail_event = json.loads(event["CloudTrailEvent"])
 
+        # Skip events with null requestParameters since we can't extract roleArn
+        if not cloudtrail_event.get("requestParameters"):
+            logger.debug(
+                f"Skipping CloudTrail AssumeRoleWithWebIdentity event due to missing requestParameters. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
         user_identity = cloudtrail_event.get("userIdentity", {})
 
         if user_identity.get("type") == "WebIdentityUser" and user_identity.get(

cartography/intel/aws/cognito.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cognito.identity_pool import CognitoIdentityPoolSchema
+from cartography.models.aws.cognito.user_pool import CognitoUserPoolSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pools(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_identity_pools")
+
+    all_identity_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        identity_pools = page.get("IdentityPools", [])
+        all_identity_pools.extend(identity_pools)
+    return all_identity_pools
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pool_roles(
+    boto3_session: boto3.Session, identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    all_identity_pool_details = []
+    for identity_pool in identity_pools:
+        response = client.get_identity_pool_roles(
+            IdentityPoolId=identity_pool["IdentityPoolId"]
+        )
+        all_identity_pool_details.append(response)
+    return all_identity_pool_details
+
+
+@timeit
+@aws_handle_regions
+def get_user_pools(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-idp", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_user_pools")
+    all_user_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        user_pools = page.get("UserPools", [])
+        all_user_pools.extend(user_pools)
+    return all_user_pools
+
+
+def transform_identity_pools(
+    identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_identity_pools = []
+    for pool in identity_pools:
+        transformed_pool = {
+            "IdentityPoolId": pool["IdentityPoolId"],
+            "Region": region,
+            "Roles": list(pool.get("Roles", {}).values()),
+        }
+        transformed_identity_pools.append(transformed_pool)
+    return transformed_identity_pools
+
+
+def transform_user_pools(
+    user_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_user_pools = []
+    for pool in user_pools:
+        transformed_pool = {
+            "Id": pool["Id"],
+            "Region": region,
+            "Name": pool["Name"],
+            "Status": pool.get("Status"),
+        }
+        transformed_user_pools.append(transformed_pool)
+    return transformed_user_pools
+
+
+@timeit
+def load_identity_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito Identity Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoIdentityPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_user_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito User Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoUserPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CognitoIdentityPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(CognitoUserPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Cognito Identity Pools for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        identity_pools = get_identity_pools(boto3_session, region)
+        if not identity_pools:
+            logger.info(
+                f"No Cognito Identity Pools found in region '{region}'. Skipping sync."
+            )
+        else:
+            identity_pool_roles = get_identity_pool_roles(
+                boto3_session, identity_pools, region
+            )
+            transformed_identity_pools = transform_identity_pools(
+                identity_pool_roles, region
+            )
+
+            load_identity_pools(
+                neo4j_session,
+                transformed_identity_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            user_pools = get_user_pools(boto3_session, region)
+            transformed_user_pools = transform_user_pools(user_pools, region)
+
+            load_user_pools(
+                neo4j_session,
+                transformed_user_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ecs.py

@@ -171,9 +171,15 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
 
 def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """
-    Extract network interface ID from task attachments.
+    Extract network interface ID from task attachments and service name from group.
     """
     for task in tasks:
+        # Extract serviceName from group field
+        group = task.get("group")
+        if group and group.startswith("service:"):
+            task["serviceName"] = group.split("service:", 1)[1]
+
+        # Extract network interface ID from task attachments
         for attachment in task.get("attachments", []):
             if attachment.get("type") == "ElasticNetworkInterface":
                 details = attachment.get("details", [])

cartography/intel/aws/eventbridge.py

@@ -0,0 +1,91 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_eventbridge_rules(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_rules")
+    rules = []
+
+    for page in paginator.paginate():
+        rules.extend(page.get("Rules", []))
+
+    return rules
+
+
+@timeit
+def load_eventbridge_rules(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} rules for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeRuleSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running EventBridge cleanup job.")
+    GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing EventBridge for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        rules = get_eventbridge_rules(boto3_session, region)
+        load_eventbridge_rules(
+            neo4j_session,
+            rules,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/glue.py

@@ -0,0 +1,181 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.models.aws.glue.job import GlueJobSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_glue_connections(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_connections")
+    connections = []
+    for page in paginator.paginate():
+        connections.extend(page.get("ConnectionList", []))
+
+    return connections
+
+
+@timeit
+@aws_handle_regions
+def get_glue_jobs(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_jobs")
+    jobs = []
+    for page in paginator.paginate():
+        jobs.extend(page.get("Jobs", []))
+    return jobs
+
+
+def transform_glue_job(jobs: List[Dict[str, Any]], region: str) -> List[Dict[str, Any]]:
+    """
+    Transform Glue job data for ingestion
+    """
+    transformed_jobs = []
+    for job in jobs:
+        transformed_job = {
+            "Name": job["Name"],
+            "ProfileName": job.get("ProfileName"),
+            "JobMode": job.get("JobMode"),
+            "Connections": job.get("Connections", {}).get("Connections"),
+            "Region": region,
+            "Description": job.get("Description"),
+        }
+        transformed_jobs.append(transformed_job)
+    return transformed_jobs
+
+
+def transform_glue_connections(
+    connections: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Glue connection data for ingestion
+    """
+    transformed_connections = []
+    for connection in connections:
+        transformed_connection = {
+            "Name": connection["Name"],
+            "Description": connection.get("Description"),
+            "ConnectionType": connection.get("ConnectionType"),
+            "Status": connection.get("Status"),
+            "StatusReason": connection.get("StatusReason"),
+            "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
+                "AuthenticationType"
+            ),
+            "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
+                "SecretArn"
+            ),
+            "Region": region,
+        }
+        transformed_connections.append(transformed_connection)
+    return transformed_connections
+
+
+@timeit
+def load_glue_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} connections for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueConnectionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_glue_jobs(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} jobs for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueJobSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Glue cleanup job.")
+    GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(GlueJobSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        connections = get_glue_connections(boto3_session, region)
+        transformed_connections = transform_glue_connections(connections, region)
+        load_glue_connections(
+            neo4j_session,
+            transformed_connections,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        jobs = get_glue_jobs(boto3_session, region)
+        transformed_jobs = transform_glue_job(jobs, region)
+        load_glue_jobs(
+            neo4j_session,
+            transformed_jobs,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)