cartography 0.110.0rc1__py3-none-any.whl → 0.111.0rc1__py3-none-any.whl

cartography/intel/aws/rds.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.rds.cluster import RDSClusterSchema
+from cartography.models.aws.rds.event_subscription import RDSEventSubscriptionSchema
 from cartography.models.aws.rds.instance import RDSInstanceSchema
 from cartography.models.aws.rds.snapshot import RDSSnapshotSchema
 from cartography.models.aws.rds.subnet_group import DBSubnetGroupSchema
@@ -136,6 +137,38 @@ def load_rds_snapshots(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_rds_event_subscription_data(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("rds", region_name=region)
+    paginator = client.get_paginator("describe_event_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page["EventSubscriptionsList"])
+    return subscriptions
+
+
+@timeit
+def load_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RDSEventSubscriptionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 def _validate_rds_endpoint(rds: Dict) -> Dict:
     """
     Get Endpoint from RDS data structure.  Log to debug if an Endpoint field does not exist.
@@ -292,6 +325,28 @@ def transform_rds_instances(
     return instances
 
 
+def transform_rds_event_subscriptions(data: List[Dict]) -> List[Dict]:
+    subscriptions = []
+    for subscription in data:
+        transformed = {
+            "CustSubscriptionId": subscription.get("CustSubscriptionId"),
+            "EventSubscriptionArn": subscription.get("EventSubscriptionArn"),
+            "CustomerAwsId": subscription.get("CustomerAwsId"),
+            "SnsTopicArn": subscription.get("SnsTopicArn"),
+            "SourceType": subscription.get("SourceType"),
+            "Status": subscription.get("Status"),
+            "Enabled": subscription.get("Enabled"),
+            "SubscriptionCreationTime": dict_value_to_str(
+                subscription, "SubscriptionCreationTime"
+            ),
+            "event_categories": subscription.get("EventCategoriesList") or None,
+            "source_ids": subscription.get("SourceIdsList") or None,
+            "lastupdated": None,  # This will be set by the loader
+        }
+        subscriptions.append(transformed)
+    return subscriptions
+
+
 def transform_rds_subnet_groups(
     data: List[Dict], region: str, current_aws_account_id: str
 ) -> List[Dict]:
@@ -412,6 +467,20 @@ def cleanup_rds_snapshots(
     )
 
 
+@timeit
+def cleanup_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Remove RDS event subscriptions that weren't updated in this sync run
+    """
+    logger.debug("Running RDS event subscriptions cleanup job")
+    GraphJob.from_node_schema(RDSEventSubscriptionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
 @timeit
 def sync_rds_clusters(
     neo4j_session: neo4j.Session,
@@ -498,6 +567,32 @@ def sync_rds_snapshots(
     cleanup_rds_snapshots(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Grab RDS event subscription data from AWS, ingest to neo4j, and run the cleanup job.
+    """
+    for region in regions:
+        logger.info(
+            "Syncing RDS event subscriptions for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        data = get_rds_event_subscription_data(boto3_session, region)
+        transformed = transform_rds_event_subscriptions(data)
+        load_rds_event_subscriptions(
+            neo4j_session, transformed, region, current_aws_account_id, update_tag
+        )
+    cleanup_rds_event_subscriptions(neo4j_session, common_job_parameters)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -531,6 +626,16 @@ def sync(
         update_tag,
         common_job_parameters,
     )
+
+    sync_rds_event_subscriptions(
+        neo4j_session,
+        boto3_session,
+        regions,
+        current_aws_account_id,
+        update_tag,
+        common_job_parameters,
+    )
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/resources.py

@@ -9,6 +9,7 @@ from . import cloudtrail
 from . import cloudtrail_management_events
 from . import cloudwatch
 from . import codebuild
+from . import cognito
 from . import config
 from . import dynamodb
 from . import ecr
@@ -116,6 +117,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "efs": efs.sync,
     "guardduty": guardduty.sync,
     "codebuild": codebuild.sync,
+    "cognito": cognito.sync,
     "eventbridge": eventbridge.sync,
     "glue": glue.sync,
 }

cartography/intel/aws/route53.py

@@ -398,7 +398,9 @@ def link_sub_zones(
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(z:AWSDNSZone)
         <-[:MEMBER_OF_DNS_ZONE]-(record:DNSRecord{type:"NS"})
         -[:DNS_POINTS_TO]->(ns:NameServer)<-[:NAMESERVER]-(z2:AWSDNSZone)
-    WHERE record.name=z2.name AND NOT z=z2
+        WHERE record.name = z2.name AND
+        z2.name ENDS WITH '.' + z.name AND
+        NOT z = z2
     RETURN z.id as zone_id, z2.id as subzone_id
     """
     zone_to_subzone = neo4j_session.read_transaction(

cartography/intel/aws/s3.py

@@ -71,6 +71,7 @@ def get_s3_bucket_details(
         Dict[str, Any],
         Dict[str, Any],
         Dict[str, Any],
+        Dict[str, Any],
     ]
 
     async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
@@ -88,6 +89,7 @@ def get_s3_bucket_details(
             versioning,
             public_access_block,
             bucket_ownership_controls,
+            bucket_logging,
         ) = await asyncio.gather(
             to_asynchronous(get_acl, bucket, client),
             to_asynchronous(get_policy, bucket, client),
@@ -95,6 +97,7 @@ def get_s3_bucket_details(
             to_asynchronous(get_versioning, bucket, client),
             to_asynchronous(get_public_access_block, bucket, client),
             to_asynchronous(get_bucket_ownership_controls, bucket, client),
+            to_asynchronous(get_bucket_logging, bucket, client),
         )
         return (
             bucket["Name"],
@@ -104,6 +107,7 @@ def get_s3_bucket_details(
             versioning,
             public_access_block,
             bucket_ownership_controls,
+            bucket_logging,
         )
 
     bucket_details = to_synchronous(
@@ -241,6 +245,29 @@ def get_bucket_ownership_controls(
     return bucket_ownership_controls
 
 
+@timeit
+@aws_handle_regions
+def get_bucket_logging(
+    bucket: Dict, client: botocore.client.BaseClient
+) -> Optional[Dict]:
+    """
+    Gets the S3 bucket logging status configuration.
+    """
+    bucket_logging = None
+    try:
+        bucket_logging = client.get_bucket_logging(Bucket=bucket["Name"])
+    except ClientError as e:
+        if _is_common_exception(e, bucket):
+            pass
+        else:
+            raise
+    except EndpointConnectionError:
+        logger.warning(
+            f"Failed to retrieve S3 bucket logging status for {bucket['Name']} - Could not connect to the endpoint URL",
+        )
+    return bucket_logging
+
+
 @timeit
 def _is_common_exception(e: Exception, bucket: Dict) -> bool:
     error_msg = "Failed to retrieve S3 bucket detail"
@@ -319,6 +346,7 @@ def _load_s3_acls(
         "aws_s3acl_analysis.json",
         neo4j_session,
         {"AWS_ID": aws_account_id},
+        package="cartography.data.jobs.scoped_analysis",
     )
 
 
@@ -479,6 +507,30 @@ def _load_bucket_ownership_controls(
     )
 
 
+@timeit
+def _load_bucket_logging(
+    neo4j_session: neo4j.Session,
+    bucket_logging_configs: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 bucket logging status configuration into neo4j.
+    """
+    # Load basic logging status
+    ingest_bucket_logging = """
+    UNWIND $bucket_logging_configs AS bucket_logging
+    MATCH (bucket:S3Bucket{name: bucket_logging.bucket})
+    SET bucket.logging_enabled = bucket_logging.logging_enabled,
+        bucket.logging_target_bucket = bucket_logging.target_bucket,
+        bucket.lastupdated = $update_tag
+    """
+    neo4j_session.run(
+        ingest_bucket_logging,
+        bucket_logging_configs=bucket_logging_configs,
+        update_tag=update_tag,
+    )
+
+
 def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
     set_defaults = """
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
@@ -516,6 +568,7 @@ def load_s3_details(
     versioning_configs: List[Dict] = []
     public_access_block_configs: List[Dict] = []
     bucket_ownership_controls_configs: List[Dict] = []
+    bucket_logging_configs: List[Dict] = []
     for (
         bucket,
         acl,
@@ -524,6 +577,7 @@ def load_s3_details(
         versioning,
         public_access_block,
         bucket_ownership_controls,
+        bucket_logging,
     ) in s3_details_iter:
         parsed_acls = parse_acl(acl, bucket, aws_account_id)
         if parsed_acls is not None:
@@ -551,6 +605,9 @@ def load_s3_details(
         )
         if parsed_bucket_ownership_controls is not None:
             bucket_ownership_controls_configs.append(parsed_bucket_ownership_controls)
+        parsed_bucket_logging = parse_bucket_logging(bucket, bucket_logging)
+        if parsed_bucket_logging is not None:
+            bucket_logging_configs.append(parsed_bucket_logging)
 
     # cleanup existing policy properties set on S3 Buckets
     run_cleanup_job(
@@ -569,6 +626,7 @@ def load_s3_details(
     _load_bucket_ownership_controls(
         neo4j_session, bucket_ownership_controls_configs, update_tag
     )
+    _load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag)
     _set_default_values(neo4j_session, aws_account_id)
 
 
@@ -851,6 +909,52 @@ def parse_bucket_ownership_controls(
     }
 
 
+def parse_bucket_logging(bucket: str, bucket_logging: Optional[Dict]) -> Optional[Dict]:
+    """Parses the S3 bucket logging status configuration and returns a dict of the relevant data"""
+    # Logging status object JSON looks like:
+    # {
+    #     'LoggingEnabled': {
+    #         'TargetBucket': 'string',
+    #         'TargetGrants': [
+    #             {
+    #                 'Grantee': {
+    #                     'DisplayName': 'string',
+    #                     'EmailAddress': 'string',
+    #                     'ID': 'string',
+    #                     'Type': 'CanonicalUser'|'AmazonCustomerByEmail'|'Group',
+    #                     'URI': 'string'
+    #                 },
+    #                 'Permission': 'FULL_CONTROL'|'READ'|'WRITE'
+    #             },
+    #         ],
+    #         'TargetPrefix': 'string',
+    #         'TargetObjectKeyFormat': {
+    #             'SimplePrefix': {},
+    #             'PartitionedPrefix': {
+    #                 'PartitionDateSource': 'EventTime'|'DeliveryTime'
+    #             }
+    #         }
+    #     }
+    # }
+    # Or empty dict {} if logging is not enabled
+    if bucket_logging is None:
+        return None
+
+    logging_config = bucket_logging.get("LoggingEnabled", {})
+    if not logging_config:
+        return {
+            "bucket": bucket,
+            "logging_enabled": False,
+            "target_bucket": None,
+        }
+
+    return {
+        "bucket": bucket,
+        "logging_enabled": True,
+        "target_bucket": logging_config.get("TargetBucket"),
+    }
+
+
 @timeit
 def parse_notification_configuration(
     bucket: str, notification_config: Optional[Dict]

cartography/intel/entra/__init__.py

@@ -1,14 +1,13 @@
 import asyncio
-import datetime
 import logging
-from traceback import TracebackException
-from typing import Awaitable
-from typing import Callable
 
 import neo4j
 
 from cartography.config import Config
-from cartography.intel.entra.resources import RESOURCE_FUNCTIONS
+from cartography.intel.entra.applications import sync_entra_applications
+from cartography.intel.entra.groups import sync_entra_groups
+from cartography.intel.entra.ou import sync_entra_ous
+from cartography.intel.entra.users import sync_entra_users
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -40,46 +39,45 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     }
 
     async def main() -> None:
-        failed_stages = []
-        exception_tracebacks = []
+        # Run user sync
+        await sync_entra_users(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        async def run_stage(name: str, func: Callable[..., Awaitable[None]]) -> None:
-            try:
-                await func(
-                    neo4j_session,
-                    config.entra_tenant_id,
-                    config.entra_client_id,
-                    config.entra_client_secret,
-                    config.update_tag,
-                    common_job_parameters,
-                )
-            except Exception as e:
-                if config.entra_best_effort_mode:
-                    timestamp = datetime.datetime.now()
-                    failed_stages.append(name)
-                    exception_traceback = TracebackException.from_exception(e)
-                    traceback_string = "".join(exception_traceback.format())
-                    exception_tracebacks.append(
-                        f"{timestamp} - Exception for stage {name}\n{traceback_string}"
-                    )
-                    logger.warning(
-                        f"Caught exception syncing {name}. entra-best-effort-mode is on so we are continuing "
-                        "on to the next Entra sync. All exceptions will be aggregated and re-logged at the end of the sync.",
-                        exc_info=True,
-                    )
-                else:
-                    logger.error("Error during Entra sync", exc_info=True)
-                    raise
+        # Run group sync
+        await sync_entra_groups(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        for name, func in RESOURCE_FUNCTIONS:
-            await run_stage(name, func)
+        # Run OU sync
+        await sync_entra_ous(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        if failed_stages:
-            logger.error(
-                f"Entra sync failed for the following stages: {', '.join(failed_stages)}. "
-                "See the logs for more details.",
-            )
-            raise Exception("\n".join(exception_tracebacks))
+        # Run application sync
+        await sync_entra_applications(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-    # Execute all syncs in sequence
+    # Execute both syncs in sequence
     asyncio.run(main())

cartography/intel/entra/applications.py

@@ -172,11 +172,12 @@ async def get_app_role_assignments(
             )
             continue
         except Exception as e:
+            # Only catch truly unexpected errors - these should be rare
             logger.error(
                 f"Unexpected error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}",
                 exc_info=True,
             )
-            raise
+            continue
 
     logger.info(f"Retrieved {len(assignments)} app role assignments total")
     return assignments

cartography/intel/entra/ou.py

@@ -43,7 +43,7 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
                 current_request = None
         except Exception as e:
             logger.error(f"Failed to retrieve administrative units: {str(e)}")
-            raise
+            current_request = None
 
     return all_units
 

cartography/intel/github/__init__.py

@@ -3,7 +3,6 @@ import json
 import logging
 
 import neo4j
-from requests import exceptions
 
 import cartography.intel.github.repos
 import cartography.intel.github.teams
@@ -34,27 +33,24 @@ def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     }
     # run sync for the provided github tokens
     for auth_data in auth_tokens["organization"]:
-        try:
-            cartography.intel.github.users.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.repos.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.teams.sync_github_teams(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-        except exceptions.RequestException as e:
-            logger.error("Could not complete request to the GitHub API: %s", e)
+        cartography.intel.github.users.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.repos.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.teams.sync_github_teams(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )

cartography/intel/github/repos.py

@@ -405,9 +405,16 @@ def _create_default_branch_id(repo_url: str, default_branch_ref_id: str) -> str:
 
 def _create_git_url_from_ssh_url(ssh_url: str) -> str:
     """
-    Return a git:// URL from the given ssh_url
+    Convert SSH URL to git:// URL.
+    Example:
+        git@github.com:cartography-cncf/cartography.git
+        -> git://github.com/cartography-cncf/cartography.git
     """
-    return ssh_url.replace("/", ":").replace("git@", "git://")
+    # Remove the user part (e.g., "git@")
+    _, host_and_path = ssh_url.split("@", 1)
+    # Replace first ':' (separating host and repo) with '/'
+    host, path = host_and_path.split(":", 1)
+    return f"git://{host}/{path}"
 
 
 def _transform_repo_objects(input_repo_object: Dict, out_repo_list: List[Dict]) -> None:
@@ -647,9 +654,6 @@ def _transform_dependency_graph(
             requirements = dep.get("requirements", "")
             package_manager = dep.get("packageManager", "").upper()
 
-            # Extract version from requirements string if available
-            pinned_version = _extract_version_from_requirements(requirements)
-
             # Create ecosystem-specific canonical name
             canonical_name = _canonicalize_dependency_name(
                 package_name, package_manager
@@ -658,11 +662,12 @@ def _transform_dependency_graph(
             # Create ecosystem identifier
             ecosystem = package_manager.lower() if package_manager else "unknown"
 
-            # Create simple dependency ID using canonical name and version
+            # Create simple dependency ID using canonical name and requirements
             # This allows the same dependency to be shared across multiple repos
+            requirements_for_id = (requirements or "").strip()
             dependency_id = (
-                f"{canonical_name}|{pinned_version}"
-                if pinned_version
+                f"{canonical_name}|{requirements_for_id}"
+                if requirements_for_id
                 else canonical_name
             )
 
@@ -677,15 +682,12 @@ def _transform_dependency_graph(
                     "id": dependency_id,
                     "name": canonical_name,
                     "original_name": package_name,  # Keep original for reference
-                    "version": pinned_version,
                     "requirements": normalized_requirements,
                     "ecosystem": ecosystem,
                     "package_manager": package_manager,
                     "manifest_path": manifest_path,
                     "manifest_id": manifest_id,
                     "repo_url": repo_url,
-                    # Add separate fields for easier querying
-                    "repo_name": repo_url.split("/")[-1] if repo_url else "",
                     "manifest_file": (
                         manifest_path.split("/")[-1] if manifest_path else ""
                     ),
@@ -698,33 +700,6 @@ def _transform_dependency_graph(
         logger.info(f"Found {dependencies_added} dependencies in {repo_name}")
 
 
-def _extract_version_from_requirements(requirements: Optional[str]) -> Optional[str]:
-    """
-    Extract a pinned version from a requirements string if it exists.
-    Examples: "1.2.3" -> "1.2.3", "^1.2.3" -> None, ">=1.0,<2.0" -> None
-    """
-    if not requirements or not requirements.strip():
-        return None
-
-    # Handle exact version specifications (no operators)
-    if requirements and not any(
-        op in requirements for op in ["^", "~", ">", "<", "=", "*"]
-    ):
-        stripped = requirements.strip()
-        return stripped if stripped else None
-
-    # Handle == specifications
-    if "==" in requirements:
-        parts = requirements.split("==")
-        if len(parts) == 2:
-            version = parts[1].strip()
-            # Remove any trailing constraints
-            version = version.split(",")[0].split(" ")[0]
-            return version if version else None
-
-    return None
-
-
 def _canonicalize_dependency_name(name: str, package_manager: Optional[str]) -> str:
     """
     Canonicalize dependency names based on ecosystem conventions.

cartography/intel/kubernetes/__init__.py

@@ -6,6 +6,7 @@ from cartography.config import Config
 from cartography.intel.kubernetes.clusters import sync_kubernetes_cluster
 from cartography.intel.kubernetes.namespaces import sync_namespaces
 from cartography.intel.kubernetes.pods import sync_pods
+from cartography.intel.kubernetes.rbac import sync_kubernetes_rbac
 from cartography.intel.kubernetes.secrets import sync_secrets
 from cartography.intel.kubernetes.services import sync_services
 from cartography.intel.kubernetes.util import get_k8s_clients
@@ -38,6 +39,9 @@ def start_k8s_ingestion(session: Session, config: Config) -> None:
             common_job_parameters["CLUSTER_ID"] = cluster_info.get("id")
 
             sync_namespaces(session, client, config.update_tag, common_job_parameters)
+            sync_kubernetes_rbac(
+                session, client, config.update_tag, common_job_parameters
+            )
             all_pods = sync_pods(
                 session,
                 client,