cartography 0.108.0rc1__py3-none-any.whl → 0.109.0__py3-none-any.whl

cartography/intel/aws/rds.py

@@ -6,12 +6,17 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.rds.cluster import RDSClusterSchema
+from cartography.models.aws.rds.instance import RDSInstanceSchema
+from cartography.models.aws.rds.snapshot import RDSSnapshotSchema
+from cartography.models.aws.rds.subnet_group import DBSubnetGroupSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
 from cartography.util import dict_value_to_str
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -39,7 +44,7 @@ def get_rds_cluster_data(
 @timeit
 def load_rds_clusters(
     neo4j_session: neo4j.Session,
-    data: Dict,
+    data: List[Dict],
     region: str,
     current_aws_account_id: str,
     aws_update_tag: int,
@@ -47,91 +52,13 @@ def load_rds_clusters(
     """
     Ingest the RDS clusters to neo4j and link them to necessary nodes.
     """
-    ingest_rds_cluster = """
-    UNWIND $Clusters as rds_cluster
-        MERGE (cluster:RDSCluster{id: rds_cluster.DBClusterArn})
-        ON CREATE SET cluster.firstseen = timestamp(),
-            cluster.arn = rds_cluster.DBClusterArn
-        SET cluster.allocated_storage = rds_cluster.AllocatedStorage,
-            cluster.availability_zones = rds_cluster.AvailabilityZones,
-            cluster.backup_retention_period = rds_cluster.BackupRetentionPeriod,
-            cluster.character_set_name = rds_cluster.CharacterSetName,
-            cluster.database_name = rds_cluster.DatabaseName,
-            cluster.db_cluster_identifier = rds_cluster.DBClusterIdentifier,
-            cluster.db_parameter_group = rds_cluster.DBClusterParameterGroup,
-            cluster.status = rds_cluster.Status,
-            cluster.earliest_restorable_time = rds_cluster.EarliestRestorableTime,
-            cluster.endpoint = rds_cluster.Endpoint,
-            cluster.reader_endpoint = rds_cluster.ReaderEndpoint,
-            cluster.multi_az = rds_cluster.MultiAZ,
-            cluster.engine = rds_cluster.Engine,
-            cluster.engine_version = rds_cluster.EngineVersion,
-            cluster.latest_restorable_time = rds_cluster.LatestRestorableTime,
-            cluster.port = rds_cluster.Port,
-            cluster.master_username = rds_cluster.MasterUsername,
-            cluster.preferred_backup_window = rds_cluster.PreferredBackupWindow,
-            cluster.preferred_maintenance_window = rds_cluster.PreferredMaintenanceWindow,
-            cluster.hosted_zone_id = rds_cluster.HostedZoneId,
-            cluster.storage_encrypted = rds_cluster.StorageEncrypted,
-            cluster.kms_key_id = rds_cluster.KmsKeyId,
-            cluster.db_cluster_resource_id = rds_cluster.DbClusterResourceId,
-            cluster.clone_group_id = rds_cluster.CloneGroupId,
-            cluster.cluster_create_time = rds_cluster.ClusterCreateTime,
-            cluster.earliest_backtrack_time = rds_cluster.EarliestBacktrackTime,
-            cluster.backtrack_window = rds_cluster.BacktrackWindow,
-            cluster.backtrack_consumed_change_records = rds_cluster.BacktrackConsumedChangeRecords,
-            cluster.capacity = rds_cluster.Capacity,
-            cluster.engine_mode = rds_cluster.EngineMode,
-            cluster.scaling_configuration_info_min_capacity = rds_cluster.ScalingConfigurationInfoMinCapacity,
-            cluster.scaling_configuration_info_max_capacity = rds_cluster.ScalingConfigurationInfoMaxCapacity,
-            cluster.scaling_configuration_info_auto_pause = rds_cluster.ScalingConfigurationInfoAutoPause,
-            cluster.deletion_protection = rds_cluster.DeletionProtection,
-            cluster.region = $Region,
-            cluster.lastupdated = $aws_update_tag
-        WITH cluster
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(cluster)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    for cluster in data:
-        # TODO: track read replicas
-        # TODO: track associated roles
-        # TODO: track security groups
-        # TODO: track subnet groups
-
-        cluster["EarliestRestorableTime"] = dict_value_to_str(
-            cluster,
-            "EarliestRestorableTime",
-        )
-        cluster["LatestRestorableTime"] = dict_value_to_str(
-            cluster,
-            "LatestRestorableTime",
-        )
-        cluster["ClusterCreateTime"] = dict_value_to_str(cluster, "ClusterCreateTime")
-        cluster["EarliestBacktrackTime"] = dict_value_to_str(
-            cluster,
-            "EarliestBacktrackTime",
-        )
-        cluster["ScalingConfigurationInfoMinCapacity"] = cluster.get(
-            "ScalingConfigurationInfo",
-            {},
-        ).get("MinCapacity")
-        cluster["ScalingConfigurationInfoMaxCapacity"] = cluster.get(
-            "ScalingConfigurationInfo",
-            {},
-        ).get("MaxCapacity")
-        cluster["ScalingConfigurationInfoAutoPause"] = cluster.get(
-            "ScalingConfigurationInfo",
-            {},
-        ).get("AutoPause")
-
-    neo4j_session.run(
-        ingest_rds_cluster,
-        Clusters=data,
+    load(
+        neo4j_session,
+        RDSClusterSchema(),
+        data,
+        lastupdated=aws_update_tag,
         Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
@@ -156,101 +83,22 @@ def get_rds_instance_data(
 @timeit
 def load_rds_instances(
     neo4j_session: neo4j.Session,
-    data: Dict,
+    data: List[Dict],
     region: str,
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     """
-    Ingest the RDS instances to neo4j and link them to necessary nodes.
-    """
-    ingest_rds_instance = """
-    UNWIND $Instances as rds_instance
-        MERGE (rds:RDSInstance{id: rds_instance.DBInstanceArn})
-        ON CREATE SET rds.firstseen = timestamp(),
-            rds.arn = rds_instance.DBInstanceArn
-        SET rds.db_instance_identifier = rds_instance.DBInstanceIdentifier,
-            rds.db_instance_class = rds_instance.DBInstanceClass,
-            rds.engine = rds_instance.Engine,
-            rds.master_username = rds_instance.MasterUsername,
-            rds.db_name = rds_instance.DBName,
-            rds.instance_create_time = rds_instance.InstanceCreateTime,
-            rds.availability_zone = rds_instance.AvailabilityZone,
-            rds.multi_az = rds_instance.MultiAZ,
-            rds.engine_version = rds_instance.EngineVersion,
-            rds.publicly_accessible = rds_instance.PubliclyAccessible,
-            rds.db_cluster_identifier = rds_instance.DBClusterIdentifier,
-            rds.storage_encrypted = rds_instance.StorageEncrypted,
-            rds.kms_key_id = rds_instance.KmsKeyId,
-            rds.dbi_resource_id = rds_instance.DbiResourceId,
-            rds.ca_certificate_identifier = rds_instance.CACertificateIdentifier,
-            rds.enhanced_monitoring_resource_arn = rds_instance.EnhancedMonitoringResourceArn,
-            rds.monitoring_role_arn = rds_instance.MonitoringRoleArn,
-            rds.performance_insights_enabled = rds_instance.PerformanceInsightsEnabled,
-            rds.performance_insights_kms_key_id = rds_instance.PerformanceInsightsKMSKeyId,
-            rds.region = rds_instance.Region,
-            rds.deletion_protection = rds_instance.DeletionProtection,
-            rds.preferred_backup_window = rds_instance.PreferredBackupWindow,
-            rds.latest_restorable_time = rds_instance.LatestRestorableTime,
-            rds.preferred_maintenance_window = rds_instance.PreferredMaintenanceWindow,
-            rds.backup_retention_period = rds_instance.BackupRetentionPeriod,
-            rds.endpoint_address = rds_instance.EndpointAddress,
-            rds.endpoint_hostedzoneid = rds_instance.EndpointHostedZoneId,
-            rds.endpoint_port = rds_instance.EndpointPort,
-            rds.iam_database_authentication_enabled = rds_instance.IAMDatabaseAuthenticationEnabled,
-            rds.auto_minor_version_upgrade = rds_instance.AutoMinorVersionUpgrade,
-            rds.lastupdated = $aws_update_tag
-        WITH rds
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(rds)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    read_replicas = []
-    clusters = []
-    secgroups = []
-    subnets = []
-
-    for rds in data:
-        ep = _validate_rds_endpoint(rds)
-
-        # Keep track of instances that are read replicas so we can attach them to their source instances later
-        if rds.get("ReadReplicaSourceDBInstanceIdentifier"):
-            read_replicas.append(rds)
-
-        # Keep track of instances that are cluster members so we can attach them to their source clusters later
-        if rds.get("DBClusterIdentifier"):
-            clusters.append(rds)
-
-        if rds.get("VpcSecurityGroups"):
-            secgroups.append(rds)
-
-        if rds.get("DBSubnetGroup"):
-            subnets.append(rds)
-
-        rds["InstanceCreateTime"] = dict_value_to_str(rds, "InstanceCreateTime")
-        rds["LatestRestorableTime"] = dict_value_to_str(rds, "LatestRestorableTime")
-        rds["EndpointAddress"] = ep.get("Address")
-        rds["EndpointHostedZoneId"] = ep.get("HostedZoneId")
-        rds["EndpointPort"] = ep.get("Port")
-
-    neo4j_session.run(
-        ingest_rds_instance,
-        Instances=data,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
-    )
-    _attach_ec2_security_groups(neo4j_session, secgroups, aws_update_tag)
-    _attach_ec2_subnet_groups(
+    Ingest the RDS instances to Neo4j and link them to necessary nodes.
+    """
+    load(
         neo4j_session,
-        subnets,
-        region,
-        current_aws_account_id,
-        aws_update_tag,
+        RDSInstanceSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
     )
-    _attach_read_replicas(neo4j_session, read_replicas, aws_update_tag)
-    _attach_clusters(neo4j_session, clusters, aws_update_tag)
 
 
 @timeit
@@ -270,7 +118,7 @@ def get_rds_snapshot_data(
 @timeit
 def load_rds_snapshots(
     neo4j_session: neo4j.Session,
-    data: Dict,
+    data: List[Dict],
     region: str,
     current_aws_account_id: str,
     aws_update_tag: int,
@@ -278,266 +126,13 @@ def load_rds_snapshots(
     """
     Ingest the RDS snapshots to neo4j and link them to necessary nodes.
     """
-    ingest_rds_snapshot = """
-    UNWIND $Snapshots as rds_snapshot
-        MERGE (snapshot:RDSSnapshot{id: rds_snapshot.DBSnapshotArn})
-        ON CREATE SET snapshot.firstseen = timestamp(),
-            snapshot.arn = rds_snapshot.DBSnapshotArn
-        SET snapshot.db_snapshot_identifier = rds_snapshot.DBSnapshotIdentifier,
-            snapshot.db_instance_identifier = rds_snapshot.DBInstanceIdentifier,
-            snapshot.snapshot_create_time = rds_snapshot.SnapshotCreateTime,
-            snapshot.engine = rds_snapshot.Engine,
-            snapshot.allocated_storage = rds_snapshot.AllocatedStorage,
-            snapshot.status = rds_snapshot.Status,
-            snapshot.port = rds_snapshot.Port,
-            snapshot.availability_zone = rds_snapshot.AvailabilityZone,
-            snapshot.vpc_id = rds_snapshot.VpcId,
-            snapshot.instance_create_time = rds_snapshot.InstanceCreateTime,
-            snapshot.master_username = rds_snapshot.MasterUsername,
-            snapshot.engine_version = rds_snapshot.EngineVersion,
-            snapshot.license_model = rds_snapshot.LicenseModel,
-            snapshot.snapshot_type = rds_snapshot.SnapshotType,
-            snapshot.iops = rds_snapshot.Iops,
-            snapshot.option_group_name = rds_snapshot.OptionGroupName,
-            snapshot.percent_progress = rds_snapshot.PercentProgress,
-            snapshot.source_region = rds_snapshot.SourceRegion,
-            snapshot.source_db_snapshot_identifier = rds_snapshot.SourceDBSnapshotIdentifier,
-            snapshot.storage_type = rds_snapshot.StorageType,
-            snapshot.tde_credential_arn = rds_snapshot.TdeCredentialArn,
-            snapshot.encrypted = rds_snapshot.Encrypted,
-            snapshot.kms_key_id = rds_snapshot.KmsKeyId,
-            snapshot.timezone = rds_snapshot.Timezone,
-            snapshot.iam_database_authentication_enabled = rds_snapshot.IAMDatabaseAuthenticationEnabled,
-            snapshot.processor_features = rds_snapshot.ProcessorFeatures,
-            snapshot.dbi_resource_id = rds_snapshot.DbiResourceId,
-            snapshot.original_snapshot_create_time = rds_snapshot.OriginalSnapshotCreateTime,
-            snapshot.snapshot_database_time = rds_snapshot.SnapshotDatabaseTime,
-            snapshot.snapshot_target = rds_snapshot.SnapshotTarget,
-            snapshot.storage_throughput = rds_snapshot.StorageThroughput,
-            snapshot.region = $Region,
-            snapshot.lastupdated = $aws_update_tag
-        WITH snapshot
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(snapshot)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-
-    snapshots = transform_rds_snapshots(data)
-
-    neo4j_session.run(
-        ingest_rds_snapshot,
-        Snapshots=snapshots,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
-    )
-    _attach_snapshots(neo4j_session, snapshots, aws_update_tag)
-
-
-@timeit
-def _attach_snapshots(
-    neo4j_session: neo4j.Session,
-    snapshots: List[Dict],
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach snapshots to their source instance
-    """
-    attach_member_to_source = """
-    UNWIND $Snapshots as snapshot
-        MATCH (rdsInstance:RDSInstance {db_instance_identifier: snapshot.DBInstanceIdentifier}),
-        (rdsSnapshot:RDSSnapshot {arn: snapshot.DBSnapshotArn})
-        MERGE (rdsInstance)-[r:IS_SNAPSHOT_SOURCE]->(rdsSnapshot)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    neo4j_session.run(
-        attach_member_to_source,
-        Snapshots=snapshots,
-        aws_update_tag=aws_update_tag,
-    )
-
-
-@timeit
-def _attach_ec2_subnet_groups(
-    neo4j_session: neo4j.Session,
-    instances: List[Dict],
-    region: str,
-    current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach RDS instances to their EC2 subnet groups
-    """
-    attach_rds_to_subnet_group = """
-    UNWIND $SubnetGroups as rds_sng
-        MERGE (sng:DBSubnetGroup{id: rds_sng.arn})
-        ON CREATE SET sng.firstseen = timestamp()
-        SET sng.name = rds_sng.DBSubnetGroupName,
-            sng.vpc_id = rds_sng.VpcId,
-            sng.description = rds_sng.DBSubnetGroupDescription,
-            sng.status = rds_sng.DBSubnetGroupStatus,
-            sng.lastupdated = $aws_update_tag
-        WITH sng, rds_sng.instance_arn AS instance_arn
-        MATCH(rds:RDSInstance{id: instance_arn})
-        MERGE(rds)-[r:MEMBER_OF_DB_SUBNET_GROUP]->(sng)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    db_sngs = []
-    for instance in instances:
-        db_sng = instance["DBSubnetGroup"]
-        db_sng["arn"] = _get_db_subnet_group_arn(
-            region,
-            current_aws_account_id,
-            db_sng["DBSubnetGroupName"],
-        )
-        db_sng["instance_arn"] = instance["DBInstanceArn"]
-        db_sngs.append(db_sng)
-    neo4j_session.run(
-        attach_rds_to_subnet_group,
-        SubnetGroups=db_sngs,
-        aws_update_tag=aws_update_tag,
-    )
-    _attach_ec2_subnets_to_subnetgroup(
+    load(
         neo4j_session,
-        db_sngs,
-        region,
-        current_aws_account_id,
-        aws_update_tag,
-    )
-
-
-@timeit
-def _attach_ec2_subnets_to_subnetgroup(
-    neo4j_session: neo4j.Session,
-    db_subnet_groups: List[Dict],
-    region: str,
-    current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach EC2Subnets to their DB Subnet Group.
-
-    From https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_VPC.WorkingWithRDSInstanceinaVPC.html:
-    `Each DB subnet group should have subnets in at least two Availability Zones in a given region. When creating a DB
-    instance in a VPC, you must select a DB subnet group. Amazon RDS uses that DB subnet group and your preferred
-    Availability Zone to select a subnet and an IP address within that subnet to associate with your DB instance.`
-    """
-    attach_subnets_to_sng = """
-    UNWIND $Subnets as rds_sn
-        MATCH(sng:DBSubnetGroup{id: rds_sn.sng_arn})
-        MERGE(subnet:EC2Subnet{subnetid: rds_sn.sn_id})
-        ON CREATE SET subnet.firstseen = timestamp()
-        MERGE(sng)-[r:RESOURCE]->(subnet)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag,
-        subnet.availability_zone = rds_sn.az,
-        subnet.lastupdated = $aws_update_tag
-    """
-    subnets = []
-    for subnet_group in db_subnet_groups:
-        for subnet in subnet_group.get("Subnets", []):
-            sn_id = subnet.get("SubnetIdentifier")
-            sng_arn = _get_db_subnet_group_arn(
-                region,
-                current_aws_account_id,
-                subnet_group["DBSubnetGroupName"],
-            )
-            az = subnet.get("SubnetAvailabilityZone", {}).get("Name")
-            subnets.append(
-                {
-                    "sn_id": sn_id,
-                    "sng_arn": sng_arn,
-                    "az": az,
-                },
-            )
-    neo4j_session.run(
-        attach_subnets_to_sng,
-        Subnets=subnets,
-        aws_update_tag=aws_update_tag,
-    )
-
-
-@timeit
-def _attach_ec2_security_groups(
-    neo4j_session: neo4j.Session,
-    instances: List[Dict],
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach an RDS instance to its EC2SecurityGroups
-    """
-    attach_rds_to_group = """
-    UNWIND $Groups as rds_sg
-        MATCH (rds:RDSInstance{id: rds_sg.arn})
-        MERGE (sg:EC2SecurityGroup{id: rds_sg.group_id})
-        MERGE (rds)-[m:MEMBER_OF_EC2_SECURITY_GROUP]->(sg)
-        ON CREATE SET m.firstseen = timestamp()
-        SET m.lastupdated = $aws_update_tag
-    """
-    groups = []
-    for instance in instances:
-        for group in instance["VpcSecurityGroups"]:
-            groups.append(
-                {
-                    "arn": instance["DBInstanceArn"],
-                    "group_id": group["VpcSecurityGroupId"],
-                },
-            )
-    neo4j_session.run(
-        attach_rds_to_group,
-        Groups=groups,
-        aws_update_tag=aws_update_tag,
-    )
-
-
-@timeit
-def _attach_read_replicas(
-    neo4j_session: neo4j.Session,
-    read_replicas: List[Dict],
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach read replicas to their source instances
-    """
-    attach_replica_to_source = """
-    UNWIND $Replicas as rds_replica
-        MATCH (replica:RDSInstance{id: rds_replica.DBInstanceArn}),
-        (source:RDSInstance{db_instance_identifier: rds_replica.ReadReplicaSourceDBInstanceIdentifier})
-        MERGE (replica)-[r:IS_READ_REPLICA_OF]->(source)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    neo4j_session.run(
-        attach_replica_to_source,
-        Replicas=read_replicas,
-        aws_update_tag=aws_update_tag,
-    )
-
-
-@timeit
-def _attach_clusters(
-    neo4j_session: neo4j.Session,
-    cluster_members: List[Dict],
-    aws_update_tag: int,
-) -> None:
-    """
-    Attach cluster members to their source clusters
-    """
-    attach_member_to_source = """
-    UNWIND $Members as rds_cluster_member
-    MATCH (member:RDSInstance{id: rds_cluster_member.DBInstanceArn}),
-    (source:RDSCluster{db_cluster_identifier: rds_cluster_member.DBClusterIdentifier})
-    MERGE (member)-[r:IS_CLUSTER_MEMBER_OF]->(source)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
-    """
-    neo4j_session.run(
-        attach_member_to_source,
-        Members=cluster_members,
-        aws_update_tag=aws_update_tag,
+        RDSSnapshotSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
     )
 
 
@@ -571,8 +166,48 @@ def _get_db_subnet_group_arn(
     )
 
 
-@timeit
-def transform_rds_snapshots(data: Dict) -> List[Dict]:
+def transform_rds_clusters(data: List[Dict]) -> List[Dict]:
+    """
+    Transform RDS cluster data for Neo4j ingestion
+    """
+    clusters = []
+
+    for cluster in data:
+        # Copy the cluster data
+        transformed_cluster = cluster.copy()
+
+        # Convert datetime fields
+        transformed_cluster["EarliestRestorableTime"] = dict_value_to_str(
+            cluster, "EarliestRestorableTime"
+        )
+        transformed_cluster["LatestRestorableTime"] = dict_value_to_str(
+            cluster, "LatestRestorableTime"
+        )
+        transformed_cluster["ClusterCreateTime"] = dict_value_to_str(
+            cluster, "ClusterCreateTime"
+        )
+        transformed_cluster["EarliestBacktrackTime"] = dict_value_to_str(
+            cluster, "EarliestBacktrackTime"
+        )
+
+        # Extract scaling configuration info
+        scaling_config = cluster.get("ScalingConfigurationInfo", {})
+        transformed_cluster["ScalingConfigurationInfoMinCapacity"] = scaling_config.get(
+            "MinCapacity"
+        )
+        transformed_cluster["ScalingConfigurationInfoMaxCapacity"] = scaling_config.get(
+            "MaxCapacity"
+        )
+        transformed_cluster["ScalingConfigurationInfoAutoPause"] = scaling_config.get(
+            "AutoPause"
+        )
+
+        clusters.append(transformed_cluster)
+
+    return clusters
+
+
+def transform_rds_snapshots(data: List[Dict]) -> List[Dict]:
     snapshots = []
 
     for snapshot in data:
@@ -599,18 +234,151 @@ def transform_rds_snapshots(data: Dict) -> List[Dict]:
     return snapshots
 
 
+def transform_rds_instances(
+    data: List[Dict], region: str, current_aws_account_id: str
+) -> List[Dict]:
+    """
+    Transform RDS instance data for Neo4j ingestion
+    """
+    instances = []
+
+    for instance in data:
+        # Copy the instance data
+        transformed_instance = instance.copy()
+
+        # Extract security group IDs for the relationship
+        security_group_ids = []
+        if instance.get("VpcSecurityGroups"):
+            for group in instance["VpcSecurityGroups"]:
+                security_group_ids.append(group["VpcSecurityGroupId"])
+
+        transformed_instance["security_group_ids"] = security_group_ids
+
+        # Handle read replica source identifier for the relationship
+        if instance.get("ReadReplicaSourceDBInstanceIdentifier"):
+            transformed_instance["read_replica_source_identifier"] = instance[
+                "ReadReplicaSourceDBInstanceIdentifier"
+            ]
+
+        # Handle cluster identifier for the relationship
+        if instance.get("DBClusterIdentifier"):
+            transformed_instance["db_cluster_identifier"] = instance[
+                "DBClusterIdentifier"
+            ]
+
+        # Handle subnet group data for the relationship
+        if instance.get("DBSubnetGroup"):
+            db_subnet_group = instance["DBSubnetGroup"]
+            transformed_instance["db_subnet_group_arn"] = _get_db_subnet_group_arn(
+                region, current_aws_account_id, db_subnet_group["DBSubnetGroupName"]
+            )
+
+        # Handle endpoint data
+        ep = _validate_rds_endpoint(instance)
+        transformed_instance["EndpointAddress"] = ep.get("Address")
+        transformed_instance["EndpointHostedZoneId"] = ep.get("HostedZoneId")
+        transformed_instance["EndpointPort"] = ep.get("Port")
+
+        # Convert datetime fields
+        transformed_instance["InstanceCreateTime"] = dict_value_to_str(
+            instance, "InstanceCreateTime"
+        )
+        transformed_instance["LatestRestorableTime"] = dict_value_to_str(
+            instance, "LatestRestorableTime"
+        )
+
+        instances.append(transformed_instance)
+
+    return instances
+
+
+def transform_rds_subnet_groups(
+    data: List[Dict], region: str, current_aws_account_id: str
+) -> List[Dict]:
+    """
+    Transform RDS subnet group data for Neo4j ingestion
+    """
+    subnet_groups_dict = {}
+
+    for instance in data:
+        if instance.get("DBSubnetGroup"):
+            db_subnet_group = instance["DBSubnetGroup"]
+            db_subnet_group_arn = _get_db_subnet_group_arn(
+                region, current_aws_account_id, db_subnet_group["DBSubnetGroupName"]
+            )
+
+            # If this subnet group doesn't exist yet, create it
+            if db_subnet_group_arn not in subnet_groups_dict:
+                subnet_groups_dict[db_subnet_group_arn] = {
+                    "id": db_subnet_group_arn,
+                    "name": db_subnet_group["DBSubnetGroupName"],
+                    "vpc_id": db_subnet_group["VpcId"],
+                    "description": db_subnet_group["DBSubnetGroupDescription"],
+                    "status": db_subnet_group["SubnetGroupStatus"],
+                    "db_instance_identifier": [],
+                    "subnet_ids": [],
+                }
+
+            # Add this RDS instance to the subnet group's list
+            if instance.get("DBInstanceIdentifier"):
+                subnet_groups_dict[db_subnet_group_arn][
+                    "db_instance_identifier"
+                ].append(instance["DBInstanceIdentifier"])
+
+            # Add subnet IDs from the DB subnet group
+            for subnet in db_subnet_group.get("Subnets", []):
+                subnet_id = subnet.get("SubnetIdentifier")
+                if (
+                    subnet_id
+                    and subnet_id
+                    not in subnet_groups_dict[db_subnet_group_arn]["subnet_ids"]
+                ):
+                    subnet_groups_dict[db_subnet_group_arn]["subnet_ids"].append(
+                        subnet_id
+                    )
+
+    return list(subnet_groups_dict.values())
+
+
+@timeit
+def load_rds_subnet_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Ingest the RDS subnet groups to Neo4j and link them to necessary nodes.
+    """
+    load(
+        neo4j_session,
+        DBSubnetGroupSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup_rds_instances_and_db_subnet_groups(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
     """
-    Remove RDS graph nodes and DBSubnetGroups that were created from other ingestion runs
+    Remove RDS instances and DB subnet groups that weren't updated in this sync run
     """
-    run_cleanup_job(
-        "aws_import_rds_instances_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    logger.debug("Running RDS instances and DB subnet groups cleanup job")
+
+    # Clean up RDS instances
+    GraphJob.from_node_schema(RDSInstanceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+    # Clean up DB subnet groups
+    GraphJob.from_node_schema(DBSubnetGroupSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -620,12 +388,12 @@ def cleanup_rds_clusters(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Remove RDS cluster graph nodes
+    Remove RDS clusters that weren't updated in this sync run
     """
-    run_cleanup_job(
-        "aws_import_rds_clusters_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    logger.debug("Running RDS clusters cleanup job")
+
+    GraphJob.from_node_schema(RDSClusterSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -635,12 +403,12 @@ def cleanup_rds_snapshots(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Remove RDS snapshots graph nodes
+    Remove RDS snapshots that weren't updated in this sync run
     """
-    run_cleanup_job(
-        "aws_import_rds_snapshots_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    logger.debug("Running RDS snapshots cleanup job")
+
+    GraphJob.from_node_schema(RDSSnapshotSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -654,16 +422,19 @@ def sync_rds_clusters(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Grab RDS instance data from AWS, ingest to neo4j, and run the cleanup job.
+    Grab RDS cluster data from AWS, ingest to neo4j, and run the cleanup job.
     """
     for region in regions:
         logger.info(
-            "Syncing RDS for region '%s' in account '%s'.",
+            "Syncing RDS clusters for region '%s' in account '%s'.",
             region,
             current_aws_account_id,
         )
         data = get_rds_cluster_data(boto3_session, region)
-        load_rds_clusters(neo4j_session, data, region, current_aws_account_id, update_tag)  # type: ignore
+        transformed_data = transform_rds_clusters(data)
+        load_rds_clusters(
+            neo4j_session, transformed_data, region, current_aws_account_id, update_tag
+        )
     cleanup_rds_clusters(neo4j_session, common_job_parameters)
 
 
@@ -681,12 +452,23 @@ def sync_rds_instances(
     """
     for region in regions:
         logger.info(
-            "Syncing RDS for region '%s' in account '%s'.",
+            "Syncing RDS instances for region '%s' in account '%s'.",
             region,
             current_aws_account_id,
         )
         data = get_rds_instance_data(boto3_session, region)
-        load_rds_instances(neo4j_session, data, region, current_aws_account_id, update_tag)  # type: ignore
+        transformed_data = transform_rds_instances(data, region, current_aws_account_id)
+        load_rds_instances(
+            neo4j_session, transformed_data, region, current_aws_account_id, update_tag
+        )
+
+        # Load subnet groups from RDS instances
+        subnet_group_data = transform_rds_subnet_groups(
+            data, region, current_aws_account_id
+        )
+        load_rds_subnet_groups(
+            neo4j_session, subnet_group_data, region, current_aws_account_id, update_tag
+        )
     cleanup_rds_instances_and_db_subnet_groups(neo4j_session, common_job_parameters)
 
 
@@ -704,12 +486,15 @@ def sync_rds_snapshots(
     """
     for region in regions:
         logger.info(
-            "Syncing RDS for region '%s' in account '%s'.",
+            "Syncing RDS snapshots for region '%s' in account '%s'.",
             region,
             current_aws_account_id,
         )
         data = get_rds_snapshot_data(boto3_session, region)
-        load_rds_snapshots(neo4j_session, data, region, current_aws_account_id, update_tag)  # type: ignore
+        transformed_data = transform_rds_snapshots(data)
+        load_rds_snapshots(
+            neo4j_session, transformed_data, region, current_aws_account_id, update_tag
+        )
     cleanup_rds_snapshots(neo4j_session, common_job_parameters)