cartography 0.84.0__py3-none-any.whl → 0.85.1__py3-none-any.whl

cartography/data/indexes.cypher

@@ -23,10 +23,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSDNSZone) ON (n.zoneid);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSDNSZone) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSGroup) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSInspectorFinding) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSInspectorFinding) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSInspectorPackage) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSInspectorPackage) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSInternetGateway) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSInternetGateway) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
@@ -93,8 +89,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint);
-CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.id);
@@ -125,8 +119,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainerDefinition) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ECSContainer) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:EKSCluster) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EKSCluster) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ElasticacheCluster) ON (n.lastupdated);

cartography/data/jobs/analysis/aws_s3acl_analysis.json

@@ -1,22 +1,27 @@
 {
   "statements": [
     {
+      "__comment__": "READ -> ListBucket, ListBucketVersions, ListBucketMultipartUploads",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:ListBucket', 's3:ListBucketVersions', 's3:ListBucketMultipartUploads']",
       "iterative": false
     },
     {
-      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE'\nAND (acl.ownerid = acl.granteeid)\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:DeleteObjectVersion']",
+      "__comment__": "WRITE -> PutObject",
+      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:PutObject']",
       "iterative": false
     },
     {
-      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:DeleteObjectVersion']",
+      "__comment__": "READ_ACP -> GetBucketAcl",
+      "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'READ_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:GetBucketAcl']",
       "iterative": false
     },
     {
+      "__comment__": "WRITE_ACP -> PutBucketAcl",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'WRITE_ACP'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:PutBucketAcl']",
       "iterative": false
     },
     {
+      "__comment__": "FULL_CONTROL -> Pretty much everything",
       "query": "MATCH (acl:S3Acl)-[:APPLIES_TO]->(bucket:S3Bucket)<-[:RESOURCE]-(aws:AWSAccount{id: $AWS_ID})\nWHERE acl.uri IN ['http://acs.amazonaws.com/groups/global/AllUsers', 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'] AND acl.permission = 'FULL_CONTROL'\nSET bucket.anonymous_access = true, bucket.anonymous_actions = coalesce(bucket.anonymous_actions, []) + ['s3:ListBucket', 's3:ListBucketVersions', 's3:ListBucketMultipartUploads', 's3:PutObject', 's3:DeleteObject', 's3:DeleteObjectVersion', 's3:PutBucketAcl']",
       "iterative": false
     }],

cartography/intel/aws/ec2/instances.py

@@ -13,10 +13,10 @@ from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.ec2.instances import EC2InstanceSchema
 from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
-from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema
+from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema
 from cartography.models.aws.ec2.reservations import EC2ReservationSchema
-from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema
-from cartography.models.aws.ec2.subnets import EC2SubnetSchema
+from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema
+from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
 from cartography.models.aws.ec2.volumes import EBSVolumeInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
@@ -183,7 +183,7 @@ def load_ec2_subnets(
 ) -> None:
     load(
         neo4j_session,
-        EC2SubnetSchema(),
+        EC2SubnetInstanceSchema(),
         subnet_list,
         Region=region,
         AWS_ID=current_aws_account_id,
@@ -219,7 +219,7 @@ def load_ec2_security_groups(
 ) -> None:
     load(
         neo4j_session,
-        EC2SecurityGroupSchema(),
+        EC2SecurityGroupInstanceSchema(),
         sg_list,
         Region=region,
         AWS_ID=current_aws_account_id,
@@ -237,7 +237,7 @@ def load_ec2_network_interfaces(
 ) -> None:
     load(
         neo4j_session,
-        EC2NetworkInterfaceSchema(),
+        EC2NetworkInterfaceInstanceSchema(),
         network_interface_list,
         Region=region,
         AWS_ID=current_aws_account_id,

cartography/intel/aws/ec2/network_interfaces.py

@@ -1,5 +1,7 @@
 import logging
 import re
+from collections import namedtuple
+from typing import Any
 from typing import Dict
 from typing import List
 
@@ -7,18 +9,30 @@ import boto3
 import neo4j
 
 from .util import get_botocore_config
+from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema
+from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema
+from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema
+from cartography.models.aws.ec2.subnet_networkinterface import EC2SubnetNetworkInterfaceSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+Ec2NetworkData = namedtuple(
+    "Ec2NetworkData", [
+        "network_interface_list",
+        "private_ip_list",
+        "sg_list",
+        "subnet_list",
+    ],
+)
+
 
 @timeit
 @aws_handle_regions
-def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_network_interfaces')
     subnets: List[Dict] = []
@@ -27,256 +41,215 @@ def get_network_interface_data(boto3_session: boto3.session.Session, region: str
     return subnets
 
 
-@timeit
-def load_network_interfaces(
-    neo4j_session: neo4j.Session, data: Dict, region: str, aws_account_id: str,
-    update_tag: int,
-) -> None:
-    """
-    Creates (:NetworkInterface),
-    (:NetworkInterface)-[:RESOURCE]->(:AWSAccount),
-    (:NetworkInterface)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(:EC2SecurityGroup),
-    (:NetworkInterface)-[:PART_OF_SUBNET]->(:EC2Subnet),
-    (:PrivateIpAddress),
-    (:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(:PrivateIpAddress)
-    """
-    logger.debug("Loading %d network interfaces in %s.", len(data), region)
-    ingest_network_interfaces = """
-    UNWIND $network_interfaces AS network_interface
-        MERGE (netinf:NetworkInterface{id: network_interface.NetworkInterfaceId})
-        ON CREATE SET netinf.firstseen = timestamp()
-        SET netinf.lastupdated = $update_tag,
-            netinf.mac_address = network_interface.MacAddress,
-            netinf.description = network_interface.Description,
-            netinf.private_ip_address = network_interface.PrivateIpAddress,
-            netinf.id = network_interface.NetworkInterfaceId,
-            netinf.private_dns_name = network_interface.PrivateDnsName,
-            netinf.status = network_interface.Status,
-            netinf.subnetid = network_interface.SubnetId,
-            netinf.interface_type = network_interface.InterfaceType,
-            netinf.requester_managed = network_interface.RequesterManaged,
-            netinf.source_dest_check = network_interface.SourceDestCheck,
-            netinf.requester_id = network_interface.RequesterId,
-            netinf.public_ip = network_interface.Association.PublicIp
-        WITH network_interface, netinf
+def transform_network_interface_data(data_list: List[Dict[str, Any]], region: str) -> Ec2NetworkData:
+    network_interface_list = []
+    private_ip_list = []
+    sg_list = []
+    subnet_list = []
 
-        UNWIND network_interface.PrivateIpAddresses AS private_ip_address
-            MERGE (private_ip:EC2PrivateIp{id: network_interface.NetworkInterfaceId + ':'
-                + private_ip_address.PrivateIpAddress})
-            ON CREATE SET private_ip.firstseen = timestamp()
-            SET private_ip.lastupdated = $update_tag,
-                private_ip.network_interface_id = network_interface.NetworkInterfaceId,
-                private_ip.primary = private_ip_address.Primary,
-                private_ip.private_ip_address = private_ip_address.PrivateIpAddress,
-                private_ip.public_ip = private_ip_address.Association.PublicIp,
-                private_ip.ip_owner_id = private_ip_address.Association.IpOwnerId
-
-            MERGE (netinf)-[r:PRIVATE_IP_ADDRESS]->(private_ip)
-            ON CREATE SET r.firstseen = timestamp()
-            SET r.lastupdated = $update_tag
-        WITH network_interface, netinf
-
-        UNWIND network_interface.Groups AS security_group
-            MERGE (sg:EC2SecurityGroup{id: security_group.GroupId})
-            ON CREATE SET sg.firstseen = timestamp()
-            SET sg.lastupdated = $update_tag
-            MERGE (netinf)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(sg)
-            ON CREATE SET r.firstseen = timestamp()
-            SET r.lastupdated = $update_tag
-        WITH network_interface, netinf
-        MERGE (acc:AWSAccount{id: $aws_account_id})
-        ON CREATE SET acc.firstseen = timestamp(), acc.inscope=true
-        SET acc.lastupdated = $update_tag
-        MERGE (acc)-[r:RESOURCE]->(netinf)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-        WITH network_interface, netinf
-
-        MERGE (snet:EC2Subnet{subnetid: network_interface.SubnetId})
-        ON CREATE SET snet.firstseen = timestamp()
-        SET snet.lastupdated = $update_tag
-        MERGE (netinf)-[r:PART_OF_SUBNET]->(snet)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-    neo4j_session.run(
-        ingest_network_interfaces, network_interfaces=data, update_tag=update_tag,
-        region=region, aws_account_id=aws_account_id,
+    for network_interface in data_list:
+        # Parse network interface description for ELB association
+        # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/
+        elb_v1_id = None
+        elb_v2_id = None
+        elb_match = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', ''))
+        if elb_match:
+            elb_v1_id = f'{elb_match[1]}-{elb_match[2]}.elb.{region}.amazonaws.com'
+        else:
+            elb_match = re.match(r'^ELB (.*)', network_interface.get('Description', ''))
+            if elb_match:
+                elb_v2_id = elb_match[1]
+        # TODO issue #1024 change this to arn when ready
+        network_interface_id = network_interface['NetworkInterfaceId']
+        network_interface_list.append(
+            {
+                'Id': network_interface_id,
+                'NetworkInterfaceId': network_interface['NetworkInterfaceId'],
+                'Description': network_interface['Description'],
+                'InstanceId': network_interface.get('Attachment', {}).get('InstanceId'),
+                'InterfaceType': network_interface['InterfaceType'],
+                'MacAddress': network_interface['MacAddress'],
+                'PrivateDnsName': network_interface['PrivateDnsName'],
+                'PrivateIpAddress': network_interface['PrivateIpAddress'],
+                'PublicIp': network_interface.get('Association', {}).get('PublicIp'),
+                'RequesterId': network_interface.get('RequesterId'),
+                'RequesterManaged': network_interface['RequesterManaged'],
+                'SourceDestCheck': network_interface['SourceDestCheck'],
+                'Status': network_interface['Status'],
+                'SubnetId': network_interface['SubnetId'],
+                'ElbV1Id': elb_v1_id,
+                'ElbV2Id': elb_v2_id,
+            },
+        )
+        if network_interface.get('PrivateIpAddresses'):
+            for private_ip_address in network_interface['PrivateIpAddresses']:
+                private_ip_list.append(
+                    {
+                        'Id': f"{network_interface['NetworkInterfaceId']}:{private_ip_address['PrivateIpAddress']}",
+                        'NetworkInterfaceId': network_interface['NetworkInterfaceId'],
+                        'IpOwnerId': private_ip_address.get('Association', {}).get('IpOwnerId'),
+                        'Primary': private_ip_address['Primary'],
+                        'PrivateIpAddress': private_ip_address['PrivateIpAddress'],
+                        'PublicIp': private_ip_address.get('Association', {}).get('PublicIp'),
+                    },
+                )
+
+        if network_interface.get("Groups"):
+            for group in network_interface["Groups"]:
+                sg_list.append(
+                    {
+                        'GroupId': group['GroupId'],
+                        'NetworkInterfaceId': network_interface_id,
+                    },
+                )
+
+        subnet_id = network_interface.get('SubnetId')
+        if subnet_id:
+            subnet_list.append(
+                {
+                    'NetworkInterfaceId': network_interface_id,
+                    'SubnetId': subnet_id,
+                    'ElbV1Id': elb_v1_id,
+                    'ElbV2Id': elb_v2_id,
+                },
+            )
+
+    return Ec2NetworkData(
+        network_interface_list=network_interface_list,
+        private_ip_list=private_ip_list,
+        sg_list=sg_list,
+        subnet_list=subnet_list,
     )
 
 
 @timeit
-def load_network_interface_instance_relations(
-    neo4j_session: neo4j.Session, instance_associations: List[Dict], region: str, aws_account_id: str, update_tag: int,
+def load_network_interfaces(
+        neo4j_session: neo4j.Session,
+        data: List[Dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
 ) -> None:
-    """
-    Creates (:EC2Instance)-[:NETWORK_INTERFACE]->(:NetworkInterface)
-    """
-    ingest_network_interface_instance_relations = """
-    UNWIND $instance_associations AS instance_association
-        MATCH (netinf:NetworkInterface{id: instance_association.netinf_id}),
-            (instance:EC2Instance{id: instance_association.instance_id})
-        MERGE (instance)-[r:NETWORK_INTERFACE]->(netinf)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-    logger.debug("Attaching %d EC2 instances to network interfaces in %s.", len(instance_associations), region)
-    neo4j_session.run(
-        ingest_network_interface_instance_relations, instance_associations=instance_associations,
-        update_tag=update_tag, region=region, aws_account_id=aws_account_id,
+    logger.info(f"Loading {len(data)} network interfaces in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkInterfaceSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
     )
 
 
 @timeit
-def load_network_interface_elb_relations(
-    neo4j_session: neo4j.Session, elb_associations: List[Dict], region: str,
-    aws_account_id: str, update_tag: int,
+def load_private_ip_network_interface(
+        neo4j_session: neo4j.Session,
+        data: List[Dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
 ) -> None:
     """
-    Creates (:LoadBalancer)-[:NETWORK_INTERFACE]->(:NetworkInterface)
+    Private IPs as known by describe-network-interfaces.
     """
-    ingest_network_interface_elb_relations = """
-    UNWIND $elb_associations AS elb_association
-        MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}),
-            (elb:LoadBalancer{name: elb_association.elb_name})
-        MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-    logger.debug("Attaching %d ELBs to network interfaces in %s.", len(elb_associations), region)
-    neo4j_session.run(
-        ingest_network_interface_elb_relations, elb_associations=elb_associations,
-        update_tag=update_tag, region=region, aws_account_id=aws_account_id,
+    logger.info(f"Loading {len(data)} private IPs in {region}.")
+    load(
+        neo4j_session,
+        EC2PrivateIpNetworkInterfaceSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
     )
 
 
 @timeit
-def load_network_interface_elbv2_relations(
-    neo4j_session: neo4j.Session, elb_associations_v2: List[Dict], region: str,
-    aws_account_id: str, update_tag: int,
+def load_security_group_network_interface(
+        neo4j_session: neo4j.Session,
+        data: List[Dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
 ) -> None:
     """
-    Creates (:LoadBalancerV2)-[:NETWORK_INTERFACE]->(:NetworkInterface)
-    """
-    ingest_network_interface_elb2_relations = """
-    UNWIND $elb_associations AS elb_association
-        MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}),
-            (elb:LoadBalancerV2{id: elb_association.elb_id})
-        MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
+    Security groups as known by describe-network-interfaces.
     """
-    logger.debug("Attaching %d ELB V2s to network interfaces in %s.", len(elb_associations_v2), region)
-    neo4j_session.run(
-        ingest_network_interface_elb2_relations, elb_associations=elb_associations_v2,
-        update_tag=update_tag, region=region, aws_account_id=aws_account_id,
+    logger.info(f"Loading {len(data)} security groups in {region}.")
+    load(
+        neo4j_session,
+        EC2SecurityGroupNetworkInterfaceSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
     )
 
 
 @timeit
-def load_network_interface_instance_to_subnet_relations(neo4j_session: neo4j.Session, update_tag: int) -> None:
-    """
-    Creates (:EC2Instance)-[:PART_OF_SUBNET]->(:EC2Subnet) if
-    (:EC2Instance)--(:NetworkInterface)--(:EC2Subnet).
-    """
-    ingest_network_interface_instance_relations = """
-    MATCH (i:EC2Instance)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet)
-    MERGE (i)-[r:PART_OF_SUBNET]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    """
-    logger.debug("-> Instance to subnet")
-    neo4j_session.run(
-        ingest_network_interface_instance_relations, update_tag=update_tag,
-    )
-
-
-@timeit
-def load_network_interface_load_balancer_relations(neo4j_session: neo4j.Session, update_tag: int) -> None:
-    """
-    Creates (:LoadBalancer)-[:PART_OF_SUBNET]->(:EC2Subnet) if
-    (:LoadBalancer)--(:NetworkInterface)--(:EC2Subnet).
-    """
-    ingest_network_interface_loadbalancer_relations = """
-    MATCH (i:LoadBalancer)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet)
-    MERGE (i)-[r:PART_OF_SUBNET]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
-    """
-    logger.debug("-> ELB to subnet")
-    neo4j_session.run(
-        ingest_network_interface_loadbalancer_relations, update_tag=update_tag,
-    )
-
-
-@timeit
-def load_network_interface_load_balancer_v2_relations(neo4j_session: neo4j.Session, update_tag: int) -> None:
-    """
-    Creates (:LoadBalancerV2)-[:PART_OF_SUBNET]->(:EC2Subnet) if
-    (:LoadBalancerV2)--(:NetworkInterface)--(:EC2Subnet).
+def load_subnet_network_interface(
+        neo4j_session: neo4j.Session,
+        data: List[Dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
     """
-    ingest_network_interface_loadbalancerv2_relations = """
-    MATCH (i:LoadBalancerV2)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet)
-    MERGE (i)-[r:PART_OF_SUBNET]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag
+    Subnets as known by describe-network-interfaces.
     """
-    logger.debug("-> ELBv2 to subnet")
-    neo4j_session.run(
-        ingest_network_interface_loadbalancerv2_relations, update_tag=update_tag,
+    logger.info(f"Loading {len(data)} subnets in {region}.")
+    load(
+        neo4j_session,
+        EC2SubnetNetworkInterfaceSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
     )
 
 
-@timeit
-def load(neo4j_session: neo4j.Session, data: List[Dict], region: str, aws_account_id: str, update_tag: int) -> None:
-    elb_associations = []
-    elb_associations_v2 = []
-    instance_associations = []
-
-    for network_interface in data:
-        # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/
-        matchObj = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', ''))
-        if matchObj:
-            elb_associations_v2.append({
-                'netinf_id': network_interface['NetworkInterfaceId'],
-                'elb_id': f'{matchObj[1]}-{matchObj[2]}.elb.{region}.amazonaws.com',
-            })
-        else:
-            matchObj = re.match(r'^ELB (.*)', network_interface.get('Description', ''))
-            if matchObj:
-                elb_associations.append({
-                    'netinf_id': network_interface['NetworkInterfaceId'],
-                    'elb_name': matchObj[1],
-                })
-
-        if 'Attachment' in network_interface and 'InstanceId' in network_interface['Attachment']:
-            instance_associations.append({
-                'netinf_id': network_interface['NetworkInterfaceId'],
-                'instance_id': network_interface['Attachment']['InstanceId'],
-            })
-    load_network_interfaces(neo4j_session, data, region, aws_account_id, update_tag)  # type: ignore
-    load_network_interface_instance_relations(
-        neo4j_session, instance_associations, region, aws_account_id, update_tag,
-    )
-    load_network_interface_elb_relations(neo4j_session, elb_associations, region, aws_account_id, update_tag)
-    load_network_interface_elbv2_relations(neo4j_session, elb_associations_v2, region, aws_account_id, update_tag)
-    load_network_interface_instance_to_subnet_relations(neo4j_session, update_tag)
-    load_network_interface_load_balancer_relations(neo4j_session, update_tag)
+def load_network_data(
+        neo4j_session: neo4j.Session,
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+        network_interface_list: List[Dict[str, Any]],
+        private_ip_list: List[Dict[str, Any]],
+        subnet_list: List[Dict[str, Any]],
+        sg_list: List[Dict[str, Any]],
+) -> None:
+    load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag)
+    load_private_ip_network_interface(neo4j_session, private_ip_list, region, current_aws_account_id, update_tag)
+    load_subnet_network_interface(neo4j_session, subnet_list, region, current_aws_account_id, update_tag)
+    load_security_group_network_interface(neo4j_session, sg_list, region, current_aws_account_id, update_tag)
 
 
 @timeit
 def cleanup_network_interfaces(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('aws_ingest_network_interfaces_cleanup.json', neo4j_session, common_job_parameters)
     GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2PrivateIpNetworkInterfaceSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
 def sync_network_interfaces(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: List[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info("Syncing EC2 network interfaces for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.info(f"Syncing EC2 network interfaces for region '{region}' in account '{current_aws_account_id}'.")
         data = get_network_interface_data(boto3_session, region)
-        load(neo4j_session, data, region, current_aws_account_id, update_tag)
+        ec2_network_data = transform_network_interface_data(data, region)
+        load_network_data(
+            neo4j_session,
+            region,
+            current_aws_account_id,
+            update_tag,
+            ec2_network_data.network_interface_list,
+            ec2_network_data.private_ip_list,
+            ec2_network_data.subnet_list,
+            ec2_network_data.sg_list,
+        )
     cleanup_network_interfaces(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/security_groups.py

@@ -8,7 +8,7 @@ import neo4j
 
 from .util import get_botocore_config
 from cartography.graph.job import GraphJob
-from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema
+from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -148,7 +148,7 @@ def cleanup_ec2_security_groupinfo(neo4j_session: neo4j.Session, common_job_para
         neo4j_session,
         common_job_parameters,
     )
-    GraphJob.from_node_schema(EC2SecurityGroupSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2SecurityGroupInstanceSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit

cartography/intel/aws/ec2/subnets.py

@@ -7,7 +7,7 @@ import neo4j
 
 from .util import get_botocore_config
 from cartography.graph.job import GraphJob
-from cartography.models.aws.ec2.subnets import EC2SubnetSchema
+from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -78,7 +78,7 @@ def load_subnets(
 @timeit
 def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters)
-    GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit