cartography 0.95.0rc1__py3-none-any.whl → 0.96.0__py3-none-any.whl

cartography/cli.py

@@ -9,6 +9,7 @@ import cartography.config
 import cartography.sync
 import cartography.util
 from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs
+from cartography.intel.semgrep.dependencies import parse_and_validate_semgrep_ecosystems
 
 
 logger = logging.getLogger(__name__)
@@ -524,6 +525,17 @@ class CLI:
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--semgrep-dependency-ecosystems',
+            type=str,
+            default=None,
+            help=(
+                'Comma-separated list of language ecosystems for which dependencies will be retrieved from Semgrep. '
+                'For example, a value of "gomod,npm" will retrieve Go and NPM dependencies. '
+                'See the full list of supported ecosystems in source code at cartography.intel.semgrep.dependencies. '
+                'Required if you are using the Semgrep dependencies intel module. Ignored otherwise.'
+            ),
+        )
         parser.add_argument(
             '--snipeit-base-uri',
             type=str,
@@ -734,6 +746,9 @@ class CLI:
             config.semgrep_app_token = os.environ.get(config.semgrep_app_token_env_var)
         else:
             config.semgrep_app_token = None
+        if config.semgrep_dependency_ecosystems:
+            # No need to store the returned value; we're using this for input validation.
+            parse_and_validate_semgrep_ecosystems(config.semgrep_dependency_ecosystems)
 
         # CVE feed config
         if config.cve_api_key_env_var:

cartography/client/core/tx.py

@@ -122,7 +122,7 @@ def read_list_of_tuples_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[
     return [tuple(val) for val in values]
 
 
-def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Dict[str, Any]:
+def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Any:
     """
     Runs the given Neo4j query in the given transaction object and returns the single dict result. This is intended to
     be run only with queries that return a single dict.

cartography/config.py

@@ -107,6 +107,8 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :param semgrep_dependency_ecosystems: Comma-separated list of Semgrep dependency ecosystems to fetch. Optional.
+    :type semgrep_dependency_ecosystems: str
     :type snipeit_base_uri: string
     :param snipeit_base_uri: SnipeIT data provider base URI. Optional.
     :type snipeit_token: string
@@ -155,7 +157,7 @@ class Config:
         pagerduty_request_timeout=None,
         nist_cve_url=None,
         cve_enabled=False,
-        cve_api_key=None,
+        cve_api_key: str | None = None,
         crowdstrike_client_id=None,
         crowdstrike_client_secret=None,
         crowdstrike_api_url=None,
@@ -170,6 +172,7 @@ class Config:
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        semgrep_dependency_ecosystems=None,
         snipeit_base_uri=None,
         snipeit_token=None,
         snipeit_tenant_id=None,
@@ -212,7 +215,7 @@ class Config:
         self.pagerduty_request_timeout = pagerduty_request_timeout
         self.nist_cve_url = nist_cve_url
         self.cve_enabled = cve_enabled
-        self.cve_api_key = cve_api_key
+        self.cve_api_key: str | None = cve_api_key
         self.crowdstrike_client_id = crowdstrike_client_id
         self.crowdstrike_client_secret = crowdstrike_client_secret
         self.crowdstrike_api_url = crowdstrike_api_url
@@ -227,6 +230,7 @@ class Config:
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.semgrep_dependency_ecosystems = semgrep_dependency_ecosystems
         self.snipeit_base_uri = snipeit_base_uri
         self.snipeit_token = snipeit_token
         self.snipeit_tenant_id = snipeit_tenant_id

cartography/data/indexes.cypher

@@ -305,8 +305,7 @@ CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.host_info_local_
 CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.lastupdated);
+CREATE INDEX IF NOT EXISTS FOR (n:UserAccount) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AzurePrincipal) ON (n.email);

cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json

@@ -0,0 +1,16 @@
+{
+  "statements": [
+
+    {
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSSSOUser)<-[r:CAN_ASSUME_IDENTITY]-(:OktaUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
+      "iterative": true,
+      "iterationsize": 100
+    },
+    {
+      "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSRole)-[r:ALLOWED_BY]->(:AWSSSOUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
+      "iterative": true,
+      "iterationsize": 100
+    }
+  ],
+  "name": "cleanup AWS Identity Center Instances and Related Data"
+}

cartography/data/jobs/cleanup/{github_users_cleanup.json → github_org_and_users_cleanup.json}

@@ -18,6 +18,11 @@
     "query": "MATCH (:GitHubUser)-[r:MEMBER_OF]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
     "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:UNAFFILIATED]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
   }],
   "name": "cleanup GitHub users data"
 }

cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -63,6 +63,31 @@
     "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
     "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_MAINTAIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_READ]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_TRIAGE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
   }],
   "name": "cleanup GitHub repos data"
 }

cartography/graph/querybuilder.py

@@ -118,6 +118,7 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     """
     match = Template("$node_var.$key = $prop_ref")
     case_insensitive_match = Template("toLower($node_var.$key) = toLower($prop_ref)")
+    fuzzy_and_ignorecase_match = Template("toLower($node_var.$key) CONTAINS toLower($prop_ref)")
 
     matcher_asdict = asdict(matcher)
 
@@ -125,7 +126,10 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
     for key, prop_ref in matcher_asdict.items():
         if prop_ref.ignore_case:
             prop_line = case_insensitive_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
+        elif prop_ref.fuzzy_and_ignore_case:
+            prop_line = fuzzy_and_ignorecase_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         else:
+            # Exact match (default; most efficient)
             prop_line = match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
         result.append(prop_line)
     return ' AND\n'.join(result)

cartography/intel/aws/apigateway.py

@@ -43,7 +43,7 @@ def get_rest_api_details(
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)
         # clientcertificate id is given by the api stage
-        certificate = get_rest_api_client_certificate(stages, client)  # type: ignore
+        certificate = get_rest_api_client_certificate(stages, client)
         resources = get_rest_api_resources(api, client)
         policy = get_rest_api_policy(api, client)
         apis.append((api['id'], stages, certificate, resources, policy))
@@ -51,7 +51,7 @@ def get_rest_api_details(
 
 
 @timeit
-def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
     """
@@ -99,7 +99,7 @@ def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> Lis
 
 
 @timeit
-def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API policy. Returns policy string or None if no policy is present.
     """

cartography/intel/aws/ec2/auto_scaling_groups.py

@@ -1,24 +1,37 @@
 import logging
-from typing import Dict
-from typing import List
+from collections import namedtuple
+from typing import Any
 
 import boto3
 import neo4j
 
 from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema
+from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
+from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
+from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+AsgData = namedtuple(
+    'AsgData', [
+        "group_list",
+        "instance_list",
+        "subnet_list",
+    ],
+)
+
 
 @timeit
 @aws_handle_regions
-def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> list[dict]:
     client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_auto_scaling_groups')
-    asgs: List[Dict] = []
+    asgs: list[dict] = []
     for page in paginator.paginate():
         asgs.extend(page['AutoScalingGroups'])
     return asgs
@@ -26,218 +39,167 @@ def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: st
 
 @timeit
 @aws_handle_regions
-def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> list[dict]:
     client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_launch_configurations')
-    lcs: List[Dict] = []
+    lcs: list[dict] = []
     for page in paginator.paginate():
         lcs.extend(page['LaunchConfigurations'])
     return lcs
 
 
+def transform_launch_configurations(configurations: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    transformed_configurations = []
+    for config in configurations:
+        transformed_configurations.append({
+            'AssociatePublicIpAddress': config.get('AssociatePublicIpAddress'),
+            'LaunchConfigurationARN': config.get('LaunchConfigurationARN'),
+            'LaunchConfigurationName': config.get('LaunchConfigurationName'),
+            'CreatedTime': config.get('CreatedTime'),
+            'ImageId': config.get('ImageId'),
+            'KeyName': config.get('KeyName'),
+            'SecurityGroups': config.get('SecurityGroups'),
+            'InstanceType': config.get('InstanceType'),
+            'KernelId': config.get('KernelId'),
+            'RamdiskId': config.get('RamdiskId'),
+            'InstanceMonitoring': config.get('InstanceMonitoring', {}).get('Enabled'),
+            'SpotPrice': config.get('SpotPrice'),
+            'IamInstanceProfile': config.get('IamInstanceProfile'),
+            'EbsOptimized': config.get('EbsOptimized'),
+            'PlacementTenancy': config.get('PlacementTenancy'),
+        })
+    return transformed_configurations
+
+
+def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
+    transformed_groups = []
+    related_vpcs = []
+    related_instances = []
+    for group in groups:
+        transformed_groups.append({
+            'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+            'CapacityRebalance': group.get('CapacityRebalance'),
+            'CreatedTime': str(group.get('CreatedTime')),
+            'DefaultCooldown': group.get('DefaultCooldown'),
+            'DesiredCapacity': group.get('DesiredCapacity'),
+            'HealthCheckGracePeriod': group.get('HealthCheckGracePeriod'),
+            'HealthCheckType': group.get('HealthCheckType'),
+            'LaunchConfigurationName': group.get('LaunchConfigurationName'),
+            'LaunchTemplateName': group.get('LaunchTemplate', {}).get('LaunchTemplateName'),
+            'LaunchTemplateId': group.get('LaunchTemplate', {}).get('LaunchTemplateId'),
+            'LaunchTemplateVersion': group.get('LaunchTemplate', {}).get('Version'),
+            'MaxInstanceLifetime': group.get('MaxInstanceLifetime'),
+            'MaxSize': group.get('MaxSize'),
+            'MinSize': group.get('MinSize'),
+            'AutoScalingGroupName': group.get('AutoScalingGroupName'),
+            'NewInstancesProtectedFromScaleIn': group.get('NewInstancesProtectedFromScaleIn'),
+            'Status': group.get('Status'),
+        })
+
+        if group.get('VPCZoneIdentifier', None):
+            vpclist = group['VPCZoneIdentifier']
+            subnet_ids = vpclist.split(',') if ',' in vpclist else [vpclist]
+            subnets = []
+            for subnet_id in subnet_ids:
+                subnets.append({
+                    'VPCZoneIdentifier': subnet_id,
+                    'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+                })
+            related_vpcs.extend(subnets)
+
+        for instance_data in group.get('Instances', []):
+            related_instances.append({
+                'InstanceId': instance_data['InstanceId'],
+                'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+            })
+
+    return AsgData(
+        group_list=transformed_groups,
+        instance_list=related_instances,
+        subnet_list=related_vpcs,
+    )
+
+
 @timeit
 def load_launch_configurations(
-        neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
 ) -> None:
-    ingest_lc = """
-    UNWIND $launch_configurations as lc
-        MERGE (config:LaunchConfiguration{id: lc.LaunchConfigurationARN})
-        ON CREATE SET config.firstseen = timestamp(), config.name = lc.LaunchConfigurationName,
-        config.arn = lc.LaunchConfigurationARN,
-        config.created_time = lc.CreatedTime
-        SET config.lastupdated = $update_tag, config.image_id = lc.ImageId,
-        config.key_name = lc.KeyName,
-        config.security_groups = lc.SecurityGroups,
-        config.instance_type = lc.InstanceType,
-        config.kernel_id = lc.KernelId,
-        config.ramdisk_id = lc.RamdiskId,
-        config.instance_monitoring_enabled = lc.InstanceMonitoring.Enabled,
-        config.spot_price = lc.SpotPrice,
-        config.iam_instance_profile = lc.IamInstanceProfile,
-        config.ebs_optimized = lc.EbsOptimized,
-        config.associate_public_ip_address = lc.AssociatePublicIpAddress,
-        config.placement_tenancy = lc.PlacementTenancy,
-        config.region=$Region
-        WITH config
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(config)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-    for lc in data:
-        lc['CreatedTime'] = str(int(lc['CreatedTime'].timestamp()))
-
-    neo4j_session.run(
-        ingest_lc,
-        launch_configurations=data,
-        AWS_ACCOUNT_ID=current_aws_account_id,
+    load(
+        neo4j_session,
+        LaunchConfigurationSchema(),
+        data,
         Region=region,
-        update_tag=update_tag,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
     )
 
 
-@timeit
-def load_ec2_auto_scaling_groups(
-        neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int,
+def load_groups(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
 ) -> None:
-    ingest_group = """
-    UNWIND $autoscaling_groups_list as ag
-        MERGE (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN})
-        ON CREATE SET group.firstseen = timestamp(),
-        group.createdtime = ag.CreatedTime
-        SET group.launchconfigurationname = ag.LaunchConfigurationName,
-        group.launchtemplatename = ag.LaunchTemplate.LaunchTemplateName,
-        group.launchtemplateid = ag.LaunchTemplate.LaunchTemplateId,
-        group.launchtemplateversion = ag.LaunchTemplate.Version,
-        group.maxsize = ag.MaxSize, group.minsize = ag.MinSize, group.defaultcooldown = ag.DefaultCooldown,
-        group.desiredcapacity = ag.DesiredCapacity, group.healthchecktype = ag.HealthCheckType,
-        group.healthcheckgraceperiod = ag.HealthCheckGracePeriod, group.status = ag.Status,
-        group.newinstancesprotectedfromscalein = ag.NewInstancesProtectedFromScaleIn,
-        group.maxinstancelifetime = ag.MaxInstanceLifetime, group.capacityrebalance = ag.CapacityRebalance,
-        group.name = ag.AutoScalingGroupName,
-        group.lastupdated = $update_tag,
-        group.region=$Region
-        WITH group
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(group)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    ingest_vpc = """
-    UNWIND $vpc_id_list as vpc_id
-        MERGE (subnet:EC2Subnet{subnetid: vpc_id})
-        ON CREATE SET subnet.firstseen = timestamp()
-        SET subnet.lastupdated = $update_tag
-        WITH subnet
-        MATCH (group:AutoScalingGroup{arn: $GROUPARN})
-        MERGE (subnet)<-[r:VPC_IDENTIFIER]-(group)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    ingest_instance = """
-    UNWIND $instances_list as i
-        MERGE (instance:Instance:EC2Instance{id: i.InstanceId})
-        ON CREATE SET instance.firstseen = timestamp()
-        SET instance.lastupdated = $update_tag, instance.region=$Region
-        WITH instance
-        MATCH (group:AutoScalingGroup{arn: $GROUPARN})
-        MERGE (instance)-[r:MEMBER_AUTO_SCALE_GROUP]->(group)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-        WITH instance
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(instance)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    ingest_lts = """
-    UNWIND $autoscaling_groups_list as ag
-        MATCH (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN})
-        MATCH (template:LaunchTemplate{id: ag.LaunchTemplate.LaunchTemplateId})
-        MERGE (group)-[r:HAS_LAUNCH_TEMPLATE]->(template)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    ingest_lcs = """
-    UNWIND $autoscaling_groups_list as ag
-        MATCH (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN})
-        MATCH (config:LaunchConfiguration{name: ag.LaunchConfigurationName})
-        MERGE (group)-[r:HAS_LAUNCH_CONFIG]->(config)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    launch_configs = []
-    launch_templates = []
-    for group in data:
-        if group.get('LaunchConfigurationName'):
-            launch_configs.append(group)
-        if group.get('LaunchTemplate'):
-            launch_templates.append(group)
-
-        group['CreatedTime'] = str(group['CreatedTime'])
-
-    neo4j_session.run(
-        ingest_group,
-        autoscaling_groups_list=data,
-        AWS_ACCOUNT_ID=current_aws_account_id,
+    load(
+        neo4j_session,
+        AutoScalingGroupSchema(),
+        data,
         Region=region,
-        update_tag=update_tag,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
     )
-    neo4j_session.run(
-        ingest_lcs,
-        autoscaling_groups_list=launch_configs,
-        AWS_ACCOUNT_ID=current_aws_account_id,
+
+
+def load_asg_subnets(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EC2SubnetAutoScalingGroupSchema(),
+        data,
         Region=region,
-        update_tag=update_tag,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
     )
-    neo4j_session.run(
-        ingest_lts,
-        autoscaling_groups_list=launch_templates,
-        AWS_ACCOUNT_ID=current_aws_account_id,
+
+
+def load_asg_instances(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EC2InstanceAutoScalingGroupSchema(),
+        data,
         Region=region,
-        update_tag=update_tag,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
     )
 
-    for group in data:
-        group_arn = group["AutoScalingGroupARN"]
-        if group.get('VPCZoneIdentifier'):
-            vpclist = group["VPCZoneIdentifier"]
-            if ',' in vpclist:
-                data = vpclist.split(',')
-            else:
-                data = vpclist
-            neo4j_session.run(
-                ingest_vpc,
-                vpc_id_list=data,
-                GROUPARN=group_arn,
-                update_tag=update_tag,
-            )
-
-        if group.get("Instances"):
-            data = group["Instances"]
-            neo4j_session.run(
-                ingest_instance,
-                instances_list=data,
-                GROUPARN=group_arn,
-                AWS_ACCOUNT_ID=current_aws_account_id,
-                Region=region,
-                update_tag=update_tag,
-            )
-
 
 @timeit
-def cleanup_ec2_auto_scaling_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        'aws_ingest_ec2_auto_scaling_groups_cleanup.json',
-        neo4j_session,
-        common_job_parameters,
-    )
+def load_auto_scaling_groups(
+    neo4j_session: neo4j.Session, data: AsgData, region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load_groups(neo4j_session, data.group_list, region, current_aws_account_id, update_tag)
+    load_asg_instances(neo4j_session, data.instance_list, region, current_aws_account_id, update_tag)
+    load_asg_subnets(neo4j_session, data.subnet_list, region, current_aws_account_id, update_tag)
 
 
 @timeit
-def cleanup_ec2_launch_configurations(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        'aws_import_ec2_launch_configurations_cleanup.json',
-        neo4j_session,
-        common_job_parameters,
-    )
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running EC2 instance cleanup")
+    GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
 def sync_ec2_auto_scaling_groups(
-        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
-        current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
+        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str],
+        current_aws_account_id: str, update_tag: int, common_job_parameters: dict,
 ) -> None:
     for region in regions:
         logger.debug("Syncing auto scaling groups for region '%s' in account '%s'.", region, current_aws_account_id)
         lc_data = get_launch_configurations(boto3_session, region)
-        load_launch_configurations(neo4j_session, lc_data, region, current_aws_account_id, update_tag)
-        data = get_ec2_auto_scaling_groups(boto3_session, region)
-        load_ec2_auto_scaling_groups(neo4j_session, data, region, current_aws_account_id, update_tag)
-    cleanup_ec2_auto_scaling_groups(neo4j_session, common_job_parameters)
-    cleanup_ec2_launch_configurations(neo4j_session, common_job_parameters)
+        asg_data = get_ec2_auto_scaling_groups(boto3_session, region)
+        lc_transformed = transform_launch_configurations(lc_data)
+        asg_transformed = transform_auto_scaling_groups(asg_data)
+        load_launch_configurations(neo4j_session, lc_transformed, region, current_aws_account_id, update_tag)
+        load_auto_scaling_groups(neo4j_session, asg_transformed, region, current_aws_account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/instances.py

@@ -11,6 +11,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
 from cartography.models.aws.ec2.instances import EC2InstanceSchema
 from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
 from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema
@@ -308,6 +309,7 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
     logger.debug("Running EC2 instance cleanup")
     GraphJob.from_node_schema(EC2ReservationSchema(), common_job_parameters).run(neo4j_session)
     GraphJob.from_node_schema(EC2InstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2InstanceAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit