cartography 0.92.0rc1__tar.gz → 0.92.0rc2__tar.gz

{cartography-0.92.0rc1/cartography.egg-info → cartography-0.92.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.92.0rc1
+Version: 0.92.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.92.0rc1 → cartography-0.92.0rc2}/cartography/data/indexes.cypher

@@ -200,12 +200,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.dnsname);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.lastupdated);

cartography-0.92.0rc2/cartography/intel/aws/ec2/launch_templates.py

@@ -0,0 +1,134 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema
+from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_launch_templates(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_launch_templates')
+    templates: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        templates.extend(page['LaunchTemplates'])
+    return templates
+
+
+def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for template in templates:
+        current = template.copy()
+        current['CreateTime'] = str(int(current['CreateTime'].timestamp()))
+        result.append(current)
+    return result
+
+
+@timeit
+def load_launch_templates(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LaunchTemplateSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+@aws_handle_regions
+def get_launch_template_versions(
+        boto3_session: boto3.session.Session,
+        templates: list[dict[str, Any]],
+        region: str,
+) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    v_paginator = client.get_paginator('describe_launch_template_versions')
+    template_versions = []
+    for template in templates:
+        for versions in v_paginator.paginate(LaunchTemplateId=template['LaunchTemplateId']):
+            template_versions.extend(versions['LaunchTemplateVersions'])
+    return template_versions
+
+
+def transform_launch_template_versions(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for version in versions:
+        current = version.copy()
+
+        # Reformat some fields
+        current['Id'] = f"{version['LaunchTemplateId']}-{version['VersionNumber']}"
+        current['CreateTime'] = str(int(version['CreateTime'].timestamp()))
+
+        # Handle the nested object returned from boto
+        ltd = version['LaunchTemplateData']
+        current['KernelId'] = ltd.get('KernelId')
+        current['EbsOptimized'] = ltd.get('EbsOptimized')
+        current['IamInstanceProfileArn'] = ltd.get('IamInstanceProfileArn')
+        current['IamInstanceProfileName'] = ltd.get('IamInstanceProfileName')
+        current['ImageId'] = ltd.get('ImageId')
+        current['InstanceType'] = ltd.get('InstanceType')
+        current['KeyName'] = ltd.get('KeyName')
+        current['MonitoringEnabled'] = ltd.get('MonitoringEnabled')
+        current['RamdiskId'] = ltd.get('RamdiskId')
+        current['DisableApiTermination'] = ltd.get('DisableApiTermination')
+        current['InstanceInitiatedShutDownBehavior'] = ltd.get('InstanceInitiatedShutDownBehavior')
+        current['SecurityGroupIds'] = ltd.get('SecurityGroupIds')
+        current['SecurityGroups'] = ltd.get('SecurityGroups')
+        result.append(current)
+    return result
+
+
+@timeit
+def load_launch_template_versions(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LaunchTemplateVersionSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def sync_ec2_launch_templates(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
+        templates = get_launch_templates(boto3_session, region)
+        templates = transform_launch_templates(templates)
+        load_launch_templates(neo4j_session, templates, region, current_aws_account_id, update_tag)
+
+        versions = get_launch_template_versions(boto3_session, templates, region)
+        versions = transform_launch_template_versions(versions)
+        load_launch_template_versions(neo4j_session, versions, region, current_aws_account_id, update_tag)

cartography-0.92.0rc2/cartography/models/aws/ec2/launch_template_versions.py

@@ -0,0 +1,81 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('Id')
+    name: PropertyRef = PropertyRef('LaunchTemplateName')
+    create_time: PropertyRef = PropertyRef('CreateTime')
+    created_by: PropertyRef = PropertyRef('CreatedBy')
+    default_version: PropertyRef = PropertyRef('DefaultVersion')
+    version_number: PropertyRef = PropertyRef('VersionNumber')
+    version_description: PropertyRef = PropertyRef('VersionDescription')
+    kernel_id: PropertyRef = PropertyRef('KernelId')
+    ebs_optimized: PropertyRef = PropertyRef('EbsOptimized')
+    iam_instance_profile_arn: PropertyRef = PropertyRef('IamInstanceProfileArn')
+    iam_instance_profile_name: PropertyRef = PropertyRef('IamInstanceProfileName')
+    image_id: PropertyRef = PropertyRef('ImageId')
+    instance_type: PropertyRef = PropertyRef('InstanceType')
+    key_name: PropertyRef = PropertyRef('KeyName')
+    monitoring_enabled: PropertyRef = PropertyRef('MonitoringEnabled')
+    ramdisk_id: PropertyRef = PropertyRef('RamdiskId')
+    disable_api_termination: PropertyRef = PropertyRef('DisableApiTermination')
+    instance_initiated_shutdown_behavior: PropertyRef = PropertyRef('InstanceInitiatedShutdownBehavior')
+    security_group_ids: PropertyRef = PropertyRef('SecurityGroupIds')
+    security_groups: PropertyRef = PropertyRef('SecurityGroups')
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: LaunchTemplateVersionToAwsAccountRelProperties = LaunchTemplateVersionToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionToLTRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionToLT(CartographyRelSchema):
+    target_node_label: str = 'LaunchTemplate'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('LaunchTemplateId')},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "VERSION"
+    properties: LaunchTemplateVersionToLTRelProperties = LaunchTemplateVersionToLTRelProperties()
+
+
+@dataclass(frozen=True)
+class LaunchTemplateVersionSchema(CartographyNodeSchema):
+    label: str = 'LaunchTemplateVersion'
+    properties: LaunchTemplateVersionNodeProperties = LaunchTemplateVersionNodeProperties()
+    sub_resource_relationship: LaunchTemplateVersionToAWSAccount = LaunchTemplateVersionToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            LaunchTemplateVersionToLT(),
+        ],
+    )

cartography-0.92.0rc2/cartography/models/aws/ec2/launch_templates.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class LaunchTemplateNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('LaunchTemplateId')
+    launch_template_id: PropertyRef = PropertyRef('LaunchTemplateId')
+    name: PropertyRef = PropertyRef('LaunchTemplateName')
+    create_time: PropertyRef = PropertyRef('CreateTime')
+    created_by: PropertyRef = PropertyRef('CreatedBy')
+    default_version_number: PropertyRef = PropertyRef('DefaultVersionNumber')
+    latest_version_number: PropertyRef = PropertyRef('LatestVersionNumber')
+    region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class LaunchTemplateToAwsAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class LaunchTemplateToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: LaunchTemplateToAwsAccountRelProperties = LaunchTemplateToAwsAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class LaunchTemplateSchema(CartographyNodeSchema):
+    label: str = 'LaunchTemplate'
+    properties: LaunchTemplateNodeProperties = LaunchTemplateNodeProperties()
+    sub_resource_relationship: LaunchTemplateToAWSAccount = LaunchTemplateToAWSAccount()

{cartography-0.92.0rc1 → cartography-0.92.0rc2/cartography.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.92.0rc1
+Version: 0.92.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.92.0rc1 → cartography-0.92.0rc2}/cartography.egg-info/SOURCES.txt

@@ -47,7 +47,6 @@ cartography/data/jobs/cleanup/aws_import_account_access_key_cleanup.json
 cartography/data/jobs/cleanup/aws_import_apigateway_cleanup.json
 cartography/data/jobs/cleanup/aws_import_config_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ec2_launch_configurations_cleanup.json
-cartography/data/jobs/cleanup/aws_import_ec2_launch_templates_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ec2_security_groupinfo_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ecr_cleanup.json
 cartography/data/jobs/cleanup/aws_import_ecs_cleanup.json
@@ -294,6 +293,8 @@ cartography/models/aws/ec2/__init__.py
 cartography/models/aws/ec2/images.py
 cartography/models/aws/ec2/instances.py
 cartography/models/aws/ec2/keypairs.py
+cartography/models/aws/ec2/launch_template_versions.py
+cartography/models/aws/ec2/launch_templates.py
 cartography/models/aws/ec2/loadbalancerv2.py
 cartography/models/aws/ec2/networkinterface_instance.py
 cartography/models/aws/ec2/networkinterfaces.py

{cartography-0.92.0rc1 → cartography-0.92.0rc2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.92.0rc1'
+__version__ = '0.92.0rc2'
 
 
 setup(

cartography-0.92.0rc1/cartography/data/jobs/cleanup/aws_import_ec2_launch_templates_cleanup.json

@@ -1,13 +0,0 @@
-{
-  "statements": [{
-    "query": "MATCH (n:LaunchTemplateVersion)<-[:VERSION]-(:LaunchTemplate)<-[:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (n:LaunchTemplate)<-[:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-    "iterative": true,
-    "iterationsize": 100
-  }],
-  "name": "cleanup LaunchTemplate"
-}

cartography-0.92.0rc1/cartography/intel/aws/ec2/launch_templates.py

@@ -1,115 +0,0 @@
-import logging
-from typing import Dict
-from typing import List
-
-import boto3
-import neo4j
-
-from .util import get_botocore_config
-from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
-from cartography.util import timeit
-
-logger = logging.getLogger(__name__)
-
-
-@timeit
-@aws_handle_regions
-def get_launch_templates(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
-    paginator = client.get_paginator('describe_launch_templates')
-    templates: List[Dict] = []
-    for page in paginator.paginate():
-        templates.extend(page['LaunchTemplates'])
-    for template in templates:
-        template_versions: List[Dict] = []
-        v_paginator = client.get_paginator('describe_launch_template_versions')
-        for versions in v_paginator.paginate(LaunchTemplateId=template['LaunchTemplateId']):
-            template_versions.extend(versions["LaunchTemplateVersions"])
-        template["_template_versions"] = template_versions
-    return templates
-
-
-@timeit
-def load_launch_templates(
-        neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int,
-) -> None:
-    ingest_lt = """
-    UNWIND $launch_templates as lt
-        MERGE (template:LaunchTemplate{id: lt.LaunchTemplateId})
-        ON CREATE SET template.firstseen = timestamp(),
-        template.name = lt.LaunchTemplateName,
-        template.create_time = lt.CreateTime,
-        template.created_by = lt.CreatedBy
-        SET template.default_version_number = lt.DefaultVersionNumber,
-        template.latest_version_number = lt.LatestVersionNumber,
-        template.lastupdated = $update_tag,
-        template.region=$Region
-        WITH template, lt._template_versions as versions
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(template)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-        WITH template, versions
-        UNWIND versions as tv
-            MERGE (version:LaunchTemplateVersion{id: tv.LaunchTemplateId + '-' + tv.VersionNumber})
-            ON CREATE SET version.firstseen = timestamp(),
-            version.name = tv.LaunchTemplateName,
-            version.create_time = tv.CreateTime,
-            version.created_by = tv.CreatedBy,
-            version.default_version = tv.DefaultVersion,
-            version.version_number = tv.VersionNumber,
-            version.version_description = tv.VersionDescription,
-            version.kernel_id = tv.LaunchTemplateData.KernelId,
-            version.ebs_optimized = tv.LaunchTemplateData.EbsOptimized,
-            version.iam_instance_profile_arn = tv.LaunchTemplateData.IamInstanceProfile.Arn,
-            version.iam_instance_profile_name = tv.LaunchTemplateData.IamInstanceProfile.Name,
-            version.image_id = tv.LaunchTemplateData.ImageId,
-            version.instance_type = tv.LaunchTemplateData.InstanceType,
-            version.key_name = tv.LaunchTemplateData.KeyName,
-            version.monitoring_enabled = tv.LaunchTemplateData.Monitoring.Enabled,
-            version.ramdisk_id = tv.LaunchTemplateData.RamdiskId,
-            version.disable_api_termination = tv.LaunchTemplateData.DisableApiTermination,
-            version.instance_initiated_shutdown_behavior = tv.LaunchTemplateData.InstanceInitiatedShutdownBehavior,
-            version.security_group_ids = tv.LaunchTemplateData.SecurityGroupIds,
-            version.security_groups = tv.LaunchTemplateData.SecurityGroups
-            SET version.lastupdated = $update_tag,
-            version.region=$Region
-            WITH template, version
-            MERGE (template)-[r:VERSION]->(version)
-            ON CREATE SET r.firstseen = timestamp()
-            SET r.lastupdated = $update_tag
-    """
-    for lt in data:
-        lt['CreateTime'] = str(int(lt['CreateTime'].timestamp()))
-        for tv in lt["_template_versions"]:
-            tv['CreateTime'] = str(int(tv['CreateTime'].timestamp()))
-
-    neo4j_session.run(
-        ingest_lt,
-        launch_templates=data,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        Region=region,
-        update_tag=update_tag,
-    )
-
-
-@timeit
-def cleanup_ec2_launch_templates(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        'aws_import_ec2_launch_templates_cleanup.json',
-        neo4j_session,
-        common_job_parameters,
-    )
-
-
-@timeit
-def sync_ec2_launch_templates(
-        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
-        current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
-) -> None:
-    for region in regions:
-        logger.debug("Syncing launch templates for region '%s' in account '%s'.", region, current_aws_account_id)
-        data = get_launch_templates(boto3_session, region)
-        load_launch_templates(neo4j_session, data, region, current_aws_account_id, update_tag)
-    cleanup_ec2_launch_templates(neo4j_session, common_job_parameters)