cartography 0.92.0rc2__py3-none-any.whl → 0.93.0__py3-none-any.whl

cartography/intel/aws/ec2/images.py

@@ -51,6 +51,7 @@ def get_images(boto3_session: boto3.session.Session, region: str, image_ids: Lis
         logger.warning(f"Failed retrieve images for region - {region}. Error - {e}")
     try:
         if image_ids:
+            image_ids = [image_id for image_id in image_ids if image_id is not None]
             images_in_use = client.describe_images(ImageIds=image_ids)['Images']
             # Ensure we're not adding duplicates
             _ids = [image["ImageId"] for image in images]

cartography/intel/aws/ec2/launch_templates.py

@@ -3,9 +3,11 @@ from typing import Any
 
 import boto3
 import neo4j
+from botocore.exceptions import ClientError
 
 from .util import get_botocore_config
 from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema
 from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema
 from cartography.util import aws_handle_regions
@@ -16,13 +18,30 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_launch_templates(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+def get_launch_templates(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_launch_templates')
     templates: list[dict[str, Any]] = []
+    template_versions: list[dict[str, Any]] = []
     for page in paginator.paginate():
-        templates.extend(page['LaunchTemplates'])
-    return templates
+        paginated_templates = page['LaunchTemplates']
+        for template in paginated_templates:
+            template_id = template['LaunchTemplateId']
+            try:
+                versions = get_launch_template_versions_by_template(boto3_session, template_id, region)
+            except ClientError as e:
+                logger.warning(
+                    f"Failed to get launch template versions for {template_id}: {e}",
+                    exc_info=True,
+                )
+                versions = []
+            # Using a key not defined in latest boto3 documentation
+            template_versions.extend(versions)
+        templates.extend(paginated_templates)
+    return templates, template_versions
 
 
 def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -54,17 +73,16 @@ def load_launch_templates(
 
 @timeit
 @aws_handle_regions
-def get_launch_template_versions(
+def get_launch_template_versions_by_template(
         boto3_session: boto3.session.Session,
-        templates: list[dict[str, Any]],
+        template: str,
         region: str,
 ) -> list[dict[str, Any]]:
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     v_paginator = client.get_paginator('describe_launch_template_versions')
     template_versions = []
-    for template in templates:
-        for versions in v_paginator.paginate(LaunchTemplateId=template['LaunchTemplateId']):
-            template_versions.extend(versions['LaunchTemplateVersions'])
+    for versions in v_paginator.paginate(LaunchTemplateId=template):
+        template_versions.extend(versions['LaunchTemplateVersions'])
     return template_versions
 
 
@@ -114,6 +132,16 @@ def load_launch_template_versions(
     )
 
 
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.info("Running launch template cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(LaunchTemplateSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(LaunchTemplateVersionSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+
 @timeit
 def sync_ec2_launch_templates(
         neo4j_session: neo4j.Session,
@@ -125,10 +153,10 @@ def sync_ec2_launch_templates(
 ) -> None:
     for region in regions:
         logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
-        templates = get_launch_templates(boto3_session, region)
+        templates, versions = get_launch_templates(boto3_session, region)
         templates = transform_launch_templates(templates)
         load_launch_templates(neo4j_session, templates, region, current_aws_account_id, update_tag)
-
-        versions = get_launch_template_versions(boto3_session, templates, region)
         versions = transform_launch_template_versions(versions)
         load_launch_template_versions(neo4j_session, versions, region, current_aws_account_id, update_tag)
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/iam.py

@@ -227,6 +227,7 @@ def get_account_access_key_data(boto3_session: boto3.session.Session, username:
         logger.warning(
             f"Could not get access key for user {username} due to NoSuchEntityException; skipping.",
         )
+        return access_keys
     for access_key in access_keys['AccessKeyMetadata']:
         access_key_id = access_key['AccessKeyId']
         last_used_info = client.get_access_key_last_used(

cartography/intel/azure/compute.py

@@ -43,7 +43,7 @@ def load_vms(neo4j_session: neo4j.Session, subscription_id: str, vm_list: List[D
     v.resourcegroup = vm.resource_group
     SET v.lastupdated = $update_tag, v.name = vm.name,
     v.plan = vm.plan.product, v.size = vm.hardware_profile.vm_size,
-    v.license_type=vm.license_type, v.computer_name=vm.os_profile.computer_ame,
+    v.license_type=vm.license_type, v.computer_name=vm.os_profile.computer_name,
     v.identity_type=vm.identity.type, v.zones=vm.zones,
     v.ultra_ssd_enabled=vm.additional_capabilities.ultra_ssd_enabled,
     v.priority=vm.priority, v.eviction_policy=vm.eviction_policy

cartography/intel/crowdstrike/endpoints.py

@@ -76,9 +76,9 @@ def load_host_data(
     )
 
 
-def get_host_ids(client: Hosts) -> List[List[str]]:
+def get_host_ids(client: Hosts, crowdstrikeapi_filter: str = '', crowdstrikeapi_limit: int = 5000) -> List[List[str]]:
     ids = []
-    parameters = {"filter": 'service_provider:"AWS_EC2"', "limit": 400}
+    parameters = {"filter": crowdstrikeapi_filter, "limit": crowdstrikeapi_limit}
     response = client.QueryDevicesByFilter(parameters=parameters)
     body = response.get("body", {})
     resources = body.get("resources", [])

cartography/intel/cve/feed.py

@@ -23,7 +23,8 @@ from cartography.util import timeit
 logger = logging.getLogger(__name__)
 
 MAX_RETRIES = 3
-REQUEST_TIMEOUT = 10
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+CONNECT_AND_READ_TIMEOUT = (60, 60)
 CVE_FEED_ID = "NIST_NVD"
 BATCH_SIZE_DAYS = 120
 RESULTS_PER_PAGE = 2000
@@ -87,7 +88,7 @@ def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any,
     while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
         try:
             res = requests.get(
-                url, params=params, headers=headers, timeout=REQUEST_TIMEOUT,
+                url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
             )
             res.raise_for_status()
         except requests.exceptions.HTTPError:

cartography/models/aws/ec2/subnet_instance.py

@@ -15,7 +15,7 @@ from cartography.models.core.relationships import TargetNodeMatcher
 class EC2SubnetInstanceNodeProperties(CartographyNodeProperties):
     # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024
     id: PropertyRef = PropertyRef('SubnetId')
-    subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True)
+    subnetid: PropertyRef = PropertyRef('SubnetId', extra_index=True)
     region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
 

cartography/util.py

@@ -225,7 +225,7 @@ If not, then the AWS datatype somehow does not have this key.''',
     return items
 
 
-AWSGetFunc = TypeVar('AWSGetFunc', bound=Callable[..., List])
+AWSGetFunc = TypeVar('AWSGetFunc', bound=Callable[..., Iterable])
 
 # fix for AWS TooManyRequestsException
 # https://github.com/lyft/cartography/issues/297

{cartography-0.92.0rc2.dist-info → cartography-0.93.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.92.0rc2
+Version: 0.93.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.92.0rc2.dist-info → cartography-0.93.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ cartography/config.py,sha256=rL1zgxZO47_R7S6E9e0CwxmhzRSN0X_q93NtcPR1G00,11368
 cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/stats.py,sha256=dbybb9V2FuvSuHjjNwz6Vjwnd1hap2C7h960rLoKcl8,4406
 cartography/sync.py,sha256=a80r_IzrZcWGSmRDRrxkesNYPiOuLte5YHvDQT3L-Lw,9730
-cartography/util.py,sha256=F3FPMJl1KDW0x_5cvt2ZGI0Dv1LVrHU7Az4OleAANBI,14474
+cartography/util.py,sha256=umfnjX8jVLu0rpYA75X-WvRpYzHQxns9qZiPwfyAlwQ,14478
 cartography/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/client/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/client/aws/iam.py,sha256=dYsGikc36DEsSeR2XVOVFFUDwuU9yWj_EVkpgVYCFgM,1293
@@ -152,7 +152,7 @@ cartography/intel/aws/eks.py,sha256=OerAX7qT2uGPbqliPvuy8JZUIgle_KMlnkkHxk8O5fk,
 cartography/intel/aws/elasticache.py,sha256=fCI47aDFmIDyE26GiReKYb6XIZUwrzcvsXBQ4ruFhuI,4427
 cartography/intel/aws/elasticsearch.py,sha256=ZL7MkXF_bXRSoXuDSI1dwGckRLG2zDB8LuAD07vSLnE,8374
 cartography/intel/aws/emr.py,sha256=xhWBVZngxJRFjMEDxwq3G6SgytRGLq0v2a_CeDvByR0,3372
-cartography/intel/aws/iam.py,sha256=fEn0XCQhBKxoEQJT5OCEzTbP58VXtfzF11GUK-Fnqog,32347
+cartography/intel/aws/iam.py,sha256=eLw0NkBGKzCI_tQ3wmrx3aUibQerrsxKJd3d0RCKcKQ,32374
 cartography/intel/aws/inspector.py,sha256=6enCu2USefuGT3FqA0Vto6i-z4BrL2HC_clbiXSLIlo,8654
 cartography/intel/aws/kms.py,sha256=bZUzMxAH_DsAcGTJBs08gg2tLKYu-QWjvMvV9C-6v50,11731
 cartography/intel/aws/lambda_function.py,sha256=KKTyn53xpaMI9WvIqxmsOASFwflHt-2_5ow-zUFc2wg,9890
@@ -171,11 +171,11 @@ cartography/intel/aws/ssm.py,sha256=IDOYa8v2FgziU8nBOZ7wyDG4o_nFYshbB-si9Ut_9Zc,
 cartography/intel/aws/ec2/__init__.py,sha256=IDK2Yap7mllK_ab6yVMLXatJ94znIkn-szv5RJP5fbo,346
 cartography/intel/aws/ec2/auto_scaling_groups.py,sha256=4erjP31KSVW-Pp2ASmDox_VLp_AQUAin4KYxfZKZcSM,9223
 cartography/intel/aws/ec2/elastic_ip_addresses.py,sha256=0k4NwS73VyWbEj5jXvSkaq2RNvmAlBlrN-UKa_Bj0uk,3957
-cartography/intel/aws/ec2/images.py,sha256=DcGHJwZf8_K5iRCDJ2QOdP837TTJTwST02IRswfk9Mc,3697
+cartography/intel/aws/ec2/images.py,sha256=heElwHJGqVD3iUJjxwA_Sdc3CmE4HPs00CTMHuQ1wkc,3782
 cartography/intel/aws/ec2/instances.py,sha256=mnTjdBY-4D-TGhH29UrSaLUW0Uft0JApDIJkkLz4zPc,12170
 cartography/intel/aws/ec2/internet_gateways.py,sha256=dI-4-85_3DGGZZBcY_DN6XqESx9P26S6jKok314lcnQ,2883
 cartography/intel/aws/ec2/key_pairs.py,sha256=SvRgd56vE4eouvTSNoFK8PP8HYoECO91goxc36oq_FY,2508
-cartography/intel/aws/ec2/launch_templates.py,sha256=GSOtwvqL6pG2fcROhhjXjOPntW-nR8msT-dQG_JuQkM,4862
+cartography/intel/aws/ec2/launch_templates.py,sha256=aeqaL8On38ET8nM8bISsIXLy6PkZoV-tqSWG38YXgkI,6010
 cartography/intel/aws/ec2/load_balancer_v2s.py,sha256=95FfQQn740gexINIHDJizOM4OKzRtQT_y2XQMipQ5Dg,8661
 cartography/intel/aws/ec2/load_balancers.py,sha256=1GwErzGqi3BKCARqfGJcD_r_D84rFKVy5kNMas9jAok,6756
 cartography/intel/aws/ec2/network_interfaces.py,sha256=CzF8PooCYUQ2pk8DR8JDAhkWRUQSBj_27OsIfkL_-Cs,9199
@@ -192,7 +192,7 @@ cartography/intel/aws/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 cartography/intel/aws/util/arns.py,sha256=TEjrQEDbJM8Zwy6Eizh-vAFd9kMr5AKsdOsHYQHUSbk,573
 cartography/intel/aws/util/common.py,sha256=j_K9_Iy42dgCSvbGLdtJW8bRs_qTiJu4wkz54OId7w8,886
 cartography/intel/azure/__init__.py,sha256=RxbDGfsfDBM1eGtS-FPULB5mBBl3yYl7fIZ5P3RD9RE,3696
-cartography/intel/azure/compute.py,sha256=WbdsYI-mKgNENrWRtV_lwEKPLQsbw-Pk1yDLTHyNLNk,8678
+cartography/intel/azure/compute.py,sha256=SIjvhmGp2tSpBpzNubIM1pgr_MrVHHuJsJxGpnFWDRw,8679
 cartography/intel/azure/cosmosdb.py,sha256=8ZtQntjkMUVsQUcjGgJw6geDjYqothPkssyNQALhJ0o,41275
 cartography/intel/azure/sql.py,sha256=Z-C-7BRjD5y-enNKg0sfWgv1-GIENCf2y0Z3bZiasdc,32865
 cartography/intel/azure/storage.py,sha256=56QyHafWTNproCOjITyAaWsmsqKhyFwJQlVEfwawBNg,27393
@@ -203,13 +203,13 @@ cartography/intel/azure/util/credentials.py,sha256=99PjTs0vZ2iu0tHD7TohN1VJYjuXY
 cartography/intel/bigfix/__init__.py,sha256=3LoDCm01VNNaDRGsRiykJm1GJgUQ8zI1HO6NodLFVIA,1058
 cartography/intel/bigfix/computers.py,sha256=HAwA-muDBLu2xkFFL2Ae-xjCH1gxKKwxGMZM4BE_Qdo,5909
 cartography/intel/crowdstrike/__init__.py,sha256=dAtgI-0vZAQZ3cTFQhMEzzt7aqiNSNuiIYiw0qbut-I,1896
-cartography/intel/crowdstrike/endpoints.py,sha256=o45309ZDNdlYdG0YicIiBain-nkr3bQahBaRx3N9Mow,3738
+cartography/intel/crowdstrike/endpoints.py,sha256=tdqokMDW3p4fK3dHKKb2T1DTogvOJBCpwyrxdQlbUhw,3815
 cartography/intel/crowdstrike/spotlight.py,sha256=yNhj44-RYF6ubck-hHMKhKiNU0fCfhQf4Oagopc31EM,4754
 cartography/intel/crowdstrike/util.py,sha256=gfJ6Ptr6YdbBS9Qj9a_-Jc-IJroADDRcXqjh5TW0qXE,277
 cartography/intel/crxcavator/__init__.py,sha256=VM6N_7dMagzuQQjUeFgqrt2_d2Is9ugDMTrgKke2c0g,1606
 cartography/intel/crxcavator/crxcavator.py,sha256=tnx6bq8Oz020mhMDmx8gKZ_ro_0UvUGeWrshmFr7bBw,13797
 cartography/intel/cve/__init__.py,sha256=A7XjKQSanmwMSIXSum1qJSegtYcQCuz_713RU-bFQz8,2504
-cartography/intel/cve/feed.py,sha256=i3EuRFhotu93MNMkhwBQp-zvK6itLgdAdRFOlZy5iLc,9878
+cartography/intel/cve/feed.py,sha256=JkfRV18JoydOuncKR1y3s8esuN2Xk4gIB6viKNXU_X0,10020
 cartography/intel/digitalocean/__init__.py,sha256=SMYB7LGIQOj_EgGSGVjWZk7SJNbP43hQuOfgOu6xYm4,1526
 cartography/intel/digitalocean/compute.py,sha256=9XctwMjq9h5dExFgExvawoqyiEwSoocNgaMm3Fgl5GM,4911
 cartography/intel/digitalocean/management.py,sha256=YWRnBLLL_bAP1vefIAQgm_-QzefGH0sZKmyU_EokHfA,3764
@@ -291,7 +291,7 @@ cartography/models/aws/ec2/privateip_networkinterface.py,sha256=j8MyiZsiUCuzuGUH
 cartography/models/aws/ec2/reservations.py,sha256=dE9uSB3Em-ca1dDbetbu79JXr4ZWHC3r5gA1S3mjhVU,1930
 cartography/models/aws/ec2/securitygroup_instance.py,sha256=RZS9TzHHatTOESQgcs5_YHmF9sM7pRkUq2VPjk9FJlU,2876
 cartography/models/aws/ec2/securitygroup_networkinterface.py,sha256=PiaA8J82kybZyZ1wDsa-ACIDa88vt4NoA3smGNiwl14,2399
-cartography/models/aws/ec2/subnet_instance.py,sha256=kPELT07l6AXIy-NYnXfkhM_RWUG20D33K1V1CJQdHyw,2753
+cartography/models/aws/ec2/subnet_instance.py,sha256=ct_ibXiPN2C5ld06TczwSTXtsnlov5VCW6elphtbvPs,2752
 cartography/models/aws/ec2/subnet_networkinterface.py,sha256=JHlxfBojBw7LfJS4a5LpVGM28MUu451PUrrwbbOPGuQ,3614
 cartography/models/aws/ec2/volumes.py,sha256=WSP7YNZeJE3s4wnY9QrIAbcJN3OathqNgEBX0cVahDg,4470
 cartography/models/aws/eks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -332,10 +332,10 @@ cartography/models/semgrep/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 cartography/models/semgrep/deployment.py,sha256=or5qZDuR51MXzINpH15jZrqmSUvXQevCNYWJ7D6v-JI,745
 cartography/models/semgrep/findings.py,sha256=xrn8sgXpNMrNJbKQagaAVxaCG9bVjTATSRR2XRBR4rg,5386
 cartography/models/semgrep/locations.py,sha256=kSk7Nn5Mn4Ob84MVZOo2GR0YFi-9Okq9pgA3FfC6_bk,3061
-cartography-0.92.0rc2.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
-cartography-0.92.0rc2.dist-info/METADATA,sha256=67pexbhOi_WeS7TLuIGswFYPCfMbzOz6eSibk8FgvUo,1991
-cartography-0.92.0rc2.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
-cartography-0.92.0rc2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cartography-0.92.0rc2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.92.0rc2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.92.0rc2.dist-info/RECORD,,
+cartography-0.93.0.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
+cartography-0.93.0.dist-info/METADATA,sha256=bZcpA3W4BYf5mQXLdALJ0KEf3_n_PqMI4TiPSda3Bv8,1988
+cartography-0.93.0.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
+cartography-0.93.0.dist-info/WHEEL,sha256=pd56usn78UTvq1xeX_ZwFhoK6jE5u5wzu4TTBIG5cQ0,91
+cartography-0.93.0.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.93.0.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.93.0.dist-info/RECORD,,

{cartography-0.92.0rc2.dist-info → cartography-0.93.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (71.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any