cartography 0.93.0rc1__py3-none-any.whl → 0.94.0rc2__py3-none-any.whl

cartography/intel/aws/ec2/launch_templates.py

@@ -3,6 +3,7 @@ from typing import Any
 
 import boto3
 import neo4j
+from botocore.exceptions import ClientError
 
 from .util import get_botocore_config
 from cartography.client.core.tx import load
@@ -17,13 +18,30 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_launch_templates(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+def get_launch_templates(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     paginator = client.get_paginator('describe_launch_templates')
     templates: list[dict[str, Any]] = []
+    template_versions: list[dict[str, Any]] = []
     for page in paginator.paginate():
-        templates.extend(page['LaunchTemplates'])
-    return templates
+        paginated_templates = page['LaunchTemplates']
+        for template in paginated_templates:
+            template_id = template['LaunchTemplateId']
+            try:
+                versions = get_launch_template_versions_by_template(boto3_session, template_id, region)
+            except ClientError as e:
+                logger.warning(
+                    f"Failed to get launch template versions for {template_id}: {e}",
+                    exc_info=True,
+                )
+                versions = []
+            # Using a key not defined in latest boto3 documentation
+            template_versions.extend(versions)
+        templates.extend(paginated_templates)
+    return templates, template_versions
 
 
 def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -55,17 +73,16 @@ def load_launch_templates(
 
 @timeit
 @aws_handle_regions
-def get_launch_template_versions(
+def get_launch_template_versions_by_template(
         boto3_session: boto3.session.Session,
-        templates: list[dict[str, Any]],
+        template: str,
         region: str,
 ) -> list[dict[str, Any]]:
     client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
     v_paginator = client.get_paginator('describe_launch_template_versions')
     template_versions = []
-    for template in templates:
-        for versions in v_paginator.paginate(LaunchTemplateId=template['LaunchTemplateId']):
-            template_versions.extend(versions['LaunchTemplateVersions'])
+    for versions in v_paginator.paginate(LaunchTemplateId=template):
+        template_versions.extend(versions['LaunchTemplateVersions'])
     return template_versions
 
 
@@ -136,11 +153,9 @@ def sync_ec2_launch_templates(
 ) -> None:
     for region in regions:
         logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
-        templates = get_launch_templates(boto3_session, region)
+        templates, versions = get_launch_templates(boto3_session, region)
         templates = transform_launch_templates(templates)
         load_launch_templates(neo4j_session, templates, region, current_aws_account_id, update_tag)
-
-        versions = get_launch_template_versions(boto3_session, templates, region)
         versions = transform_launch_template_versions(versions)
         load_launch_template_versions(neo4j_session, versions, region, current_aws_account_id, update_tag)
 

cartography/intel/aws/inspector.py

@@ -18,6 +18,15 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# As of 7/22/24, Inspector is only available in the below regions. We will need to update this hardcoded list here over
+# time. :\ https://docs.aws.amazon.com/general/latest/gr/inspector2.html
+AWS_INSPECTOR_REGIONS = {
+    "us-east-2", "us-east-1", "us-west-1", "us-west-2", "af-south-1", "ap-east-1", "ap-southeast-3", "ap-south-1",
+    "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1",
+    "eu-central-1", "eu-west-1", "eu-west-2", "eu-south-1", "eu-west-3", "eu-north-1", "eu-central-2", "me-south-1",
+    "sa-east-1",
+}
+
 
 @timeit
 @aws_handle_regions
@@ -206,7 +215,9 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    for region in regions:
+    inspector_regions = [region for region in regions if region in AWS_INSPECTOR_REGIONS]
+
+    for region in inspector_regions:
         logger.info(f"Syncing AWS Inspector findings for account {current_aws_account_id} and region {region}")
         findings = get_inspector_findings(boto3_session, region, current_aws_account_id)
         finding_data, package_data = transform_inspector_findings(findings)

cartography/intel/github/teams.py

@@ -1,4 +1,6 @@
 import logging
+from collections import namedtuple
+from time import sleep
 from typing import Any
 from typing import Dict
 from typing import List
@@ -15,6 +17,8 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
+
 
 @timeit
 def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
@@ -45,26 +49,53 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
 
 @timeit
 def _get_team_repos_for_multiple_teams(
-        team_raw_data: List[Dict[str, Any]],
+        team_raw_data: list[dict[str, Any]],
         org: str,
         api_url: str,
         token: str,
-) -> Dict[str, Any]:
-    result = {}
+) -> dict[str, list[RepoPermission]]:
+    result: dict[str, list[RepoPermission]] = {}
     for team in team_raw_data:
         team_name = team['slug']
         repo_count = team['repositories']['totalCount']
 
-        team_repos = _get_team_repos(org, api_url, token, team_name) if repo_count > 0 else None
+        if repo_count == 0:
+            # This team has access to no repos so let's move on
+            result[team_name] = []
+            continue
 
         repo_urls = []
         repo_permissions = []
-        if team_repos:
-            repo_urls = [t['url'] for t in team_repos.nodes] if team_repos.nodes else []
-            repo_permissions = [t['permission'] for t in team_repos.edges] if team_repos.edges else []
+
+        max_tries = 5
+
+        for current_try in range(1, max_tries + 1):
+            team_repos = _get_team_repos(org, api_url, token, team_name)
+
+            try:
+                # The `or []` is because `.nodes` can be None. See:
+                # https://docs.github.com/en/graphql/reference/objects#teamrepositoryconnection
+                for repo in team_repos.nodes or []:
+                    repo_urls.append(repo['url'])
+
+                # The `or []` is because `.edges` can be None.
+                for edge in team_repos.edges or []:
+                    repo_permissions.append(edge['permission'])
+                # We're done! Break out of the retry loop.
+                break
+
+            except TypeError:
+                # Handles issue #1334
+                logger.warning(
+                    f"GitHub returned None when trying to find repo or permission data for team {team_name}.",
+                    exc_info=True,
+                )
+                if current_try == max_tries:
+                    raise RuntimeError(f"GitHub returned a None repo url for team {team_name}, retries exhausted.")
+                sleep(current_try ** 2)
 
         # Shape = [(repo_url, 'WRITE'), ...]]
-        result[team_name] = list(zip(repo_urls, repo_permissions))
+        result[team_name] = [RepoPermission(url, perm) for url, perm in zip(repo_urls, repo_permissions)]
     return result
 
 
@@ -114,8 +145,8 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
 def transform_teams(
         team_paginated_data: PaginatedGraphqlData,
         org_data: Dict[str, Any],
-        team_repo_data: Dict[str, Any],
-) -> List[Dict[str, Any]]:
+        team_repo_data: dict[str, list[RepoPermission]],
+) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
         team_name = team['slug']

cartography/util.py

@@ -225,7 +225,7 @@ If not, then the AWS datatype somehow does not have this key.''',
     return items
 
 
-AWSGetFunc = TypeVar('AWSGetFunc', bound=Callable[..., List])
+AWSGetFunc = TypeVar('AWSGetFunc', bound=Callable[..., Iterable])
 
 # fix for AWS TooManyRequestsException
 # https://github.com/lyft/cartography/issues/297

{cartography-0.93.0rc1.dist-info → cartography-0.94.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.93.0rc1
+Version: 0.94.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.93.0rc1.dist-info → cartography-0.94.0rc2.dist-info}/RECORD

@@ -5,7 +5,7 @@ cartography/config.py,sha256=rL1zgxZO47_R7S6E9e0CwxmhzRSN0X_q93NtcPR1G00,11368
 cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/stats.py,sha256=dbybb9V2FuvSuHjjNwz6Vjwnd1hap2C7h960rLoKcl8,4406
 cartography/sync.py,sha256=a80r_IzrZcWGSmRDRrxkesNYPiOuLte5YHvDQT3L-Lw,9730
-cartography/util.py,sha256=F3FPMJl1KDW0x_5cvt2ZGI0Dv1LVrHU7Az4OleAANBI,14474
+cartography/util.py,sha256=umfnjX8jVLu0rpYA75X-WvRpYzHQxns9qZiPwfyAlwQ,14478
 cartography/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/client/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cartography/client/aws/iam.py,sha256=dYsGikc36DEsSeR2XVOVFFUDwuU9yWj_EVkpgVYCFgM,1293
@@ -153,7 +153,7 @@ cartography/intel/aws/elasticache.py,sha256=fCI47aDFmIDyE26GiReKYb6XIZUwrzcvsXBQ
 cartography/intel/aws/elasticsearch.py,sha256=ZL7MkXF_bXRSoXuDSI1dwGckRLG2zDB8LuAD07vSLnE,8374
 cartography/intel/aws/emr.py,sha256=xhWBVZngxJRFjMEDxwq3G6SgytRGLq0v2a_CeDvByR0,3372
 cartography/intel/aws/iam.py,sha256=eLw0NkBGKzCI_tQ3wmrx3aUibQerrsxKJd3d0RCKcKQ,32374
-cartography/intel/aws/inspector.py,sha256=6enCu2USefuGT3FqA0Vto6i-z4BrL2HC_clbiXSLIlo,8654
+cartography/intel/aws/inspector.py,sha256=S22ZgRKEnmnBTJ-u0rodqRPB7_LkSIek47NeBxN4XJw,9336
 cartography/intel/aws/kms.py,sha256=bZUzMxAH_DsAcGTJBs08gg2tLKYu-QWjvMvV9C-6v50,11731
 cartography/intel/aws/lambda_function.py,sha256=KKTyn53xpaMI9WvIqxmsOASFwflHt-2_5ow-zUFc2wg,9890
 cartography/intel/aws/organizations.py,sha256=HaQZ3J5XF15BuykuDypqFORDYpnoHuRRr4DuceewH4s,4485
@@ -175,7 +175,7 @@ cartography/intel/aws/ec2/images.py,sha256=heElwHJGqVD3iUJjxwA_Sdc3CmE4HPs00CTMH
 cartography/intel/aws/ec2/instances.py,sha256=mnTjdBY-4D-TGhH29UrSaLUW0Uft0JApDIJkkLz4zPc,12170
 cartography/intel/aws/ec2/internet_gateways.py,sha256=dI-4-85_3DGGZZBcY_DN6XqESx9P26S6jKok314lcnQ,2883
 cartography/intel/aws/ec2/key_pairs.py,sha256=SvRgd56vE4eouvTSNoFK8PP8HYoECO91goxc36oq_FY,2508
-cartography/intel/aws/ec2/launch_templates.py,sha256=YhLh_O2cZbeoKXA6MXmwkxJJi0ubZb5FyouTYykuq1k,5372
+cartography/intel/aws/ec2/launch_templates.py,sha256=aeqaL8On38ET8nM8bISsIXLy6PkZoV-tqSWG38YXgkI,6010
 cartography/intel/aws/ec2/load_balancer_v2s.py,sha256=95FfQQn740gexINIHDJizOM4OKzRtQT_y2XQMipQ5Dg,8661
 cartography/intel/aws/ec2/load_balancers.py,sha256=1GwErzGqi3BKCARqfGJcD_r_D84rFKVy5kNMas9jAok,6756
 cartography/intel/aws/ec2/network_interfaces.py,sha256=CzF8PooCYUQ2pk8DR8JDAhkWRUQSBj_27OsIfkL_-Cs,9199
@@ -230,7 +230,7 @@ cartography/intel/gcp/gke.py,sha256=qaTwsVaxkwNhW5_Mw4bedOk7fgJK8y0LwwcYlUABXDg,
 cartography/intel/gcp/storage.py,sha256=oO_ayEhkXlj2Gn7T5MU41ZXiqwRwe6Ud4wzqyRTsyf4,9075
 cartography/intel/github/__init__.py,sha256=y876JJGTDJZEOFCDiNCJfcLNxN24pVj4s2N0YmuuoaE,1914
 cartography/intel/github/repos.py,sha256=YPDdBMk6NkZjwPcqPW5LlCy_OS9tKcrZD6ygiUG93J0,21766
-cartography/intel/github/teams.py,sha256=mofyJeJVOD7Umh9Rq6QnAwom9bBHBx18kyvFMvQX5YE,5383
+cartography/intel/github/teams.py,sha256=aXI-XbxlA1IDaAUX0XSdEt6pA2n4ew5j_doj1iNYCDM,6618
 cartography/intel/github/users.py,sha256=kQp0dxzP08DVrdvfVeCciQbrKPbbFvwbR_p_I_XGt7s,3826
 cartography/intel/github/util.py,sha256=K6hbxypy4luKhIE1Uh5VWZc9OyjMK2OuO00vBAQfloA,8049
 cartography/intel/gsuite/__init__.py,sha256=AGIUskGlLCVGHbnQicNpNWi9AvmV7_7hUKTt-hsB2J8,4306
@@ -332,10 +332,10 @@ cartography/models/semgrep/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 cartography/models/semgrep/deployment.py,sha256=or5qZDuR51MXzINpH15jZrqmSUvXQevCNYWJ7D6v-JI,745
 cartography/models/semgrep/findings.py,sha256=xrn8sgXpNMrNJbKQagaAVxaCG9bVjTATSRR2XRBR4rg,5386
 cartography/models/semgrep/locations.py,sha256=kSk7Nn5Mn4Ob84MVZOo2GR0YFi-9Okq9pgA3FfC6_bk,3061
-cartography-0.93.0rc1.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
-cartography-0.93.0rc1.dist-info/METADATA,sha256=1r_kEpkQWTPR77ToFbTkU78iBSTCM9JuQdleKFo3dJE,1991
-cartography-0.93.0rc1.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
-cartography-0.93.0rc1.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-cartography-0.93.0rc1.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.93.0rc1.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.93.0rc1.dist-info/RECORD,,
+cartography-0.94.0rc2.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
+cartography-0.94.0rc2.dist-info/METADATA,sha256=UEwu0NxrNz1H3YQWCQT4grTn4p_yvqC1nFoou0HCvig,1991
+cartography-0.94.0rc2.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
+cartography-0.94.0rc2.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+cartography-0.94.0rc2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.94.0rc2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.94.0rc2.dist-info/RECORD,,

{cartography-0.93.0rc1.dist-info → cartography-0.94.0rc2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any