cartography 0.96.0rc2__tar.gz → 0.96.1__tar.gz

{cartography-0.96.0rc2 → cartography-0.96.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.0rc2
+Version: 0.96.1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/client/core/tx.py

@@ -122,7 +122,7 @@ def read_list_of_tuples_tx(tx: neo4j.Transaction, query: str, **kwargs) -> List[
     return [tuple(val) for val in values]
 
 
-def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Dict[str, Any]:
+def read_single_dict_tx(tx: neo4j.Transaction, query: str, **kwargs) -> Any:
     """
     Runs the given Neo4j query in the given transaction object and returns the single dict result. This is intended to
     be run only with queries that return a single dict.

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/config.py

@@ -157,7 +157,7 @@ class Config:
         pagerduty_request_timeout=None,
         nist_cve_url=None,
         cve_enabled=False,
-        cve_api_key=None,
+        cve_api_key: str | None = None,
         crowdstrike_client_id=None,
         crowdstrike_client_secret=None,
         crowdstrike_api_url=None,
@@ -215,7 +215,7 @@ class Config:
         self.pagerduty_request_timeout = pagerduty_request_timeout
         self.nist_cve_url = nist_cve_url
         self.cve_enabled = cve_enabled
-        self.cve_api_key = cve_api_key
+        self.cve_api_key: str | None = cve_api_key
         self.crowdstrike_client_id = crowdstrike_client_id
         self.crowdstrike_client_secret = crowdstrike_client_secret
         self.crowdstrike_api_url = crowdstrike_api_url

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -63,6 +63,31 @@
     "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
     "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_MAINTAIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_READ]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_TRIAGE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:DIRECT_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
   }],
   "name": "cleanup GitHub repos data"
 }

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/aws/apigateway.py

@@ -43,7 +43,7 @@ def get_rest_api_details(
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)
         # clientcertificate id is given by the api stage
-        certificate = get_rest_api_client_certificate(stages, client)  # type: ignore
+        certificate = get_rest_api_client_certificate(stages, client)
         resources = get_rest_api_resources(api, client)
         policy = get_rest_api_policy(api, client)
         apis.append((api['id'], stages, certificate, resources, policy))
@@ -51,7 +51,7 @@ def get_rest_api_details(
 
 
 @timeit
-def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
     """
@@ -99,7 +99,7 @@ def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> Lis
 
 
 @timeit
-def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API policy. Returns policy string or None if no policy is present.
     """

cartography-0.96.1/cartography/intel/aws/ec2/auto_scaling_groups.py

@@ -0,0 +1,205 @@
+import logging
+from collections import namedtuple
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema
+from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
+from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
+from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+AsgData = namedtuple(
+    'AsgData', [
+        "group_list",
+        "instance_list",
+        "subnet_list",
+    ],
+)
+
+
+@timeit
+@aws_handle_regions
+def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> list[dict]:
+    client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_auto_scaling_groups')
+    asgs: list[dict] = []
+    for page in paginator.paginate():
+        asgs.extend(page['AutoScalingGroups'])
+    return asgs
+
+
+@timeit
+@aws_handle_regions
+def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> list[dict]:
+    client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_launch_configurations')
+    lcs: list[dict] = []
+    for page in paginator.paginate():
+        lcs.extend(page['LaunchConfigurations'])
+    return lcs
+
+
+def transform_launch_configurations(configurations: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    transformed_configurations = []
+    for config in configurations:
+        transformed_configurations.append({
+            'AssociatePublicIpAddress': config.get('AssociatePublicIpAddress'),
+            'LaunchConfigurationARN': config.get('LaunchConfigurationARN'),
+            'LaunchConfigurationName': config.get('LaunchConfigurationName'),
+            'CreatedTime': config.get('CreatedTime'),
+            'ImageId': config.get('ImageId'),
+            'KeyName': config.get('KeyName'),
+            'SecurityGroups': config.get('SecurityGroups'),
+            'InstanceType': config.get('InstanceType'),
+            'KernelId': config.get('KernelId'),
+            'RamdiskId': config.get('RamdiskId'),
+            'InstanceMonitoring': config.get('InstanceMonitoring', {}).get('Enabled'),
+            'SpotPrice': config.get('SpotPrice'),
+            'IamInstanceProfile': config.get('IamInstanceProfile'),
+            'EbsOptimized': config.get('EbsOptimized'),
+            'PlacementTenancy': config.get('PlacementTenancy'),
+        })
+    return transformed_configurations
+
+
+def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
+    transformed_groups = []
+    related_vpcs = []
+    related_instances = []
+    for group in groups:
+        transformed_groups.append({
+            'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+            'CapacityRebalance': group.get('CapacityRebalance'),
+            'CreatedTime': str(group.get('CreatedTime')),
+            'DefaultCooldown': group.get('DefaultCooldown'),
+            'DesiredCapacity': group.get('DesiredCapacity'),
+            'HealthCheckGracePeriod': group.get('HealthCheckGracePeriod'),
+            'HealthCheckType': group.get('HealthCheckType'),
+            'LaunchConfigurationName': group.get('LaunchConfigurationName'),
+            'LaunchTemplateName': group.get('LaunchTemplate', {}).get('LaunchTemplateName'),
+            'LaunchTemplateId': group.get('LaunchTemplate', {}).get('LaunchTemplateId'),
+            'LaunchTemplateVersion': group.get('LaunchTemplate', {}).get('Version'),
+            'MaxInstanceLifetime': group.get('MaxInstanceLifetime'),
+            'MaxSize': group.get('MaxSize'),
+            'MinSize': group.get('MinSize'),
+            'AutoScalingGroupName': group.get('AutoScalingGroupName'),
+            'NewInstancesProtectedFromScaleIn': group.get('NewInstancesProtectedFromScaleIn'),
+            'Status': group.get('Status'),
+        })
+
+        if group.get('VPCZoneIdentifier', None):
+            vpclist = group['VPCZoneIdentifier']
+            subnet_ids = vpclist.split(',') if ',' in vpclist else [vpclist]
+            subnets = []
+            for subnet_id in subnet_ids:
+                subnets.append({
+                    'VPCZoneIdentifier': subnet_id,
+                    'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+                })
+            related_vpcs.extend(subnets)
+
+        for instance_data in group.get('Instances', []):
+            related_instances.append({
+                'InstanceId': instance_data['InstanceId'],
+                'AutoScalingGroupARN': group['AutoScalingGroupARN'],
+            })
+
+    return AsgData(
+        group_list=transformed_groups,
+        instance_list=related_instances,
+        subnet_list=related_vpcs,
+    )
+
+
+@timeit
+def load_launch_configurations(
+    neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LaunchConfigurationSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+def load_groups(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AutoScalingGroupSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+def load_asg_subnets(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EC2SubnetAutoScalingGroupSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+def load_asg_instances(
+        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EC2InstanceAutoScalingGroupSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_auto_scaling_groups(
+    neo4j_session: neo4j.Session, data: AsgData, region: str, current_aws_account_id: str, update_tag: int,
+) -> None:
+    load_groups(neo4j_session, data.group_list, region, current_aws_account_id, update_tag)
+    load_asg_instances(neo4j_session, data.instance_list, region, current_aws_account_id, update_tag)
+    load_asg_subnets(neo4j_session, data.subnet_list, region, current_aws_account_id, update_tag)
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running EC2 instance cleanup")
+    GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_ec2_auto_scaling_groups(
+        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str],
+        current_aws_account_id: str, update_tag: int, common_job_parameters: dict,
+) -> None:
+    for region in regions:
+        logger.debug("Syncing auto scaling groups for region '%s' in account '%s'.", region, current_aws_account_id)
+        lc_data = get_launch_configurations(boto3_session, region)
+        asg_data = get_ec2_auto_scaling_groups(boto3_session, region)
+        lc_transformed = transform_launch_configurations(lc_data)
+        asg_transformed = transform_auto_scaling_groups(asg_data)
+        load_launch_configurations(neo4j_session, lc_transformed, region, current_aws_account_id, update_tag)
+        load_auto_scaling_groups(neo4j_session, asg_transformed, region, current_aws_account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/aws/ec2/instances.py

@@ -11,6 +11,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
 from cartography.models.aws.ec2.instances import EC2InstanceSchema
 from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
 from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema
@@ -308,6 +309,7 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
     logger.debug("Running EC2 instance cleanup")
     GraphJob.from_node_schema(EC2ReservationSchema(), common_job_parameters).run(neo4j_session)
     GraphJob.from_node_schema(EC2InstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2InstanceAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/aws/ec2/network_acls.py

@@ -69,7 +69,8 @@ def transform_network_acl_data(
                 direction = 'egress' if rule['Egress'] else 'inbound'
                 transformed_rule = {
                     'Id': f"{network_acl['NetworkAclId']}/{direction}/{rule['RuleNumber']}",
-                    'CidrBlock': rule['CidrBlock'],
+                    'CidrBlock': rule.get('CidrBlock'),
+                    'Ipv6CidrBlock': rule.get('Ipv6CidrBlock'),
                     'Egress': rule['Egress'],
                     'Protocol': rule['Protocol'],
                     'RuleAction': rule['RuleAction'],

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/aws/ec2/subnets.py

@@ -7,6 +7,7 @@ import neo4j
 
 from .util import get_botocore_config
 from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
 from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
@@ -79,6 +80,7 @@ def load_subnets(
 def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters)
     GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2SubnetAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/aws/iam.py

@@ -539,11 +539,12 @@ def _transform_policy_statements(statements: Any, policy_id: str) -> List[Dict]:
     if not isinstance(statements, list):
         statements = [statements]
     for stmt in statements:
-        if "Sid" not in stmt:
+        if "Sid" in stmt and stmt["Sid"]:
+            statement_id = stmt["Sid"]
+        else:
             statement_id = count
             count += 1
-        else:
-            statement_id = stmt["Sid"]
+
         stmt["id"] = f"{policy_id}/statement/{statement_id}"
         if "Resource" in stmt:
             stmt["Resource"] = ensure_list(stmt["Resource"])

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/cve/__init__.py

@@ -25,7 +25,7 @@ def start_cve_ingestion(
     """
     if not config.cve_enabled:
         return
-    cve_api_key = config.cve_api_key if config.cve_api_key else None
+    cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
 
     # sync CVE year archives, if not yet synced
     existing_years = feed.get_cve_sync_metadata(neo4j_session)

{cartography-0.96.0rc2 → cartography-0.96.1}/cartography/intel/cve/feed.py

@@ -22,9 +22,9 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
-MAX_RETRIES = 3
-# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
-CONNECT_AND_READ_TIMEOUT = (60, 60)
+MAX_RETRIES = 8
+# Connect and read timeouts of 120 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+CONNECT_AND_READ_TIMEOUT = (30, 120)
 CVE_FEED_ID = "NIST_NVD"
 BATCH_SIZE_DAYS = 120
 RESULTS_PER_PAGE = 2000
@@ -68,7 +68,7 @@ def _map_cve_dict(cve_dict: Dict[Any, Any], data: Dict[Any, Any]) -> None:
     cve_dict["startIndex"] = data["startIndex"]
 
 
-def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any, Any]:
+def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dict[Any, Any]:
     totalResults = 0
     sleep_time = DEFAULT_SLEEP_TIME
     retries = 0
@@ -98,6 +98,9 @@ def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any,
             retries += 1
             if retries >= MAX_RETRIES:
                 raise
+            # Exponential backoff
+            sleep_time *= 2
+            time.sleep(sleep_time)
             continue
         data = res.json()
         _map_cve_dict(results, data)
@@ -114,7 +117,7 @@ def get_cves_in_batches(
     start_date: datetime,
     end_date: datetime,
     date_param_names: Dict[str, str],
-    api_key: str,
+    api_key: str | None,
 ) -> Dict[Any, Any]:
     cves: Dict[Any, Any] = dict()
     current_start_date: datetime = start_date
@@ -153,7 +156,7 @@ def get_cves_in_batches(
 
 
 def get_modified_cves(
-    nist_cve_url: str, last_modified_date: str, api_key: str,
+    nist_cve_url: str, last_modified_date: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = dict()
     end_date = datetime.now(tz=timezone.utc)
@@ -171,7 +174,7 @@ def get_modified_cves(
 
 
 def get_published_cves_per_year(
-    nist_cve_url: str, year: str, api_key: str,
+    nist_cve_url: str, year: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = {}
     start_of_year = datetime.strptime(f"{year}-01-01", "%Y-%m-%d")