cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/aws/apigateway.py

@@ -15,7 +15,9 @@ from policyuniverse.policy import Policy
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.apigateway import APIGatewayRestAPISchema
-from cartography.models.aws.apigatewaycertificate import APIGatewayClientCertificateSchema
+from cartography.models.aws.apigatewaycertificate import (
+    APIGatewayClientCertificateSchema,
+)
 from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
 from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
@@ -26,24 +28,29 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('apigateway', region_name=region)
-    paginator = client.get_paginator('get_rest_apis')
+def get_apigateway_rest_apis(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict]:
+    client = boto3_session.client("apigateway", region_name=region)
+    paginator = client.get_paginator("get_rest_apis")
     apis: List[Any] = []
     for page in paginator.paginate():
-        apis.extend(page['items'])
+        apis.extend(page["items"])
     return apis
 
 
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
-        boto3_session: boto3.session.Session, rest_apis: List[Dict], region: str,
+    boto3_session: boto3.session.Session,
+    rest_apis: List[Dict],
+    region: str,
 ) -> List[Tuple[Any, Any, Any, Any, Any]]:
     """
     Iterates over all API Gateway REST APIs.
     """
-    client = boto3_session.client('apigateway', region_name=region)
+    client = boto3_session.client("apigateway", region_name=region)
     apis = []
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)
@@ -51,7 +58,7 @@ def get_rest_api_details(
         certificate = get_rest_api_client_certificate(stages, client)
         resources = get_rest_api_resources(api, client)
         policy = get_rest_api_policy(api, client)
-        apis.append((api['id'], stages, certificate, resources, policy))
+        apis.append((api["id"], stages, certificate, resources, policy))
     return apis
 
 
@@ -61,27 +68,34 @@ def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     Gets the REST API Stage Resources.
     """
     try:
-        stages = client.get_stages(restApiId=api['id'])
+        stages = client.get_stages(restApiId=api["id"])
     except ClientError as e:
         logger.warning(f'Failed to retrieve Stages for Api Id - {api["id"]} - {e}')
         raise
 
-    return stages['item']
+    return stages["item"]
 
 
 @timeit
-def get_rest_api_client_certificate(stages: Dict, client: botocore.client.BaseClient) -> Optional[Any]:
+def get_rest_api_client_certificate(
+    stages: Dict,
+    client: botocore.client.BaseClient,
+) -> Optional[Any]:
     """
     Gets the current ClientCertificate resource if present, else returns None.
     """
     response = None
     for stage in stages:
-        if 'clientCertificateId' in stage:
+        if "clientCertificateId" in stage:
             try:
-                response = client.get_client_certificate(clientCertificateId=stage['clientCertificateId'])
-                response['stageName'] = stage['stageName']
+                response = client.get_client_certificate(
+                    clientCertificateId=stage["clientCertificateId"],
+                )
+                response["stageName"] = stage["stageName"]
             except ClientError as e:
-                logger.warning(f"Failed to retrive Client Certificate for Stage {stage['stageName']} - {e}")
+                logger.warning(
+                    f"Failed to retrieve Client Certificate for Stage {stage['stageName']} - {e}",
+                )
                 raise
         else:
             return []
@@ -95,10 +109,10 @@ def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> Lis
     Gets the collection of Resource resources.
     """
     resources: List[Any] = []
-    paginator = client.get_paginator('get_resources')
-    response_iterator = paginator.paginate(restApiId=api['id'])
+    paginator = client.get_paginator("get_resources")
+    response_iterator = paginator.paginate(restApiId=api["id"])
     for page in response_iterator:
-        resources.extend(page['items'])
+        resources.extend(page["items"])
 
     return resources
 
@@ -108,34 +122,35 @@ def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API policy. Returns policy string or None if no policy is present.
     """
-    policy = api['policy'] if 'policy' in api and api['policy'] else None
+    policy = api["policy"] if "policy" in api and api["policy"] else None
     return policy
 
 
 def transform_apigateway_rest_apis(
-    rest_apis: List[Dict], resource_policies: List[Dict], region: str, current_aws_account_id: str, aws_update_tag: int,
+    rest_apis: List[Dict],
+    resource_policies: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> List[Dict]:
     """
     Transform API Gateway REST API data for ingestion, including policy analysis
     """
     # Create a mapping of api_id to policy data for easier lookup
-    policy_map = {
-        policy['api_id']: policy
-        for policy in resource_policies
-    }
+    policy_map = {policy["api_id"]: policy for policy in resource_policies}
 
     transformed_apis = []
     for api in rest_apis:
-        policy_data = policy_map.get(api['id'], {})
+        policy_data = policy_map.get(api["id"], {})
         transformed_api = {
-            'id': api['id'],
-            'createdDate': str(api['createdDate']) if 'createdDate' in api else None,
-            'version': api.get('version'),
-            'minimumCompressionSize': api.get('minimumCompressionSize'),
-            'disableExecuteApiEndpoint': api.get('disableExecuteApiEndpoint'),
+            "id": api["id"],
+            "createdDate": str(api["createdDate"]) if "createdDate" in api else None,
+            "version": api.get("version"),
+            "minimumCompressionSize": api.get("minimumCompressionSize"),
+            "disableExecuteApiEndpoint": api.get("disableExecuteApiEndpoint"),
             # Set defaults in the transform function
-            'anonymous_access': policy_data.get('internet_accessible', False),
-            'anonymous_actions': policy_data.get('accessible_actions', []),
+            "anonymous_access": policy_data.get("internet_accessible", False),
+            "anonymous_actions": policy_data.get("accessible_actions", []),
             # TODO Issue #1452: clarify internet exposure vs anonymous access
         }
         transformed_apis.append(transformed_api)
@@ -145,7 +160,10 @@ def transform_apigateway_rest_apis(
 
 @timeit
 def load_apigateway_rest_apis(
-    neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     """
@@ -167,21 +185,26 @@ def transform_apigateway_stages(stages: List[Dict], update_tag: int) -> List[Dic
     """
     stage_data = []
     for stage in stages:
-        stage['createdDate'] = str(stage['createdDate'])
-        stage['arn'] = f"arn:aws:apigateway:::{stage['apiId']}/{stage['stageName']}"
+        stage["createdDate"] = str(stage["createdDate"])
+        stage["arn"] = f"arn:aws:apigateway:::{stage['apiId']}/{stage['stageName']}"
         stage_data.append(stage)
     return stage_data
 
 
-def transform_apigateway_certificates(certificates: List[Dict], update_tag: int) -> List[Dict]:
+def transform_apigateway_certificates(
+    certificates: List[Dict],
+    update_tag: int,
+) -> List[Dict]:
     """
     Transform API Gateway Client Certificate data for ingestion
     """
     cert_data = []
     for certificate in certificates:
-        certificate['createdDate'] = str(certificate['createdDate'])
-        certificate['expirationDate'] = str(certificate.get('expirationDate'))
-        certificate['stageArn'] = f"arn:aws:apigateway:::{certificate['apiId']}/{certificate['stageName']}"
+        certificate["createdDate"] = str(certificate["createdDate"])
+        certificate["expirationDate"] = str(certificate.get("expirationDate"))
+        certificate["stageArn"] = (
+            f"arn:aws:apigateway:::{certificate['apiId']}/{certificate['stageName']}"
+        )
         cert_data.append(certificate)
     return cert_data
 
@@ -199,21 +222,23 @@ def transform_rest_api_details(
     for api_id, stage, certificate, resource, _ in stages_certificate_resources:
         if len(stage) > 0:
             for s in stage:
-                s['apiId'] = api_id
-                s['createdDate'] = str(s['createdDate'])
-                s['arn'] = f"arn:aws:apigateway:::{api_id}/{s['stageName']}"
+                s["apiId"] = api_id
+                s["createdDate"] = str(s["createdDate"])
+                s["arn"] = f"arn:aws:apigateway:::{api_id}/{s['stageName']}"
             stages.extend(stage)
 
         if certificate:
-            certificate['apiId'] = api_id
-            certificate['createdDate'] = str(certificate['createdDate'])
-            certificate['expirationDate'] = str(certificate.get('expirationDate'))
-            certificate['stageArn'] = f"arn:aws:apigateway:::{api_id}/{certificate['stageName']}"
+            certificate["apiId"] = api_id
+            certificate["createdDate"] = str(certificate["createdDate"])
+            certificate["expirationDate"] = str(certificate.get("expirationDate"))
+            certificate["stageArn"] = (
+                f"arn:aws:apigateway:::{api_id}/{certificate['stageName']}"
+            )
             certificates.append(certificate)
 
         if len(resource) > 0:
             for r in resource:
-                r['apiId'] = api_id
+                r["apiId"] = api_id
             resources.extend(resource)
 
     return stages, certificates, resources
@@ -221,13 +246,17 @@ def transform_rest_api_details(
 
 @timeit
 def load_rest_api_details(
-    neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
-    aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
+    aws_account_id: str,
+    update_tag: int,
 ) -> None:
     """
     Transform and load Stage, Client Certificate, and Resource data
     """
-    stages, certificates, resources = transform_rest_api_details(stages_certificate_resources)
+    stages, certificates, resources = transform_rest_api_details(
+        stages_certificate_resources,
+    )
 
     load(
         neo4j_session,
@@ -274,7 +303,7 @@ def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
             else:
                 return None
         except json.JSONDecodeError:
-            logger.warn(f"failed to decode policy json : {policy}")
+            logger.warning(f"failed to decode policy json : {policy}")
             return None
     else:
         return None
@@ -289,29 +318,48 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     logger.info("Running API Gateway cleanup job.")
 
     # Clean up certificates first
-    cleanup_job = GraphJob.from_node_schema(APIGatewayClientCertificateSchema(), common_job_parameters)
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayClientCertificateSchema(),
+        common_job_parameters,
+    )
     cleanup_job.run(neo4j_session)
 
     # Then stages
-    cleanup_job = GraphJob.from_node_schema(APIGatewayStageSchema(), common_job_parameters)
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayStageSchema(),
+        common_job_parameters,
+    )
     cleanup_job.run(neo4j_session)
 
     # Then resources
-    cleanup_job = GraphJob.from_node_schema(APIGatewayResourceSchema(), common_job_parameters)
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayResourceSchema(),
+        common_job_parameters,
+    )
     cleanup_job.run(neo4j_session)
 
     # Finally REST APIs
-    cleanup_job = GraphJob.from_node_schema(APIGatewayRestAPISchema(), common_job_parameters)
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayRestAPISchema(),
+        common_job_parameters,
+    )
     cleanup_job.run(neo4j_session)
 
 
 @timeit
 def sync_apigateway_rest_apis(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, region: str, current_aws_account_id: str,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     rest_apis = get_apigateway_rest_apis(boto3_session, region)
-    stages_certificate_resources = get_rest_api_details(boto3_session, rest_apis, region)
+    stages_certificate_resources = get_rest_api_details(
+        boto3_session,
+        rest_apis,
+        region,
+    )
 
     # Extract policies and transform the data
     policies = []
@@ -327,16 +375,39 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
-    load_apigateway_rest_apis(neo4j_session, transformed_apis, region, current_aws_account_id, aws_update_tag)
-    load_rest_api_details(neo4j_session, stages_certificate_resources, current_aws_account_id, aws_update_tag)
+    load_apigateway_rest_apis(
+        neo4j_session,
+        transformed_apis,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+    load_rest_api_details(
+        neo4j_session,
+        stages_certificate_resources,
+        current_aws_account_id,
+        aws_update_tag,
+    )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing AWS APIGateway Rest APIs for region '{region}' in account '{current_aws_account_id}'.")
-        sync_apigateway_rest_apis(neo4j_session, boto3_session, region, current_aws_account_id, update_tag)
+        logger.info(
+            f"Syncing AWS APIGateway Rest APIs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        sync_apigateway_rest_apis(
+            neo4j_session,
+            boto3_session,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/cloudtrail.py

@@ -0,0 +1,127 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import botocore.exceptions
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudtrail.trail import CloudTrailTrailSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_cloudtrail_trails(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cloudtrail", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_trails")
+    trails = []
+    for page in paginator.paginate():
+        trails.extend(page["Trails"])
+
+    # CloudTrail multi-region trails are shown in list_trails,
+    # but the get_trail call only works in the home region
+    trails_filtered = [trail for trail in trails if trail.get("HomeRegion") == region]
+    return trails_filtered
+
+
+@timeit
+def get_cloudtrail_trail(
+    boto3_session: boto3.Session,
+    region: str,
+    trail_name: str,
+) -> Dict[str, Any]:
+    client = boto3_session.client(
+        "cloudtrail", region_name=region, config=get_botocore_config()
+    )
+    trail_details: Dict[str, Any] = {}
+    try:
+        response = client.get_trail(Name=trail_name)
+        trail_details = response["Trail"]
+    except botocore.exceptions.ClientError as e:
+        code = e.response["Error"]["Code"]
+        msg = e.response["Error"]["Message"]
+        logger.warning(
+            f"Could not run CloudTrail get_trail due to boto3 error {code}: {msg}. Skipping.",
+        )
+    return trail_details
+
+
+@timeit
+def load_cloudtrail_trails(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudTrail {len(data)} trails for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudTrailTrailSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running CloudTrail cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        CloudTrailTrailSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        trails = get_cloudtrail_trails(boto3_session, region)
+        trail_data: List[Dict[str, Any]] = []
+        for trail in trails:
+            trail_name = trail["Name"]
+            trail_details = get_cloudtrail_trail(
+                boto3_session,
+                region,
+                trail_name,
+            )
+            if trail_details:
+                trail_data.append(trail_details)
+
+        load_cloudtrail_trails(
+            neo4j_session,
+            trail_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/cloudwatch.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_groups(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cloudwatch", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_log_groups")
+    logGroups = []
+    for page in paginator.paginate():
+        logGroups.extend(page["logGroups"])
+    return logGroups
+
+
+@timeit
+def load_cloudwatch_log_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log groups for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogGroupSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running CloudWatch cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        CloudWatchLogGroupSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        logGroups = get_cloudwatch_log_groups(boto3_session, region)
+        group_data: List[Dict[str, Any]] = []
+        for logGroup in logGroups:
+            group_data.append(logGroup)
+
+        load_cloudwatch_log_groups(
+            neo4j_session,
+            group_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)