cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/apigateway.py

@@ -14,12 +14,22 @@ from policyuniverse.policy import Policy
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.models.aws.apigateway import APIGatewayRestAPISchema
-from cartography.models.aws.apigatewaycertificate import (
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.apigateway.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigateway.apigatewaycertificate import (
     APIGatewayClientCertificateSchema,
 )
-from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
-from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
+from cartography.models.aws.apigateway.apigatewaydeployment import (
+    APIGatewayDeploymentSchema,
+)
+from cartography.models.aws.apigateway.apigatewayintegration import (
+    APIGatewayIntegrationSchema,
+)
+from cartography.models.aws.apigateway.apigatewaymethod import APIGatewayMethodSchema
+from cartography.models.aws.apigateway.apigatewayresource import (
+    APIGatewayResourceSchema,
+)
+from cartography.models.aws.apigateway.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -40,13 +50,45 @@ def get_apigateway_rest_apis(
     return apis
 
 
+def get_rest_api_ids(
+    rest_apis: List[Dict],
+) -> List[str]:
+    """
+    Extracts the IDs of the REST APIs from the provided list.
+    """
+    return [api["id"] for api in rest_apis if "id" in api]
+
+
+@timeit
+@aws_handle_regions
+def get_rest_api_deployments(
+    boto3_session: boto3.session.Session,
+    rest_api_ids: List[str],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Retrieves the deployments for each REST API in the provided list.
+    """
+    client = boto3_session.client(
+        "apigateway", region_name=region, config=get_botocore_config()
+    )
+    deployments: List[Dict[str, Any]] = []
+    for api_id in rest_api_ids:
+        paginator = client.get_paginator("get_deployments")
+        for page in paginator.paginate(restApiId=api_id):
+            for deployment in page.get("items", []):
+                deployment["api_id"] = api_id
+                deployments.append(deployment)
+    return deployments
+
+
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
     boto3_session: boto3.session.Session,
     rest_apis: List[Dict],
     region: str,
-) -> List[Tuple[Any, Any, Any, Any, Any]]:
+) -> List[Tuple[Any, Any, Any, Any, Any, Any, Any]]:
     """
     Iterates over all API Gateway REST APIs.
     """
@@ -56,13 +98,19 @@ def get_rest_api_details(
         stages = get_rest_api_stages(api, client)
         # clientcertificate id is given by the api stage
         certificate = get_rest_api_client_certificate(stages, client)
-        resources = get_rest_api_resources(api, client)
+        resources, methods, integrations = get_rest_api_resources_methods_integrations(
+            api,
+            client,
+        )
         policy = get_rest_api_policy(api, client)
-        apis.append((api["id"], stages, certificate, resources, policy))
+        apis.append(
+            (api["id"], stages, certificate, resources, methods, integrations, policy)
+        )
     return apis
 
 
 @timeit
+@aws_handle_regions
 def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
@@ -104,17 +152,57 @@ def get_rest_api_client_certificate(
 
 
 @timeit
-def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+@aws_handle_regions
+def get_rest_api_resources_methods_integrations(
+    api: Dict, client: botocore.client.BaseClient
+) -> Tuple[List[Any], List[Dict], List[Dict]]:
     """
     Gets the collection of Resource resources.
     """
     resources: List[Any] = []
+    methods: List[Any] = []
+    integrations: List[Any] = []
+
     paginator = client.get_paginator("get_resources")
     response_iterator = paginator.paginate(restApiId=api["id"])
     for page in response_iterator:
-        resources.extend(page["items"])
-
-    return resources
+        page_resources = page["items"]
+        resources.extend(page_resources)
+
+        for resource in page_resources:
+            resource_id = resource["id"]
+            resource_methods = resource.get("resourceMethods", {})
+
+            for http_method, method in resource_methods.items():
+                method["resourceId"] = resource_id
+                method["apiId"] = api["id"]
+                method["httpMethod"] = http_method
+                methods.append(method)
+                try:
+                    integration = client.get_integration(
+                        restApiId=api["id"],
+                        resourceId=resource_id,
+                        httpMethod=http_method,
+                    )
+                except ClientError as e:
+                    error_code = e.response.get("Error", {}).get("Code")
+                    if error_code == "NotFoundException":
+                        logger.warning(
+                            "No integration found for API %s resource %s method %s: %s",
+                            api["id"],
+                            resource_id,
+                            http_method,
+                            e,
+                        )
+                        continue
+                    raise
+                integration["resourceId"] = resource_id
+                integration["apiId"] = api["id"]
+                integration["integrationHttpMethod"] = integration.get("httpMethod")
+                integration["httpMethod"] = http_method
+                integrations.append(integration)
+
+    return resources, methods, integrations
 
 
 @timeit
@@ -210,16 +298,27 @@ def transform_apigateway_certificates(
 
 
 def transform_rest_api_details(
-    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
-) -> Tuple[List[Dict], List[Dict], List[Dict]]:
+    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any, Any, Any]],
+) -> Tuple[List[Dict], List[Dict], List[Dict], List[Dict], List[Dict]]:
     """
-    Transform Stage, Client Certificate, and Resource data for ingestion
+    Transform Stage, Client Certificate, Resource, Method and Integration data for ingestion
     """
     stages: List[Dict] = []
     certificates: List[Dict] = []
     resources: List[Dict] = []
+    methods: List[Dict] = []
+    integrations: List[Dict] = []
+
+    for (
+        api_id,
+        stage,
+        certificate,
+        resource,
+        method_list,
+        integration_list,
+        _,
+    ) in stages_certificate_resources:
 
-    for api_id, stage, certificate, resource, _ in stages_certificate_resources:
         if len(stage) > 0:
             for s in stage:
                 s["apiId"] = api_id
@@ -241,21 +340,67 @@ def transform_rest_api_details(
                 r["apiId"] = api_id
             resources.extend(resource)
 
-    return stages, certificates, resources
+        if len(method_list) > 0:
+            for method in method_list:
+                method["id"] = (
+                    f"{method['apiId']}/{method['resourceId']}/{method['httpMethod']}"
+                )
+                method["authorizationType"] = method.get("authorizationType")
+                method["authorizerId"] = method.get("authorizerId")
+                method["requestValidatorId"] = method.get("requestValidatorId")
+                method["operationName"] = method.get("operationName")
+                method["apiKeyRequired"] = method.get("apiKeyRequired", False)
+            methods.extend(method_list)
+
+        if len(integration_list) > 0:
+            for integration in integration_list:
+                if not integration.get("id"):
+                    integration["id"] = (
+                        f"{integration['apiId']}/{integration['resourceId']}/{integration['httpMethod']}"
+                    )
+                integration["type"] = integration.get("type")
+                integration["uri"] = integration.get("uri")
+                integration["connectionType"] = integration.get("connectionType")
+                integration["connectionId"] = integration.get("connectionId")
+                integration["credentials"] = integration.get("credentials")
+            integrations.extend(integration_list)
+
+    return stages, certificates, resources, methods, integrations
+
+
+def transform_apigateway_deployments(
+    deployments: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform API Gateway Deployment data for ingestion
+    """
+    transformed_deployments = []
+    for deployment in deployments:
+        transformed_deployment = {
+            "id": f"{deployment['api_id']}/{deployment['id']}",
+            "api_id": deployment["api_id"],
+            "description": deployment.get("description"),
+            "region": region,
+        }
+        transformed_deployments.append(transformed_deployment)
+    return transformed_deployments
 
 
 @timeit
 def load_rest_api_details(
     neo4j_session: neo4j.Session,
-    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
+    stages_certificate_resources_methods_integrations: List[
+        Tuple[Any, Any, Any, Any, Any, Any, Any]
+    ],
     aws_account_id: str,
     update_tag: int,
 ) -> None:
     """
     Transform and load Stage, Client Certificate, and Resource data
     """
-    stages, certificates, resources = transform_rest_api_details(
-        stages_certificate_resources,
+    stages, certificates, resources, methods, integrations = transform_rest_api_details(
+        stages_certificate_resources_methods_integrations,
     )
 
     load(
@@ -282,6 +427,46 @@ def load_rest_api_details(
         AWS_ID=aws_account_id,
     )
 
+    load(
+        neo4j_session,
+        APIGatewayMethodSchema(),
+        methods,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+    load(
+        neo4j_session,
+        APIGatewayIntegrationSchema(),
+        integrations,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def load_apigateway_deployments(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load API Gateway Deployment data into neo4j.
+    """
+    logger.info(
+        f"Loading API Gateway {len(data)} deployments for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        APIGatewayDeploymentSchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
+    )
+
 
 @timeit
 def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
@@ -345,6 +530,24 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     )
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayDeploymentSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayMethodSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayIntegrationSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync_apigateway_rest_apis(
@@ -355,7 +558,7 @@ def sync_apigateway_rest_apis(
     aws_update_tag: int,
 ) -> None:
     rest_apis = get_apigateway_rest_apis(boto3_session, region)
-    stages_certificate_resources = get_rest_api_details(
+    stages_certificate_resources_methods_integrations = get_rest_api_details(
         boto3_session,
         rest_apis,
         region,
@@ -363,7 +566,15 @@ def sync_apigateway_rest_apis(
 
     # Extract policies and transform the data
     policies = []
-    for api_id, _, _, _, policy in stages_certificate_resources:
+    for (
+        api_id,
+        _,
+        _,
+        _,
+        _,
+        _,
+        policy,
+    ) in stages_certificate_resources_methods_integrations:
         parsed_policy = parse_policy(api_id, policy)
         if parsed_policy is not None:
             policies.append(parsed_policy)
@@ -375,6 +586,19 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+
+    api_ids = get_rest_api_ids(rest_apis)
+    deployments = get_rest_api_deployments(
+        boto3_session,
+        api_ids,
+        region,
+    )
+
+    transformed_deployments = transform_apigateway_deployments(
+        deployments,
+        region,
+    )
+
     load_apigateway_rest_apis(
         neo4j_session,
         transformed_apis,
@@ -384,7 +608,14 @@ def sync_apigateway_rest_apis(
     )
     load_rest_api_details(
         neo4j_session,
-        stages_certificate_resources,
+        stages_certificate_resources_methods_integrations,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+    load_apigateway_deployments(
+        neo4j_session,
+        transformed_deployments,
+        region,
         current_aws_account_id,
         aws_update_tag,
     )

cartography/intel/aws/apigatewayv2.py

@@ -0,0 +1,116 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.apigatewayv2.apigatewayv2 import APIGatewayV2APISchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_apigatewayv2_apis(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict[str, Any]]:
+    client = boto3_session.client("apigatewayv2", region_name=region)
+    paginator = client.get_paginator("get_apis")
+    apis: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        apis.extend(page.get("Items", []))
+    return apis
+
+
+def transform_apigatewayv2_apis(apis: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    transformed: list[dict[str, Any]] = []
+    for api in apis:
+        transformed.append(
+            {
+                "id": api.get("ApiId"),
+                "name": api.get("Name"),
+                "protocoltype": api.get("ProtocolType"),
+                "routeselectionexpression": api.get("RouteSelectionExpression"),
+                "apikeyselectionexpression": api.get("ApiKeySelectionExpression"),
+                "apiendpoint": api.get("ApiEndpoint"),
+                "version": api.get("Version"),
+                "createddate": api.get("CreatedDate"),
+                "description": api.get("Description"),
+            },
+        )
+    return transformed
+
+
+@timeit
+def load_apigatewayv2_apis(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        APIGatewayV2APISchema(),
+        data,
+        lastupdated=update_tag,
+        AWS_ID=current_aws_account_id,
+        region=region,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        APIGatewayV2APISchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_apigatewayv2_apis(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    apis = get_apigatewayv2_apis(boto3_session, region)
+    transformed = transform_apigatewayv2_apis(apis)
+    load_apigatewayv2_apis(
+        neo4j_session,
+        transformed,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing AWS APIGatewayV2 APIs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        sync_apigatewayv2_apis(
+            neo4j_session,
+            boto3_session,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/cloudtrail.py

@@ -4,7 +4,6 @@ from typing import Dict
 from typing import List
 
 import boto3
-import botocore.exceptions
 import neo4j
 
 from cartography.client.core.tx import load
@@ -25,37 +24,25 @@ def get_cloudtrail_trails(
     client = boto3_session.client(
         "cloudtrail", region_name=region, config=get_botocore_config()
     )
-    paginator = client.get_paginator("list_trails")
-    trails = []
-    for page in paginator.paginate():
-        trails.extend(page["Trails"])
 
-    # CloudTrail multi-region trails are shown in list_trails,
-    # but the get_trail call only works in the home region
+    trails = client.describe_trails()["trailList"]
     trails_filtered = [trail for trail in trails if trail.get("HomeRegion") == region]
+
     return trails_filtered
 
 
-@timeit
-def get_cloudtrail_trail(
-    boto3_session: boto3.Session,
-    region: str,
-    trail_name: str,
-) -> Dict[str, Any]:
-    client = boto3_session.client(
-        "cloudtrail", region_name=region, config=get_botocore_config()
-    )
-    trail_details: Dict[str, Any] = {}
-    try:
-        response = client.get_trail(Name=trail_name)
-        trail_details = response["Trail"]
-    except botocore.exceptions.ClientError as e:
-        code = e.response["Error"]["Code"]
-        msg = e.response["Error"]["Message"]
-        logger.warning(
-            f"Could not run CloudTrail get_trail due to boto3 error {code}: {msg}. Skipping.",
-        )
-    return trail_details
+def transform_cloudtrail_trails(
+    trails: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudTrail trail data for ingestion
+    """
+    for trail in trails:
+        arn = trail.get("CloudWatchLogsLogGroupArn")
+        if arn:
+            trail["CloudWatchLogsLogGroupArn"] = arn.split(":*")[0]
+
+    return trails
 
 
 @timeit
@@ -104,21 +91,12 @@ def sync(
         logger.info(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
-        trails = get_cloudtrail_trails(boto3_session, region)
-        trail_data: List[Dict[str, Any]] = []
-        for trail in trails:
-            trail_name = trail["Name"]
-            trail_details = get_cloudtrail_trail(
-                boto3_session,
-                region,
-                trail_name,
-            )
-            if trail_details:
-                trail_data.append(trail_details)
+        trails_filtered = get_cloudtrail_trails(boto3_session, region)
+        trails = transform_cloudtrail_trails(trails_filtered, region)
 
         load_cloudtrail_trails(
             neo4j_session,
-            trail_data,
+            trails,
             region,
             current_aws_account_id,
             update_tag,