cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/apigateway.py

@@ -12,8 +12,25 @@ import neo4j
 from botocore.exceptions import ClientError
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.apigateway.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigateway.apigatewaycertificate import (
+    APIGatewayClientCertificateSchema,
+)
+from cartography.models.aws.apigateway.apigatewaydeployment import (
+    APIGatewayDeploymentSchema,
+)
+from cartography.models.aws.apigateway.apigatewayintegration import (
+    APIGatewayIntegrationSchema,
+)
+from cartography.models.aws.apigateway.apigatewaymethod import APIGatewayMethodSchema
+from cartography.models.aws.apigateway.apigatewayresource import (
+    APIGatewayResourceSchema,
+)
+from cartography.models.aws.apigateway.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -21,62 +38,112 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('apigateway', region_name=region)
-    paginator = client.get_paginator('get_rest_apis')
+def get_apigateway_rest_apis(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict]:
+    client = boto3_session.client("apigateway", region_name=region)
+    paginator = client.get_paginator("get_rest_apis")
     apis: List[Any] = []
     for page in paginator.paginate():
-        apis.extend(page['items'])
+        apis.extend(page["items"])
     return apis
 
 
+def get_rest_api_ids(
+    rest_apis: List[Dict],
+) -> List[str]:
+    """
+    Extracts the IDs of the REST APIs from the provided list.
+    """
+    return [api["id"] for api in rest_apis if "id" in api]
+
+
+@timeit
+@aws_handle_regions
+def get_rest_api_deployments(
+    boto3_session: boto3.session.Session,
+    rest_api_ids: List[str],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Retrieves the deployments for each REST API in the provided list.
+    """
+    client = boto3_session.client(
+        "apigateway", region_name=region, config=get_botocore_config()
+    )
+    deployments: List[Dict[str, Any]] = []
+    for api_id in rest_api_ids:
+        paginator = client.get_paginator("get_deployments")
+        for page in paginator.paginate(restApiId=api_id):
+            for deployment in page.get("items", []):
+                deployment["api_id"] = api_id
+                deployments.append(deployment)
+    return deployments
+
+
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
-        boto3_session: boto3.session.Session, rest_apis: List[Dict], region: str,
-) -> List[Tuple[Any, Any, Any, Any, Any]]:
+    boto3_session: boto3.session.Session,
+    rest_apis: List[Dict],
+    region: str,
+) -> List[Tuple[Any, Any, Any, Any, Any, Any, Any]]:
     """
     Iterates over all API Gateway REST APIs.
     """
-    client = boto3_session.client('apigateway', region_name=region)
+    client = boto3_session.client("apigateway", region_name=region)
     apis = []
     for api in rest_apis:
         stages = get_rest_api_stages(api, client)
         # clientcertificate id is given by the api stage
-        certificate = get_rest_api_client_certificate(stages, client)  # type: ignore
-        resources = get_rest_api_resources(api, client)
+        certificate = get_rest_api_client_certificate(stages, client)
+        resources, methods, integrations = get_rest_api_resources_methods_integrations(
+            api,
+            client,
+        )
         policy = get_rest_api_policy(api, client)
-        apis.append((api['id'], stages, certificate, resources, policy))
+        apis.append(
+            (api["id"], stages, certificate, resources, methods, integrations, policy)
+        )
     return apis
 
 
 @timeit
-def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+@aws_handle_regions
+def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API Stage Resources.
     """
     try:
-        stages = client.get_stages(restApiId=api['id'])
+        stages = client.get_stages(restApiId=api["id"])
     except ClientError as e:
         logger.warning(f'Failed to retrieve Stages for Api Id - {api["id"]} - {e}')
         raise
 
-    return stages['item']
+    return stages["item"]
 
 
 @timeit
-def get_rest_api_client_certificate(stages: Dict, client: botocore.client.BaseClient) -> Optional[Any]:
+def get_rest_api_client_certificate(
+    stages: Dict,
+    client: botocore.client.BaseClient,
+) -> Optional[Any]:
     """
     Gets the current ClientCertificate resource if present, else returns None.
     """
     response = None
     for stage in stages:
-        if 'clientCertificateId' in stage:
+        if "clientCertificateId" in stage:
             try:
-                response = client.get_client_certificate(clientCertificateId=stage['clientCertificateId'])
-                response['stageName'] = stage['stageName']
+                response = client.get_client_certificate(
+                    clientCertificateId=stage["clientCertificateId"],
+                )
+                response["stageName"] = stage["stageName"]
             except ClientError as e:
-                logger.warning(f"Failed to retrive Client Certificate for Stage {stage['stageName']} - {e}")
+                logger.warning(
+                    f"Failed to retrieve Client Certificate for Stage {stage['stageName']} - {e}",
+                )
                 raise
         else:
             return []
@@ -85,245 +152,321 @@ def get_rest_api_client_certificate(stages: Dict, client: botocore.client.BaseCl
 
 
 @timeit
-def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+@aws_handle_regions
+def get_rest_api_resources_methods_integrations(
+    api: Dict, client: botocore.client.BaseClient
+) -> Tuple[List[Any], List[Dict], List[Dict]]:
     """
     Gets the collection of Resource resources.
     """
     resources: List[Any] = []
-    paginator = client.get_paginator('get_resources')
-    response_iterator = paginator.paginate(restApiId=api['id'])
-    for page in response_iterator:
-        resources.extend(page['items'])
+    methods: List[Any] = []
+    integrations: List[Any] = []
 
-    return resources
+    paginator = client.get_paginator("get_resources")
+    response_iterator = paginator.paginate(restApiId=api["id"])
+    for page in response_iterator:
+        page_resources = page["items"]
+        resources.extend(page_resources)
+
+        for resource in page_resources:
+            resource_id = resource["id"]
+            resource_methods = resource.get("resourceMethods", {})
+
+            for http_method, method in resource_methods.items():
+                method["resourceId"] = resource_id
+                method["apiId"] = api["id"]
+                method["httpMethod"] = http_method
+                methods.append(method)
+                try:
+                    integration = client.get_integration(
+                        restApiId=api["id"],
+                        resourceId=resource_id,
+                        httpMethod=http_method,
+                    )
+                except ClientError as e:
+                    error_code = e.response.get("Error", {}).get("Code")
+                    if error_code == "NotFoundException":
+                        logger.warning(
+                            "No integration found for API %s resource %s method %s: %s",
+                            api["id"],
+                            resource_id,
+                            http_method,
+                            e,
+                        )
+                        continue
+                    raise
+                integration["resourceId"] = resource_id
+                integration["apiId"] = api["id"]
+                integration["integrationHttpMethod"] = integration.get("httpMethod")
+                integration["httpMethod"] = http_method
+                integrations.append(integration)
+
+    return resources, methods, integrations
 
 
 @timeit
-def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
+def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     """
     Gets the REST API policy. Returns policy string or None if no policy is present.
     """
-    policy = api['policy'] if 'policy' in api and api['policy'] else None
+    policy = api["policy"] if "policy" in api and api["policy"] else None
     return policy
 
 
-@timeit
-def load_apigateway_rest_apis(
-    neo4j_session: neo4j.Session, rest_apis: List[Dict], region: str, current_aws_account_id: str,
+def transform_apigateway_rest_apis(
+    rest_apis: List[Dict],
+    resource_policies: List[Dict],
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
-) -> None:
+) -> List[Dict]:
+    """
+    Transform API Gateway REST API data for ingestion, including policy analysis
     """
-    Ingest the details of API Gateway REST APIs into neo4j.
-    """
-    ingest_rest_apis = """
-    UNWIND $rest_apis_list AS r
-    MERGE (rest_api:APIGatewayRestAPI{id:r.id})
-    ON CREATE SET rest_api.firstseen = timestamp(),
-    rest_api.createddate = r.createdDate
-    SET rest_api.version = r.version,
-    rest_api.minimumcompressionsize = r.minimumCompressionSize,
-    rest_api.disableexecuteapiendpoint = r.disableExecuteApiEndpoint,
-    rest_api.lastupdated = $aws_update_tag,
-    rest_api.region = $Region
-    WITH rest_api
-    MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (aa)-[r:RESOURCE]->(rest_api)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
-    """
-
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
+    # Create a mapping of api_id to policy data for easier lookup
+    policy_map = {policy["api_id"]: policy for policy in resource_policies}
+
+    transformed_apis = []
     for api in rest_apis:
-        api['createdDate'] = str(api['createdDate']) if 'createdDate' in api else None
-
-    neo4j_session.run(
-        ingest_rest_apis,
-        rest_apis_list=rest_apis,
-        aws_update_tag=aws_update_tag,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-    )
+        policy_data = policy_map.get(api["id"], {})
+        transformed_api = {
+            "id": api["id"],
+            "createdDate": str(api["createdDate"]) if "createdDate" in api else None,
+            "version": api.get("version"),
+            "minimumCompressionSize": api.get("minimumCompressionSize"),
+            "disableExecuteApiEndpoint": api.get("disableExecuteApiEndpoint"),
+            # Set defaults in the transform function
+            "anonymous_access": policy_data.get("internet_accessible", False),
+            "anonymous_actions": policy_data.get("accessible_actions", []),
+            # TODO Issue #1452: clarify internet exposure vs anonymous access
+        }
+        transformed_apis.append(transformed_api)
+
+    return transformed_apis
 
 
 @timeit
-def _load_apigateway_policies(
-        neo4j_session: neo4j.Session, policies: List, update_tag: int,
+def load_apigateway_rest_apis(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     """
-    Ingest API Gateway REST API policy results into neo4j.
+    Ingest API Gateway REST API data into neo4j.
     """
-    ingest_policies = """
-    UNWIND $policies as policy
-    MATCH (r:APIGatewayRestAPI) where r.name = policy.api_id
-    SET r.anonymous_access = (coalesce(r.anonymous_access, false) OR policy.internet_accessible),
-    r.anonymous_actions = coalesce(r.anonymous_actions, []) + policy.accessible_actions,
-    r.lastupdated = $UpdateTag
-    """
-
-    neo4j_session.run(
-        ingest_policies,
-        policies=policies,
-        UpdateTag=update_tag,
+    load(
+        neo4j_session,
+        APIGatewayRestAPISchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
-def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
-    set_defaults = """
-    MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(restApi:APIGatewayRestAPI)
-    where restApi.anonymous_actions IS NULL
-    SET restApi.anonymous_access = false, restApi.anonymous_actions = []
+def transform_apigateway_stages(stages: List[Dict], update_tag: int) -> List[Dict]:
     """
-
-    neo4j_session.run(
-        set_defaults,
-        AWS_ID=aws_account_id,
-    )
+    Transform API Gateway Stage data for ingestion
+    """
+    stage_data = []
+    for stage in stages:
+        stage["createdDate"] = str(stage["createdDate"])
+        stage["arn"] = f"arn:aws:apigateway:::{stage['apiId']}/{stage['stageName']}"
+        stage_data.append(stage)
+    return stage_data
 
 
-@timeit
-def _load_apigateway_stages(
-        neo4j_session: neo4j.Session, stages: List, update_tag: int,
-) -> None:
+def transform_apigateway_certificates(
+    certificates: List[Dict],
+    update_tag: int,
+) -> List[Dict]:
     """
-    Ingest the Stage resource details into neo4j.
-    """
-    ingest_stages = """
-    UNWIND $stages_list AS stage
-    MERGE (s:APIGatewayStage{id: stage.arn})
-    ON CREATE SET s.firstseen = timestamp(), s.stagename = stage.stageName,
-    s.createddate = stage.createdDate
-    SET s.deploymentid = stage.deploymentId,
-    s.clientcertificateid = stage.clientCertificateId,
-    s.cacheclusterenabled = stage.cacheClusterEnabled,
-    s.cacheclusterstatus = stage.cacheClusterStatus,
-    s.tracingenabled = stage.tracingEnabled,
-    s.webaclarn = stage.webAclArn,
-    s.lastupdated = $UpdateTag
-    WITH s, stage
-    MATCH (rest_api:APIGatewayRestAPI{id: stage.apiId})
-    MERGE (rest_api)-[r:ASSOCIATED_WITH]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
-    for stage in stages:
-        stage['createdDate'] = str(stage['createdDate'])
-        stage['arn'] = "arn:aws:apigateway:::" + stage['apiId'] + "/" + stage['stageName']
+    Transform API Gateway Client Certificate data for ingestion
+    """
+    cert_data = []
+    for certificate in certificates:
+        certificate["createdDate"] = str(certificate["createdDate"])
+        certificate["expirationDate"] = str(certificate.get("expirationDate"))
+        certificate["stageArn"] = (
+            f"arn:aws:apigateway:::{certificate['apiId']}/{certificate['stageName']}"
+        )
+        cert_data.append(certificate)
+    return cert_data
+
+
+def transform_rest_api_details(
+    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any, Any, Any]],
+) -> Tuple[List[Dict], List[Dict], List[Dict], List[Dict], List[Dict]]:
+    """
+    Transform Stage, Client Certificate, Resource, Method and Integration data for ingestion
+    """
+    stages: List[Dict] = []
+    certificates: List[Dict] = []
+    resources: List[Dict] = []
+    methods: List[Dict] = []
+    integrations: List[Dict] = []
+
+    for (
+        api_id,
+        stage,
+        certificate,
+        resource,
+        method_list,
+        integration_list,
+        _,
+    ) in stages_certificate_resources:
 
-    neo4j_session.run(
-        ingest_stages,
-        stages_list=stages,
-        UpdateTag=update_tag,
-    )
+        if len(stage) > 0:
+            for s in stage:
+                s["apiId"] = api_id
+                s["createdDate"] = str(s["createdDate"])
+                s["arn"] = f"arn:aws:apigateway:::{api_id}/{s['stageName']}"
+            stages.extend(stage)
+
+        if certificate:
+            certificate["apiId"] = api_id
+            certificate["createdDate"] = str(certificate["createdDate"])
+            certificate["expirationDate"] = str(certificate.get("expirationDate"))
+            certificate["stageArn"] = (
+                f"arn:aws:apigateway:::{api_id}/{certificate['stageName']}"
+            )
+            certificates.append(certificate)
+
+        if len(resource) > 0:
+            for r in resource:
+                r["apiId"] = api_id
+            resources.extend(resource)
+
+        if len(method_list) > 0:
+            for method in method_list:
+                method["id"] = (
+                    f"{method['apiId']}/{method['resourceId']}/{method['httpMethod']}"
+                )
+                method["authorizationType"] = method.get("authorizationType")
+                method["authorizerId"] = method.get("authorizerId")
+                method["requestValidatorId"] = method.get("requestValidatorId")
+                method["operationName"] = method.get("operationName")
+                method["apiKeyRequired"] = method.get("apiKeyRequired", False)
+            methods.extend(method_list)
+
+        if len(integration_list) > 0:
+            for integration in integration_list:
+                if not integration.get("id"):
+                    integration["id"] = (
+                        f"{integration['apiId']}/{integration['resourceId']}/{integration['httpMethod']}"
+                    )
+                integration["type"] = integration.get("type")
+                integration["uri"] = integration.get("uri")
+                integration["connectionType"] = integration.get("connectionType")
+                integration["connectionId"] = integration.get("connectionId")
+                integration["credentials"] = integration.get("credentials")
+            integrations.extend(integration_list)
+
+    return stages, certificates, resources, methods, integrations
+
+
+def transform_apigateway_deployments(
+    deployments: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform API Gateway Deployment data for ingestion
+    """
+    transformed_deployments = []
+    for deployment in deployments:
+        transformed_deployment = {
+            "id": f"{deployment['api_id']}/{deployment['id']}",
+            "api_id": deployment["api_id"],
+            "description": deployment.get("description"),
+            "region": region,
+        }
+        transformed_deployments.append(transformed_deployment)
+    return transformed_deployments
 
 
 @timeit
-def _load_apigateway_certificates(
-        neo4j_session: neo4j.Session, certificates: List, update_tag: int,
+def load_rest_api_details(
+    neo4j_session: neo4j.Session,
+    stages_certificate_resources_methods_integrations: List[
+        Tuple[Any, Any, Any, Any, Any, Any, Any]
+    ],
+    aws_account_id: str,
+    update_tag: int,
 ) -> None:
     """
-    Ingest the API Gateway Client Certificate details into neo4j.
-    """
-    ingest_certificates = """
-    UNWIND $certificates_list as certificate
-    MERGE (c:APIGatewayClientCertificate{id: certificate.clientCertificateId})
-    ON CREATE SET c.firstseen = timestamp(), c.createddate = certificate.createdDate
-    SET c.lastupdated = $UpdateTag, c.expirationdate = certificate.expirationDate
-    WITH c, certificate
-    MATCH (stage:APIGatewayStage{id: certificate.stageArn})
-    MERGE (stage)-[r:HAS_CERTIFICATE]->(c)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
+    Transform and load Stage, Client Certificate, and Resource data
     """
+    stages, certificates, resources, methods, integrations = transform_rest_api_details(
+        stages_certificate_resources_methods_integrations,
+    )
 
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
-    for certificate in certificates:
-        certificate['createdDate'] = str(certificate['createdDate'])
-        certificate['expirationDate'] = str(certificate.get('expirationDate'))
-        certificate['stageArn'] = "arn:aws:apigateway:::" + certificate['apiId'] + "/" + certificate['stageName']
-
-    neo4j_session.run(
-        ingest_certificates,
-        certificates_list=certificates,
-        UpdateTag=update_tag,
+    load(
+        neo4j_session,
+        APIGatewayStageSchema(),
+        stages,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
     )
 
+    load(
+        neo4j_session,
+        APIGatewayClientCertificateSchema(),
+        certificates,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
 
-@timeit
-def _load_apigateway_resources(
-        neo4j_session: neo4j.Session, resources: List, update_tag: int,
-) -> None:
-    """
-    Ingest the API Gateway Resource details into neo4j.
-    """
-    ingest_resources = """
-    UNWIND $resources_list AS res
-    MERGE (s:APIGatewayResource{id: res.id})
-    ON CREATE SET s.firstseen = timestamp()
-    SET s.path = res.path,
-    s.pathpart = res.pathPart,
-    s.parentid = res.parentId,
-    s.lastupdated =$UpdateTag
-    WITH s, res
-    MATCH (rest_api:APIGatewayRestAPI{id: res.apiId})
-    MERGE (rest_api)-[r:RESOURCE]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-
-    neo4j_session.run(
-        ingest_resources,
-        resources_list=resources,
-        UpdateTag=update_tag,
+    load(
+        neo4j_session,
+        APIGatewayResourceSchema(),
+        resources,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+    load(
+        neo4j_session,
+        APIGatewayMethodSchema(),
+        methods,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+    load(
+        neo4j_session,
+        APIGatewayIntegrationSchema(),
+        integrations,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
     )
 
 
 @timeit
-def load_rest_api_details(
-        neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
-        aws_account_id: str, update_tag: int,
+def load_apigateway_deployments(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     """
-    Create dictionaries for Stages, Client certificates, policies and Resource resources
-    so we can import them in a single query
+    Load API Gateway Deployment data into neo4j.
     """
-    stages: List[Dict] = []
-    certificates: List[Dict] = []
-    resources: List[Dict] = []
-    policies: List = []
-    for api_id, stage, certificate, resource, policy in stages_certificate_resources:
-        parsed_policy = parse_policy(api_id, policy)
-        if parsed_policy is not None:
-            policies.append(parsed_policy)
-        if len(stage) > 0:
-            for s in stage:
-                s['apiId'] = api_id
-            stages.extend(stage)
-        if len(resource) > 0:
-            for r in resource:
-                r['apiId'] = api_id
-            resources.extend(resource)
-        if certificate:
-            certificate['apiId'] = api_id
-            certificates.append(certificate)
-
-    # cleanup existing properties
-    run_cleanup_job(
-        'aws_apigateway_details.json',
+    logger.info(
+        f"Loading API Gateway {len(data)} deployments for region '{region}' into graph.",
+    )
+    load(
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': aws_account_id},
+        APIGatewayDeploymentSchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
-    _load_apigateway_policies(neo4j_session, policies, update_tag)
-    _load_apigateway_stages(neo4j_session, stages, update_tag)
-    _load_apigateway_certificates(neo4j_session, certificates, update_tag)
-    _load_apigateway_resources(neo4j_session, resources, update_tag)
-    _set_default_values(neo4j_session, aws_account_id)
-
 
 @timeit
 def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
@@ -345,7 +488,7 @@ def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
             else:
                 return None
         except json.JSONDecodeError:
-            logger.warn(f"failed to decode policy json : {policy}")
+            logger.warning(f"failed to decode policy json : {policy}")
             return None
     else:
         return None
@@ -353,27 +496,149 @@ def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('aws_import_apigateway_cleanup.json', neo4j_session, common_job_parameters)
+    """
+    Delete out-of-date API Gateway resources and relationships.
+    Order matters - clean up certificates, stages, and resources before cleaning up the REST APIs they connect to.
+    """
+    logger.info("Running API Gateway cleanup job.")
+
+    # Clean up certificates first
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayClientCertificateSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    # Then stages
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayStageSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    # Then resources
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayResourceSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    # Finally REST APIs
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayRestAPISchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayDeploymentSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayMethodSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayIntegrationSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
 
 
 @timeit
 def sync_apigateway_rest_apis(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, region: str, current_aws_account_id: str,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     rest_apis = get_apigateway_rest_apis(boto3_session, region)
-    load_apigateway_rest_apis(neo4j_session, rest_apis, region, current_aws_account_id, aws_update_tag)
+    stages_certificate_resources_methods_integrations = get_rest_api_details(
+        boto3_session,
+        rest_apis,
+        region,
+    )
 
-    stages_certificate_resources = get_rest_api_details(boto3_session, rest_apis, region)
-    load_rest_api_details(neo4j_session, stages_certificate_resources, current_aws_account_id, aws_update_tag)
+    # Extract policies and transform the data
+    policies = []
+    for (
+        api_id,
+        _,
+        _,
+        _,
+        _,
+        _,
+        policy,
+    ) in stages_certificate_resources_methods_integrations:
+        parsed_policy = parse_policy(api_id, policy)
+        if parsed_policy is not None:
+            policies.append(parsed_policy)
+
+    transformed_apis = transform_apigateway_rest_apis(
+        rest_apis,
+        policies,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+
+    api_ids = get_rest_api_ids(rest_apis)
+    deployments = get_rest_api_deployments(
+        boto3_session,
+        api_ids,
+        region,
+    )
+
+    transformed_deployments = transform_apigateway_deployments(
+        deployments,
+        region,
+    )
+
+    load_apigateway_rest_apis(
+        neo4j_session,
+        transformed_apis,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+    load_rest_api_details(
+        neo4j_session,
+        stages_certificate_resources_methods_integrations,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+    load_apigateway_deployments(
+        neo4j_session,
+        transformed_deployments,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing AWS APIGateway Rest APIs for region '{region}' in account '{current_aws_account_id}'.")
-        sync_apigateway_rest_apis(neo4j_session, boto3_session, region, current_aws_account_id, update_tag)
+        logger.info(
+            f"Syncing AWS APIGateway Rest APIs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        sync_apigateway_rest_apis(
+            neo4j_session,
+            boto3_session,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)