cartography 0.99.0__tar.gz → 0.100.0__tar.gz

cartography-0.100.0/.devcontainer/devcontainer.json

@@ -0,0 +1,41 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+	"name": "Existing Docker Compose (Extend)",
+
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../docker-compose.yml",
+		"docker-compose.yml"
+	],
+
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "cartography-dev",
+
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}"
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	// "shutdownAction": "none",
+
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

cartography-0.100.0/.devcontainer/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.8'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  cartography-dev:
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    #
+    # build:
+    #   context: .
+    #   dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity

{cartography-0.99.0 → cartography-0.100.0}/.github/workflows/ossf-scorecard.yml

@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -49,7 +49,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: SARIF file
           path: results.sarif
@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8
+        uses: github/codeql-action/upload-sarif@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3.28.13
         with:
           sarif_file: results.sarif

{cartography-0.99.0 → cartography-0.100.0}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -53,15 +53,15 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/${{ github.repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           # This is the user that triggered the Workflow. In this case, it will
@@ -83,7 +83,7 @@ jobs:
           sleep 10
 
       - name: Build and push
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: Dockerfile
           context: .

{cartography-0.99.0 → cartography-0.100.0}/.github/workflows/sphinx.yml

@@ -33,7 +33,7 @@ jobs:
       # https://docs.github.com/en/actions/guides/building-and-testing-python#caching-dependencies
       # ^-- How to set up caching for pip on Ubuntu
       - name: Cache pip
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

{cartography-0.99.0 → cartography-0.100.0}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       # Cache our pip dir for efficiency; see https://medium.com/ai2-blog/python-caching-in-github-actions-e9452698e98d.
-      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: ~/.cache/pip
           key: ${{ hashFiles('setup.py') }}-${{ hashFiles('pyproject.toml') }}
@@ -74,7 +74,7 @@ jobs:
         with:
           python-version: "3.10"
       # Cache our pip dir for efficiency; see https://medium.com/ai2-blog/python-caching-in-github-actions-e9452698e98d.
-      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: ~/.cache/pip
           key: ${{ hashFiles('setup.py') }}-${{ hashFiles('pyproject.toml') }}
@@ -97,15 +97,15 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/${{ github.repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: Dockerfile
           push: false # only build the image, don't push it anywhere

{cartography-0.99.0 → cartography-0.100.0}/.gitignore

@@ -19,3 +19,7 @@ generated
 dist/
 .local
 cartography/_version.py
+uv.lock
+.cursor-server
+.gitconfig
+.python-version

{cartography-0.99.0 → cartography-0.100.0}/Dockerfile

@@ -1,6 +1,6 @@
 # This is a thin distribution of the cartography software.
 # It is published at ghcr.io.
-FROM python:3.10-slim@sha256:66aad90b231f011cb80e1966e03526a7175f0586724981969b23903abac19081
+FROM python:3.10-slim@sha256:06f6d69d229bb55fab83dded514e54eede977e33e92d855ba3f97ce0e3234abc
 
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER
@@ -14,7 +14,7 @@ WORKDIR /var/cartography
 ENV HOME=/var/cartography
 
 # Install cartography at the given version specifier. Can be ''.
-RUN pip install cartography${VERSION_SPECIFIER}
+RUN pip install --no-cache-dir cartography${VERSION_SPECIFIER}
 
 USER ${uid}:${gid}
 

{cartography-0.99.0 → cartography-0.100.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: cartography
-Version: 0.99.0
+Version: 0.100.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2
@@ -59,10 +59,11 @@ Requires-Dist: moto; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pytest>=6.2.4; extra == "dev"
 Requires-Dist: pytest-mock; extra == "dev"
-Requires-Dist: pytest-cov==2.10.0; extra == "dev"
+Requires-Dist: pytest-cov==6.0.0; extra == "dev"
 Requires-Dist: pytest-rerunfailures; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: types-requests<2.32.0.20241017; extra == "dev"
+Requires-Dist: types-requests<2.32.0.20250307; extra == "dev"
+Dynamic: license-file
 
 ![Cartography](docs/root/images/logo-horizontal.png)
 

{cartography-0.99.0 → cartography-0.100.0}/cartography/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.99.0'
-__version_tuple__ = version_tuple = (0, 99, 0)
+__version__ = version = '0.100.0'
+__version_tuple__ = version_tuple = (0, 100, 0)

{cartography-0.99.0 → cartography-0.100.0}/cartography/cli.py

@@ -439,9 +439,10 @@ class CLI:
             '--gsuite-auth-method',
             type=str,
             default='delegated',
-            choices=['delegated', 'oauth'],
+            choices=['delegated', 'oauth', 'default'],
             help=(
-                'The method used by GSuite to authenticate. delegated is the legacy one.'
+                'GSuite authentication method. Can be "delegated" for service account or "oauth" for OAuth. '
+                '"Default" best if using gcloud CLI.'
             ),
         )
         parser.add_argument(

{cartography-0.99.0 → cartography-0.100.0}/cartography/intel/aws/apigateway.py

@@ -12,8 +12,13 @@ import neo4j
 from botocore.exceptions import ClientError
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigatewaycertificate import APIGatewayClientCertificateSchema
+from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
+from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -107,222 +112,146 @@ def get_rest_api_policy(api: Dict, client: botocore.client.BaseClient) -> Any:
     return policy
 
 
-@timeit
-def load_apigateway_rest_apis(
-    neo4j_session: neo4j.Session, rest_apis: List[Dict], region: str, current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    """
-    Ingest the details of API Gateway REST APIs into neo4j.
+def transform_apigateway_rest_apis(
+    rest_apis: List[Dict], resource_policies: List[Dict], region: str, current_aws_account_id: str, aws_update_tag: int,
+) -> List[Dict]:
     """
-    ingest_rest_apis = """
-    UNWIND $rest_apis_list AS r
-    MERGE (rest_api:APIGatewayRestAPI{id:r.id})
-    ON CREATE SET rest_api.firstseen = timestamp(),
-    rest_api.createddate = r.createdDate
-    SET rest_api.version = r.version,
-    rest_api.minimumcompressionsize = r.minimumCompressionSize,
-    rest_api.disableexecuteapiendpoint = r.disableExecuteApiEndpoint,
-    rest_api.lastupdated = $aws_update_tag,
-    rest_api.region = $Region
-    WITH rest_api
-    MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (aa)-[r:RESOURCE]->(rest_api)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
+    Transform API Gateway REST API data for ingestion, including policy analysis
     """
+    # Create a mapping of api_id to policy data for easier lookup
+    policy_map = {
+        policy['api_id']: policy
+        for policy in resource_policies
+    }
 
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
+    transformed_apis = []
     for api in rest_apis:
-        api['createdDate'] = str(api['createdDate']) if 'createdDate' in api else None
-
-    neo4j_session.run(
-        ingest_rest_apis,
-        rest_apis_list=rest_apis,
-        aws_update_tag=aws_update_tag,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-    )
+        policy_data = policy_map.get(api['id'], {})
+        transformed_api = {
+            'id': api['id'],
+            'createdDate': str(api['createdDate']) if 'createdDate' in api else None,
+            'version': api.get('version'),
+            'minimumCompressionSize': api.get('minimumCompressionSize'),
+            'disableExecuteApiEndpoint': api.get('disableExecuteApiEndpoint'),
+            # Set defaults in the transform function
+            'anonymous_access': policy_data.get('internet_accessible', False),
+            'anonymous_actions': policy_data.get('accessible_actions', []),
+            # TODO Issue #1452: clarify internet exposure vs anonymous access
+        }
+        transformed_apis.append(transformed_api)
+
+    return transformed_apis
 
 
 @timeit
-def _load_apigateway_policies(
-        neo4j_session: neo4j.Session, policies: List, update_tag: int,
+def load_apigateway_rest_apis(
+    neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     """
-    Ingest API Gateway REST API policy results into neo4j.
-    """
-    ingest_policies = """
-    UNWIND $policies as policy
-    MATCH (r:APIGatewayRestAPI) where r.name = policy.api_id
-    SET r.anonymous_access = (coalesce(r.anonymous_access, false) OR policy.internet_accessible),
-    r.anonymous_actions = coalesce(r.anonymous_actions, []) + policy.accessible_actions,
-    r.lastupdated = $UpdateTag
+    Ingest API Gateway REST API data into neo4j.
     """
-
-    neo4j_session.run(
-        ingest_policies,
-        policies=policies,
-        UpdateTag=update_tag,
-    )
-
-
-def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
-    set_defaults = """
-    MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(restApi:APIGatewayRestAPI)
-    where restApi.anonymous_actions IS NULL
-    SET restApi.anonymous_access = false, restApi.anonymous_actions = []
-    """
-
-    neo4j_session.run(
-        set_defaults,
-        AWS_ID=aws_account_id,
+    load(
+        neo4j_session,
+        APIGatewayRestAPISchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
-@timeit
-def _load_apigateway_stages(
-        neo4j_session: neo4j.Session, stages: List, update_tag: int,
-) -> None:
+def transform_apigateway_stages(stages: List[Dict], update_tag: int) -> List[Dict]:
     """
-    Ingest the Stage resource details into neo4j.
+    Transform API Gateway Stage data for ingestion
     """
-    ingest_stages = """
-    UNWIND $stages_list AS stage
-    MERGE (s:APIGatewayStage{id: stage.arn})
-    ON CREATE SET s.firstseen = timestamp(), s.stagename = stage.stageName,
-    s.createddate = stage.createdDate
-    SET s.deploymentid = stage.deploymentId,
-    s.clientcertificateid = stage.clientCertificateId,
-    s.cacheclusterenabled = stage.cacheClusterEnabled,
-    s.cacheclusterstatus = stage.cacheClusterStatus,
-    s.tracingenabled = stage.tracingEnabled,
-    s.webaclarn = stage.webAclArn,
-    s.lastupdated = $UpdateTag
-    WITH s, stage
-    MATCH (rest_api:APIGatewayRestAPI{id: stage.apiId})
-    MERGE (rest_api)-[r:ASSOCIATED_WITH]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
+    stage_data = []
     for stage in stages:
         stage['createdDate'] = str(stage['createdDate'])
-        stage['arn'] = "arn:aws:apigateway:::" + stage['apiId'] + "/" + stage['stageName']
-
-    neo4j_session.run(
-        ingest_stages,
-        stages_list=stages,
-        UpdateTag=update_tag,
-    )
+        stage['arn'] = f"arn:aws:apigateway:::{stage['apiId']}/{stage['stageName']}"
+        stage_data.append(stage)
+    return stage_data
 
 
-@timeit
-def _load_apigateway_certificates(
-        neo4j_session: neo4j.Session, certificates: List, update_tag: int,
-) -> None:
-    """
-    Ingest the API Gateway Client Certificate details into neo4j.
+def transform_apigateway_certificates(certificates: List[Dict], update_tag: int) -> List[Dict]:
     """
-    ingest_certificates = """
-    UNWIND $certificates_list as certificate
-    MERGE (c:APIGatewayClientCertificate{id: certificate.clientCertificateId})
-    ON CREATE SET c.firstseen = timestamp(), c.createddate = certificate.createdDate
-    SET c.lastupdated = $UpdateTag, c.expirationdate = certificate.expirationDate
-    WITH c, certificate
-    MATCH (stage:APIGatewayStage{id: certificate.stageArn})
-    MERGE (stage)-[r:HAS_CERTIFICATE]->(c)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
+    Transform API Gateway Client Certificate data for ingestion
     """
-
-    # neo4j does not accept datetime objects and values. This loop is used to convert
-    # these values to string.
+    cert_data = []
     for certificate in certificates:
         certificate['createdDate'] = str(certificate['createdDate'])
         certificate['expirationDate'] = str(certificate.get('expirationDate'))
-        certificate['stageArn'] = "arn:aws:apigateway:::" + certificate['apiId'] + "/" + certificate['stageName']
-
-    neo4j_session.run(
-        ingest_certificates,
-        certificates_list=certificates,
-        UpdateTag=update_tag,
-    )
-
-
-@timeit
-def _load_apigateway_resources(
-        neo4j_session: neo4j.Session, resources: List, update_tag: int,
-) -> None:
-    """
-    Ingest the API Gateway Resource details into neo4j.
-    """
-    ingest_resources = """
-    UNWIND $resources_list AS res
-    MERGE (s:APIGatewayResource{id: res.id})
-    ON CREATE SET s.firstseen = timestamp()
-    SET s.path = res.path,
-    s.pathpart = res.pathPart,
-    s.parentid = res.parentId,
-    s.lastupdated =$UpdateTag
-    WITH s, res
-    MATCH (rest_api:APIGatewayRestAPI{id: res.apiId})
-    MERGE (rest_api)-[r:RESOURCE]->(s)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-
-    neo4j_session.run(
-        ingest_resources,
-        resources_list=resources,
-        UpdateTag=update_tag,
-    )
+        certificate['stageArn'] = f"arn:aws:apigateway:::{certificate['apiId']}/{certificate['stageName']}"
+        cert_data.append(certificate)
+    return cert_data
 
 
-@timeit
-def load_rest_api_details(
-        neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
-        aws_account_id: str, update_tag: int,
-) -> None:
+def transform_rest_api_details(
+    stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
+) -> Tuple[List[Dict], List[Dict], List[Dict]]:
     """
-    Create dictionaries for Stages, Client certificates, policies and Resource resources
-    so we can import them in a single query
+    Transform Stage, Client Certificate, and Resource data for ingestion
     """
     stages: List[Dict] = []
     certificates: List[Dict] = []
     resources: List[Dict] = []
-    policies: List = []
-    for api_id, stage, certificate, resource, policy in stages_certificate_resources:
-        parsed_policy = parse_policy(api_id, policy)
-        if parsed_policy is not None:
-            policies.append(parsed_policy)
+
+    for api_id, stage, certificate, resource, _ in stages_certificate_resources:
         if len(stage) > 0:
             for s in stage:
                 s['apiId'] = api_id
+                s['createdDate'] = str(s['createdDate'])
+                s['arn'] = f"arn:aws:apigateway:::{api_id}/{s['stageName']}"
             stages.extend(stage)
+
+        if certificate:
+            certificate['apiId'] = api_id
+            certificate['createdDate'] = str(certificate['createdDate'])
+            certificate['expirationDate'] = str(certificate.get('expirationDate'))
+            certificate['stageArn'] = f"arn:aws:apigateway:::{api_id}/{certificate['stageName']}"
+            certificates.append(certificate)
+
         if len(resource) > 0:
             for r in resource:
                 r['apiId'] = api_id
             resources.extend(resource)
-        if certificate:
-            certificate['apiId'] = api_id
-            certificates.append(certificate)
 
-    # cleanup existing properties
-    run_cleanup_job(
-        'aws_apigateway_details.json',
+    return stages, certificates, resources
+
+
+@timeit
+def load_rest_api_details(
+    neo4j_session: neo4j.Session, stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
+    aws_account_id: str, update_tag: int,
+) -> None:
+    """
+    Transform and load Stage, Client Certificate, and Resource data
+    """
+    stages, certificates, resources = transform_rest_api_details(stages_certificate_resources)
+
+    load(
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': aws_account_id},
+        APIGatewayStageSchema(),
+        stages,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
     )
 
-    _load_apigateway_policies(neo4j_session, policies, update_tag)
-    _load_apigateway_stages(neo4j_session, stages, update_tag)
-    _load_apigateway_certificates(neo4j_session, certificates, update_tag)
-    _load_apigateway_resources(neo4j_session, resources, update_tag)
-    _set_default_values(neo4j_session, aws_account_id)
+    load(
+        neo4j_session,
+        APIGatewayClientCertificateSchema(),
+        certificates,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+    load(
+        neo4j_session,
+        APIGatewayResourceSchema(),
+        resources,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
 
 
 @timeit
@@ -353,7 +282,27 @@ def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('aws_import_apigateway_cleanup.json', neo4j_session, common_job_parameters)
+    """
+    Delete out-of-date API Gateway resources and relationships.
+    Order matters - clean up certificates, stages, and resources before cleaning up the REST APIs they connect to.
+    """
+    logger.info("Running API Gateway cleanup job.")
+
+    # Clean up certificates first
+    cleanup_job = GraphJob.from_node_schema(APIGatewayClientCertificateSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+    # Then stages
+    cleanup_job = GraphJob.from_node_schema(APIGatewayStageSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+    # Then resources
+    cleanup_job = GraphJob.from_node_schema(APIGatewayResourceSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+    # Finally REST APIs
+    cleanup_job = GraphJob.from_node_schema(APIGatewayRestAPISchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
 
 
 @timeit
@@ -362,9 +311,23 @@ def sync_apigateway_rest_apis(
     aws_update_tag: int,
 ) -> None:
     rest_apis = get_apigateway_rest_apis(boto3_session, region)
-    load_apigateway_rest_apis(neo4j_session, rest_apis, region, current_aws_account_id, aws_update_tag)
-
     stages_certificate_resources = get_rest_api_details(boto3_session, rest_apis, region)
+
+    # Extract policies and transform the data
+    policies = []
+    for api_id, _, _, _, policy in stages_certificate_resources:
+        parsed_policy = parse_policy(api_id, policy)
+        if parsed_policy is not None:
+            policies.append(parsed_policy)
+
+    transformed_apis = transform_apigateway_rest_apis(
+        rest_apis,
+        policies,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+    load_apigateway_rest_apis(neo4j_session, transformed_apis, region, current_aws_account_id, aws_update_tag)
     load_rest_api_details(neo4j_session, stages_certificate_resources, current_aws_account_id, aws_update_tag)