cartography 0.110.0rc2__tar.gz → 0.111.0rc1__tar.gz

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/.github/dependabot.yml

@@ -17,6 +17,8 @@ updates:
           - "minor"
           - "patch"
   # Maintain dependencies for Python
+  # TODO: Remove when uv ecosystem supports updating pyproject.toml
+  # https://github.com/dependabot/dependabot-core/issues/12609
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
@@ -33,12 +35,29 @@ updates:
           - "minor"
           - "patch"
     # Loose versioning strategy, increase the version only if necessary
-    # This should be removed if we stick to a pinned version strategy
-    # Ref: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/controlling-dependencies-updated#defining-a-versioning-strategy
+    # As we use uv.lock for build reproducibility, we can be more permissive with version updates
     versioning-strategy: increase-if-necessary
     # TODO: Remove when #857 is fixed
     ignore:
       - dependency-name: "okta"
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "master"
+    commit-message:
+      # Prefix all commit messages with "chore: "
+      prefix: "chore: "
+    groups:
+      # Group all minor and patch updates for Python
+      # Keep individual updates for major updates
+      minor-and-patch:
+        update-types:
+          - "minor"
+          - "patch"
+    # TODO: Remove when #857 is fixed
+    ignore:
+      - dependency-name: "okta"
   # Maintain dependencies for Docker
   - package-ecosystem: "docker"
     directory: "/"

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/.github/workflows/ossf-scorecard.yml

@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@76621b61decf072c1cee8dd1ce2d2a82d33c17ed # v3.29.5
+        uses: github/codeql-action/upload-sarif@96f518a34f7a870018057716cc4d7a5c014bd61c # v3.29.5
         with:
           sarif_file: results.sarif

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -9,22 +9,17 @@ on:
 permissions: {}
 
 jobs:
-  build-and-publish-to-pypi-and-ghcr:
-    # Explicitly grant the `secrets.GITHUB_TOKEN` permissions.
-    permissions:
-      # Grant the ability to write to GitHub Packages (push Docker images to
-      # GitHub Container Registry).
-      packages: write
-    name: Build and publish Python 🐍 distributions 📦 to PyPI, and Docker 🐳 images 📦 to GitHub Container Registry
+  # 1. Publish to PyPI
+  build-and-publish-to-pypi:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
     runs-on: ubuntu-latest
     steps:
-      # 1. Publish to PyPI
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -34,7 +29,27 @@ jobs:
         if: startsWith(github.ref, 'refs/tags')
         run: uv publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
 
-      # 2. Publish to GHCR
+  # 2. Publish to GHCR
+  build-and-publish-to-ghcr:
+    # Explicitly grant the `secrets.GITHUB_TOKEN` permissions.
+    name: Build and publish Docker 🐳 images 📦 to GitHub Container Registry
+    runs-on: ubuntu-latest
+    permissions:
+      # Grant the ability to write to GitHub Packages (push Docker images to
+      # GitHub Container Registry).
+      packages: write
+    steps:
+      # See: https://docs.docker.com/build/ci/github-actions/multi-platform/#build-and-load-multi-platform-images
+      - name: Set up Docker
+        uses: docker/setup-docker-action@b60f85385d03ac8acfca6d9996982511d8620a19 # v4.3.0
+        with:
+          daemon-config: |
+            {
+              "debug": true,
+              "features": {
+                "containerd-snapshotter": true
+              }
+            }
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Extract metadata (tags, labels) for Docker
         id: meta
@@ -42,6 +57,10 @@ jobs:
         with:
           images: ghcr.io/${{ github.repository }}
 
+      # Install QEMU for multi-architecture builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
@@ -75,5 +94,6 @@ jobs:
           push: true # push the image to ghcr
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
           # e.g. '==0.98.0'
           build-args: VERSION_SPECIFIER===${{ env.VERSION }}

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/.github/workflows/sphinx.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -72,7 +72,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/Dockerfile

@@ -1,5 +1,5 @@
 # Base image
-FROM python:3.10-slim@sha256:81f1cdb3770d54ecfdbddcc52c2125fce674c14a1d976dfd8f65dc0734f9c3c5 AS base
+FROM python:3.10-slim@sha256:420fbb0e468d3eaf0f7e93ea6f7a48792cbcadc39d43ac95b96bee2afe4367da AS base
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER
 # the UID and GID to run cartography as

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.110.0rc2
+Version: 0.111.0rc1
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2

cartography-0.111.0rc1/cartography/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.111.0rc1'
+__version_tuple__ = version_tuple = (0, 111, 0, 'rc1')
+
+__commit_id__ = commit_id = 'g21e6f9792'

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/cartography/data/indexes.cypher

@@ -51,8 +51,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSTransitGatewayAttachment) ON (n.lastupdated
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSUser) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSVpc) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.accesskeyid);
 CREATE INDEX IF NOT EXISTS FOR (n:AccountAccessKey) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AutoScalingGroup) ON (n.arn);

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/cartography/graph/querybuilder.py

@@ -1,5 +1,7 @@
 import logging
 from dataclasses import asdict
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
 from string import Template
 from typing import Dict
 from typing import List
@@ -223,6 +225,8 @@ def _build_attach_sub_resource_statement(
         $RelMergeClause
         ON CREATE SET r.firstseen = timestamp()
         SET
+            r._module_name = "$module_name",
+            r._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -244,6 +248,8 @@ def _build_attach_sub_resource_statement(
         SubResourceLabel=sub_resource_link.target_node_label,
         MatchClause=_build_match_clause(sub_resource_link.target_node_matcher),
         RelMergeClause=rel_merge_clause,
+        module_name=_get_module_from_schema(sub_resource_link),
+        module_version=_get_cartography_version(),
         SubResourceRelLabel=sub_resource_link.rel_label,
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
@@ -278,6 +284,8 @@ def _build_attach_additional_links_statement(
         $RelMerge
         ON CREATE SET $rel_var.firstseen = timestamp()
         SET
+            $rel_var._module_name = "$module_name",
+            $rel_var._module_version = "$module_version",
             $set_rel_properties_statement
         """,
     )
@@ -312,6 +320,8 @@ def _build_attach_additional_links_statement(
             node_var=node_var,
             rel_var=rel_var,
             RelMerge=rel_merge,
+            module_name=_get_module_from_schema(link),
+            module_version=_get_cartography_version(),
             set_rel_properties_statement=_build_rel_properties_statement(
                 rel_var,
                 rel_props_as_dict,
@@ -453,6 +463,8 @@ def build_ingestion_query(
             MERGE (i:$node_label{id: $dict_id_field})
             ON CREATE SET i.firstseen = timestamp()
             SET
+                i._module_name = "$module_name",
+                i._module_version = "$module_version",
                 $set_node_properties_statement
             $attach_relationships_statement
         """,
@@ -475,6 +487,8 @@ def build_ingestion_query(
     ingest_query = query_template.safe_substitute(
         node_label=node_schema.label,
         dict_id_field=node_props.id,
+        module_name=_get_module_from_schema(node_schema),
+        module_version=_get_cartography_version(),
         set_node_properties_statement=_build_node_properties_statement(
             node_props_as_dict,
             node_schema.extra_node_labels,
@@ -650,6 +664,8 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
             MERGE $rel
             ON CREATE SET r.firstseen = timestamp()
             SET
+                r._module_name = "$module_name",
+                r._module_version = "$module_version",
                 $set_rel_properties_statement;
         """
     )
@@ -677,8 +693,62 @@ def build_matchlink_query(rel_schema: CartographyRelSchema) -> str:
         source_match=source_match,
         target_match=target_match,
         rel=rel,
+        module_name=_get_module_from_schema(rel_schema),
+        module_version=_get_cartography_version(),
         set_rel_properties_statement=_build_rel_properties_statement(
             "r",
             rel_props_as_dict,
         ),
     )
+
+
+def _get_cartography_version() -> str:
+    """
+    Get the current version of the cartography package.
+
+    This function attempts to retrieve the version of the installed cartography package
+    using importlib.metadata. If the package is not found (typically in development
+    or testing environments), it returns 'dev' as a fallback.
+
+    Returns:
+        The version string of the cartography package, or 'dev' if not found
+    """
+    try:
+        return version("cartography")
+    except PackageNotFoundError:
+        # This can occured if the cartography package is not installed in the environment, typically in development or testing environments.
+        logger.warning("cartography package not found. Returning 'dev' version.")
+        # Fallback to reading the VERSION file if the package is not found
+        return "dev"
+
+
+def _get_module_from_schema(
+    schema,  #: "CartographyNodeSchema" | "CartographyRelSchema",
+) -> str:
+    """
+    Extract the module name from a Cartography schema object.
+
+    This function extracts and formats the module name from a CartographyNodeSchema
+    or CartographyRelSchema object. It expects schemas to be part of the official
+    cartography.models package hierarchy and returns a formatted string indicating
+    the specific cartography module.
+
+    Args:
+        schema: A CartographyNodeSchema or CartographyRelSchema object
+
+    Returns:
+        A formatted module name string in the format 'cartography:<module_name>'
+        or 'unknown:<full_module_path>' if the schema is not from cartography.models
+    """
+    # If the entity schema does not belong to the cartography.models package,
+    # we log a warning and return the full module path.
+    if not schema.__module__.startswith("cartography.models."):
+        logger.warning(
+            "The schema %s does not start with 'cartography.models.'. "
+            "This may indicate that the schema is not part of the official cartography models.",
+            schema.__module__,
+        )
+        return f"unknown:{schema.__module__}"
+    # Otherwise, we return the module path as a string.
+    parts = schema.__module__.split(".")
+    return f"cartography:{parts[2]}"

{cartography-0.110.0rc2 → cartography-0.111.0rc1}/cartography/intel/aws/apigateway.py

@@ -14,12 +14,18 @@ from policyuniverse.policy import Policy
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.models.aws.apigateway import APIGatewayRestAPISchema
-from cartography.models.aws.apigatewaycertificate import (
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.apigateway.apigateway import APIGatewayRestAPISchema
+from cartography.models.aws.apigateway.apigatewaycertificate import (
     APIGatewayClientCertificateSchema,
 )
-from cartography.models.aws.apigatewayresource import APIGatewayResourceSchema
-from cartography.models.aws.apigatewaystage import APIGatewayStageSchema
+from cartography.models.aws.apigateway.apigatewaydeployment import (
+    APIGatewayDeploymentSchema,
+)
+from cartography.models.aws.apigateway.apigatewayresource import (
+    APIGatewayResourceSchema,
+)
+from cartography.models.aws.apigateway.apigatewaystage import APIGatewayStageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -40,6 +46,38 @@ def get_apigateway_rest_apis(
     return apis
 
 
+def get_rest_api_ids(
+    rest_apis: List[Dict],
+) -> List[str]:
+    """
+    Extracts the IDs of the REST APIs from the provided list.
+    """
+    return [api["id"] for api in rest_apis if "id" in api]
+
+
+@timeit
+@aws_handle_regions
+def get_rest_api_deployments(
+    boto3_session: boto3.session.Session,
+    rest_api_ids: List[str],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Retrieves the deployments for each REST API in the provided list.
+    """
+    client = boto3_session.client(
+        "apigateway", region_name=region, config=get_botocore_config()
+    )
+    deployments: List[Dict[str, Any]] = []
+    for api_id in rest_api_ids:
+        paginator = client.get_paginator("get_deployments")
+        for page in paginator.paginate(restApiId=api_id):
+            for deployment in page.get("items", []):
+                deployment["api_id"] = api_id
+                deployments.append(deployment)
+    return deployments
+
+
 @timeit
 @aws_handle_regions
 def get_rest_api_details(
@@ -244,6 +282,25 @@ def transform_rest_api_details(
     return stages, certificates, resources
 
 
+def transform_apigateway_deployments(
+    deployments: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform API Gateway Deployment data for ingestion
+    """
+    transformed_deployments = []
+    for deployment in deployments:
+        transformed_deployment = {
+            "id": f"{deployment['api_id']}/{deployment['id']}",
+            "api_id": deployment["api_id"],
+            "description": deployment.get("description"),
+            "region": region,
+        }
+        transformed_deployments.append(transformed_deployment)
+    return transformed_deployments
+
+
 @timeit
 def load_rest_api_details(
     neo4j_session: neo4j.Session,
@@ -283,6 +340,30 @@ def load_rest_api_details(
     )
 
 
+@timeit
+def load_apigateway_deployments(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load API Gateway Deployment data into neo4j.
+    """
+    logger.info(
+        f"Loading API Gateway {len(data)} deployments for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        APIGatewayDeploymentSchema(),
+        data,
+        region=region,
+        lastupdated=aws_update_tag,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def parse_policy(api_id: str, policy: Policy) -> Optional[Dict[Any, Any]]:
     """
@@ -345,6 +426,12 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     )
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        APIGatewayDeploymentSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync_apigateway_rest_apis(
@@ -375,6 +462,19 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+
+    api_ids = get_rest_api_ids(rest_apis)
+    deployments = get_rest_api_deployments(
+        boto3_session,
+        api_ids,
+        region,
+    )
+
+    transformed_deployments = transform_apigateway_deployments(
+        deployments,
+        region,
+    )
+
     load_apigateway_rest_apis(
         neo4j_session,
         transformed_apis,
@@ -388,6 +488,13 @@ def sync_apigateway_rest_apis(
         current_aws_account_id,
         aws_update_tag,
     )
+    load_apigateway_deployments(
+        neo4j_session,
+        transformed_deployments,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
 
 
 @timeit