cartography 0.112.0__tar.gz → 0.113.0__tar.gz

{cartography-0.112.0 → cartography-0.113.0}/.github/workflows/ossf-scorecard.yml

@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@2d92b76c45b91eb80fc44c74ce3fce0ee94e8f9d # v3.29.5
+        uses: github/codeql-action/upload-sarif@f1f6e5f6af878fb37288ce1c627459e94dbf7d01 # v3.29.5
         with:
           sarif_file: results.sarif

{cartography-0.112.0 → cartography-0.113.0}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: "3.10"
       - name: Install uv

{cartography-0.112.0 → cartography-0.113.0}/.github/workflows/sphinx.yml

@@ -27,7 +27,7 @@ jobs:
       # https://github.com/marketplace/actions/setup-python
       # ^-- This gives info on matrix testing.
       - name: Install Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: "3.10"
       - name: Install uv

{cartography-0.112.0 → cartography-0.113.0}/.github/workflows/test_suite.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: "3.10"
       - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
@@ -36,7 +36,7 @@ jobs:
             --publish 7473:7473 \
             --publish 7687:7687 \
             neo4j:4.4-community
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: "3.10"
       - name: Install uv
@@ -68,7 +68,7 @@ jobs:
             --publish 7473:7473 \
             --publish 7687:7687 \
             neo4j:5
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: "3.10"
       - name: Install uv

{cartography-0.112.0 → cartography-0.113.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.112.0
+Version: 0.113.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License-Expression: Apache-2.0

{cartography-0.112.0 → cartography-0.113.0}/cartography/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.112.0'
-__version_tuple__ = version_tuple = (0, 112, 0)
+__version__ = version = '0.113.0'
+__version_tuple__ = version_tuple = (0, 113, 0)
 
-__commit_id__ = commit_id = 'gc784aa30d'
+__commit_id__ = commit_id = 'g087e2f612'

{cartography-0.112.0 → cartography-0.113.0}/cartography/data/indexes.cypher

@@ -100,10 +100,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GCPDNSZone) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GCPDNSZone) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GCPRecordSet) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GCPRecordSet) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPFolder) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPFolder) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPForwardingRule) ON (n.id);

cartography-0.113.0/cartography/intel/aws/apigatewayv2.py

@@ -0,0 +1,116 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.apigatewayv2.apigatewayv2 import APIGatewayV2APISchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_apigatewayv2_apis(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict[str, Any]]:
+    client = boto3_session.client("apigatewayv2", region_name=region)
+    paginator = client.get_paginator("get_apis")
+    apis: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        apis.extend(page.get("Items", []))
+    return apis
+
+
+def transform_apigatewayv2_apis(apis: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    transformed: list[dict[str, Any]] = []
+    for api in apis:
+        transformed.append(
+            {
+                "id": api.get("ApiId"),
+                "name": api.get("Name"),
+                "protocoltype": api.get("ProtocolType"),
+                "routeselectionexpression": api.get("RouteSelectionExpression"),
+                "apikeyselectionexpression": api.get("ApiKeySelectionExpression"),
+                "apiendpoint": api.get("ApiEndpoint"),
+                "version": api.get("Version"),
+                "createddate": api.get("CreatedDate"),
+                "description": api.get("Description"),
+            },
+        )
+    return transformed
+
+
+@timeit
+def load_apigatewayv2_apis(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        APIGatewayV2APISchema(),
+        data,
+        lastupdated=update_tag,
+        AWS_ID=current_aws_account_id,
+        region=region,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        APIGatewayV2APISchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_apigatewayv2_apis(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    apis = get_apigatewayv2_apis(boto3_session, region)
+    transformed = transform_apigatewayv2_apis(apis)
+    load_apigatewayv2_apis(
+        neo4j_session,
+        transformed,
+        region,
+        current_aws_account_id,
+        aws_update_tag,
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing AWS APIGatewayV2 APIs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        sync_apigatewayv2_apis(
+            neo4j_session,
+            boto3_session,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.112.0 → cartography-0.113.0}/cartography/intel/aws/resources.py

@@ -5,6 +5,7 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
 from . import acm
 from . import apigateway
+from . import apigatewayv2
 from . import cloudtrail
 from . import cloudtrail_management_events
 from . import cloudwatch
@@ -101,6 +102,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "permission_relationships": permission_relationships.sync,
     "resourcegroupstaggingapi": resourcegroupstaggingapi.sync,
     "apigateway": apigateway.sync,
+    "apigatewayv2": apigatewayv2.sync,
     "secretsmanager": secretsmanager.sync,
     "securityhub": securityhub.sync,
     "s3accountpublicaccessblock": s3accountpublicaccessblock.sync,

{cartography-0.112.0 → cartography-0.113.0}/cartography/intel/gcp/__init__.py

@@ -480,9 +480,13 @@ def get_gcp_credentials() -> Optional[GoogleCredentials]:
     :return: GoogleCredentials
     """
     try:
-        # Explicitly use Application Default Credentials.
-        # See https://google-auth.readthedocs.io/en/master/user-guide.html#application-default-credentials
-        credentials, project_id = default()
+        # Explicitly use Application Default Credentials with the cloud-platform scope.
+        # Some versions of google-auth/google-api-python-client require explicit scopes for service accounts;
+        # without this, token refresh may fail with `invalid_scope`.
+        # See: https://cloud.google.com/docs/authentication#calling
+        credentials, project_id = default(
+            scopes=["https://www.googleapis.com/auth/cloud-platform"],
+        )
         return credentials
     except DefaultCredentialsError as e:
         logger.debug(

cartography-0.113.0/cartography/intel/gcp/dns.py

@@ -0,0 +1,195 @@
+import json
+import logging
+from typing import Dict
+from typing import List
+
+import neo4j
+from googleapiclient.discovery import HttpError
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.dns import GCPDNSZoneSchema
+from cartography.models.gcp.dns import GCPRecordSetSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_dns_zones(dns: Resource, project_id: str) -> List[Dict]:
+    """Returns a list of DNS zones within the given project."""
+    try:
+        zones: List[Dict] = []
+        request = dns.managedZones().list(project=project_id)
+        while request is not None:
+            response = request.execute()
+            for managed_zone in response["managedZones"]:
+                zones.append(managed_zone)
+            request = dns.managedZones().list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        return zones
+    except HttpError as e:
+        err = json.loads(e.content.decode("utf-8"))["error"]
+        if (
+            err.get("status", "") == "PERMISSION_DENIED"
+            or err.get("message", "") == "Forbidden"
+        ):
+            logger.warning(
+                (
+                    "Could not retrieve DNS zones on project %s due to permissions issues. "
+                    "Code: %s, Message: %s"
+                ),
+                project_id,
+                err["code"],
+                err["message"],
+            )
+            return []
+        raise
+
+
+@timeit
+def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[Dict]:
+    """Returns a list of DNS Resource Record Sets within the given project."""
+    try:
+        rrs: List[Dict] = []
+        for zone in dns_zones:
+            request = dns.resourceRecordSets().list(
+                project=project_id,
+                managedZone=zone["id"],
+            )
+            while request is not None:
+                response = request.execute()
+                for resource_record_set in response["rrsets"]:
+                    resource_record_set["zone"] = zone["id"]
+                    rrs.append(resource_record_set)
+                request = dns.resourceRecordSets().list_next(
+                    previous_request=request,
+                    previous_response=response,
+                )
+        return rrs
+    except HttpError as e:
+        err = json.loads(e.content.decode("utf-8"))["error"]
+        if (
+            err.get("status", "") == "PERMISSION_DENIED"
+            or err.get("message", "") == "Forbidden"
+        ):
+            logger.warning(
+                (
+                    "Could not retrieve DNS RRS on project %s due to permissions issues. "
+                    "Code: %s, Message: %s"
+                ),
+                project_id,
+                err["code"],
+                err["message"],
+            )
+            return []
+        raise
+
+
+@timeit
+def transform_dns_zones(dns_zones: List[Dict]) -> List[Dict]:
+    """Transform raw DNS zone responses into Neo4j-ready dicts."""
+    zones: List[Dict] = []
+    for z in dns_zones:
+        zones.append(
+            {
+                "id": z["id"],
+                "name": z.get("name"),
+                "dns_name": z.get("dnsName"),
+                "description": z.get("description"),
+                "visibility": z.get("visibility"),
+                "kind": z.get("kind"),
+                "nameservers": z.get("nameServers"),
+                "created_at": z.get("creationTime"),
+            }
+        )
+    return zones
+
+
+@timeit
+def transform_dns_rrs(dns_rrs: List[Dict]) -> List[Dict]:
+    """Transform raw DNS record set responses into Neo4j-ready dicts."""
+    records: List[Dict] = []
+    for r in dns_rrs:
+        records.append(
+            {
+                "id": r["name"],
+                "name": r["name"],
+                "type": r.get("type"),
+                "ttl": r.get("ttl"),
+                "data": r.get("rrdatas"),
+                "zone_id": r.get("zone"),
+            }
+        )
+    return records
+
+
+@timeit
+def load_dns_zones(
+    neo4j_session: neo4j.Session,
+    dns_zones: List[Dict],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """Ingest GCP DNS Zones into Neo4j."""
+    load(
+        neo4j_session,
+        GCPDNSZoneSchema(),
+        dns_zones,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def load_rrs(
+    neo4j_session: neo4j.Session,
+    dns_rrs: List[Dict],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """Ingest GCP DNS Resource Record Sets into Neo4j."""
+    load(
+        neo4j_session,
+        GCPRecordSetSchema(),
+        dns_rrs,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_dns_records(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
+    """Delete out-of-date GCP DNS Zones and Record Sets nodes and relationships."""
+    # Record sets depend on zones, so clean them up first.
+    GraphJob.from_node_schema(GCPRecordSetSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(GCPDNSZoneSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    dns: Resource,
+    project_id: str,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """Get GCP DNS Zones and Record Sets, load them into Neo4j, and clean up old data."""
+    logger.info("Syncing DNS records for project %s.", project_id)
+    dns_zones_resp = get_dns_zones(dns, project_id)
+    dns_zones = transform_dns_zones(dns_zones_resp)
+    load_dns_zones(neo4j_session, dns_zones, project_id, gcp_update_tag)
+    dns_rrs_resp = get_dns_rrs(dns, dns_zones_resp, project_id)
+    dns_rrs = transform_dns_rrs(dns_rrs_resp)
+    load_rrs(neo4j_session, dns_rrs, project_id, gcp_update_tag)
+    cleanup_dns_records(neo4j_session, common_job_parameters)

{cartography-0.112.0 → cartography-0.113.0}/cartography/intel/gcp/iam.py

@@ -90,6 +90,29 @@ def get_gcp_roles(iam_client: Resource, project_id: str) -> List[Dict]:
         return []
 
 
+def transform_gcp_service_accounts(
+    raw_accounts: List[Dict[str, Any]],
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform raw GCP service accounts into loader-friendly dicts.
+    """
+    result: List[Dict[str, Any]] = []
+    for sa in raw_accounts:
+        result.append(
+            {
+                "id": sa["uniqueId"],
+                "email": sa.get("email"),
+                "displayName": sa.get("displayName"),
+                "oauth2ClientId": sa.get("oauth2ClientId"),
+                "uniqueId": sa.get("uniqueId"),
+                "disabled": sa.get("disabled", False),
+                "projectId": project_id,
+            },
+        )
+    return result
+
+
 @timeit
 def load_gcp_service_accounts(
     neo4j_session: neo4j.Session,
@@ -99,38 +122,55 @@ def load_gcp_service_accounts(
 ) -> None:
     """
     Load GCP service account data into Neo4j.
-
-    :param neo4j_session: The Neo4j session.
-    :param service_accounts: A list of service account data to load.
-    :param project_id: The GCP Project ID associated with the service accounts.
-    :param gcp_update_tag: The timestamp of the current sync run.
     """
     logger.debug(
         f"Loading {len(service_accounts)} service accounts for project {project_id}"
     )
-    transformed_service_accounts = []
-    for sa in service_accounts:
-        transformed_sa = {
-            "id": sa["uniqueId"],
-            "email": sa.get("email"),
-            "displayName": sa.get("displayName"),
-            "oauth2ClientId": sa.get("oauth2ClientId"),
-            "uniqueId": sa.get("uniqueId"),
-            "disabled": sa.get("disabled", False),
-            "projectId": project_id,
-        }
-        transformed_service_accounts.append(transformed_sa)
 
     load(
         neo4j_session,
         GCPServiceAccountSchema(),
-        transformed_service_accounts,
+        service_accounts,
         lastupdated=gcp_update_tag,
         projectId=project_id,
-        additional_labels=["GCPPrincipal"],
     )
 
 
+def transform_gcp_roles(
+    raw_roles: List[Dict[str, Any]],
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform raw GCP roles into loader-friendly dicts.
+    """
+    result: List[Dict[str, Any]] = []
+    for role in raw_roles:
+        role_name = role["name"]
+        if role_name.startswith("roles/"):
+            role_type = (
+                "BASIC"
+                if role_name in ["roles/owner", "roles/editor", "roles/viewer"]
+                else "PREDEFINED"
+            )
+        else:
+            role_type = "CUSTOM"
+
+        result.append(
+            {
+                "id": role_name,
+                "name": role_name,
+                "title": role.get("title"),
+                "description": role.get("description"),
+                "deleted": role.get("deleted", False),
+                "etag": role.get("etag"),
+                "includedPermissions": role.get("includedPermissions", []),
+                "roleType": role_type,
+                "projectId": project_id,
+            },
+        )
+    return result
+
+
 @timeit
 def load_gcp_roles(
     neo4j_session: neo4j.Session,
@@ -140,41 +180,13 @@ def load_gcp_roles(
 ) -> None:
     """
     Load GCP role data into Neo4j.
-
-    :param neo4j_session: The Neo4j session.
-    :param roles: A list of role data to load.
-    :param project_id: The GCP Project ID associated with the roles.
-    :param gcp_update_tag: The timestamp of the current sync run.
     """
     logger.debug(f"Loading {len(roles)} roles for project {project_id}")
-    transformed_roles = []
-    for role in roles:
-        role_name = role["name"]
-        if role_name.startswith("roles/"):
-            if role_name in ["roles/owner", "roles/editor", "roles/viewer"]:
-                role_type = "BASIC"
-            else:
-                role_type = "PREDEFINED"
-        else:
-            role_type = "CUSTOM"
-
-        transformed_role = {
-            "id": role_name,
-            "name": role_name,
-            "title": role.get("title"),
-            "description": role.get("description"),
-            "deleted": role.get("deleted", False),
-            "etag": role.get("etag"),
-            "includedPermissions": role.get("includedPermissions", []),
-            "roleType": role_type,
-            "projectId": project_id,
-        }
-        transformed_roles.append(transformed_role)
 
     load(
         neo4j_session,
         GCPRoleSchema(),
-        transformed_roles,
+        roles,
         lastupdated=gcp_update_tag,
         projectId=project_id,
     )
@@ -224,18 +236,18 @@ def sync(
     """
     logger.info(f"Syncing GCP IAM for project {project_id}")
 
-    # Get and load service accounts
-    service_accounts = get_gcp_service_accounts(iam_client, project_id)
+    service_accounts_raw = get_gcp_service_accounts(iam_client, project_id)
     logger.info(
-        f"Found {len(service_accounts)} service accounts in project {project_id}"
+        f"Found {len(service_accounts_raw)} service accounts in project {project_id}"
     )
+    service_accounts = transform_gcp_service_accounts(service_accounts_raw, project_id)
     load_gcp_service_accounts(
         neo4j_session, service_accounts, project_id, gcp_update_tag
     )
 
-    # Get and load roles
-    roles = get_gcp_roles(iam_client, project_id)
-    logger.info(f"Found {len(roles)} roles in project {project_id}")
+    roles_raw = get_gcp_roles(iam_client, project_id)
+    logger.info(f"Found {len(roles_raw)} roles in project {project_id}")
+    roles = transform_gcp_roles(roles_raw, project_id)
     load_gcp_roles(neo4j_session, roles, project_id, gcp_update_tag)
 
     # Run cleanup