cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gcp/policy_bindings.py

@@ -0,0 +1,225 @@
+import hashlib
+import logging
+from typing import Any
+
+import neo4j
+from google.api_core.exceptions import PermissionDenied
+from google.cloud.asset_v1 import AssetServiceClient
+from google.cloud.asset_v1.types import BatchGetEffectiveIamPoliciesRequest
+from google.cloud.asset_v1.types import SearchAllIamPoliciesRequest
+from google.protobuf.json_format import MessageToDict
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.policy_bindings import GCPPolicyBindingSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_policy_bindings(
+    project_id: str,
+    common_job_parameters: dict[str, Any],
+    client: AssetServiceClient,
+) -> dict[str, Any]:
+    org_id = common_job_parameters.get("ORG_RESOURCE_NAME")
+    project_resource_name = (
+        f"//cloudresourcemanager.googleapis.com/projects/{project_id}"
+    )
+
+    policies = []
+
+    # Fetch effective policies for project resource (using org scope for inheritance)
+    effective_scope = org_id
+    response = client.batch_get_effective_iam_policies(
+        request=BatchGetEffectiveIamPoliciesRequest(
+            scope=effective_scope, names=[project_resource_name]
+        )
+    )
+    effective_dict = MessageToDict(response._pb, preserving_proto_field_name=True)
+
+    policies.extend(
+        effective_dict["policy_results"]
+    )  # Fail Loudly if policy_results is not present
+
+    # Fetch direct policy bindings for all child resources using search_all_iam_policies (project scope - no inheritance)
+    search_request = SearchAllIamPoliciesRequest(
+        scope=f"projects/{project_id}",
+        asset_types=[],
+    )
+    for policy in client.search_all_iam_policies(request=search_request):
+        policy_dict = MessageToDict(policy._pb, preserving_proto_field_name=True)
+        # Filter out project resource itself (we already have effective policies for it)
+        resource = policy_dict.get("resource", "")
+        if resource != project_resource_name:
+            policy_data = policy_dict.get("policy", {})
+            bindings = policy_data.get("bindings", [])
+
+            policies.append(
+                {
+                    "full_resource_name": resource,
+                    "policies": [
+                        {
+                            "attached_resource": resource,
+                            "policy": {"bindings": bindings},
+                        }
+                    ],
+                }
+            )
+
+    return {
+        "project_id": project_id,
+        "organization": org_id,
+        "policy_results": policies,
+    }
+
+
+def transform_bindings(data: dict[str, Any]) -> list[dict[str, Any]]:
+    project_id = data["project_id"]
+    bindings: dict[tuple[str, str, str | None], dict[str, Any]] = {}
+
+    for policy_result in data["policy_results"]:
+        for policy in policy_result.get("policies", []):
+            resource = policy.get("attached_resource", "")
+
+            # Determine resource type
+            if "/organizations/" in resource:
+                resource_type = "organization"
+            elif "/folders/" in resource:
+                resource_type = "folder"
+            elif f"/projects/{project_id}" in resource and resource.endswith(
+                f"/projects/{project_id}"
+            ):
+                resource_type = "project"
+            else:
+                resource_type = "resource"
+
+            for binding in policy.get("policy", {}).get("bindings", []):
+                role = binding.get("role")
+                members = binding.get("members", [])
+                condition = binding.get("condition")
+
+                if not role or not members:
+                    continue
+
+                # Filter members to only user:, serviceAccount:, and group: types
+                # Extract email part from each member (format: "type:email@example.com")
+                filtered_members = []
+                for member in members:
+                    if ":" not in member:
+                        continue
+                    member_type, identifier = member.split(":", 1)
+                    if member_type in ("user", "serviceAccount", "group"):
+                        # Store only the email part
+                        filtered_members.append(identifier)
+
+                # Don't process if members(principals) are not from the supported types. For example -> allUsers:, allAuthenticatedUsers, etc.
+                if not filtered_members:
+                    continue
+
+                # Extract condition expression for deduplication key
+                # Include condition expression in key so conditional bindings stay distinct
+                condition_expression = (
+                    condition.get("expression") if condition else None
+                )
+
+                # Deduplicate bindings by (resource, role, condition_expression)
+                # This ensures conditional bindings with different expressions are kept separate
+                key = (resource, role, condition_expression)
+
+                if key in bindings:
+                    existing_members = set(bindings[key]["members"])
+                    existing_members.update(filtered_members)
+                    bindings[key]["members"] = list(existing_members)
+                else:
+                    # Generate unique ID that includes condition expression hash
+                    condition_hash = ""
+                    if condition_expression:
+                        condition_hash = hashlib.sha256(
+                            condition_expression.encode("utf-8")
+                        ).hexdigest()[
+                            :8
+                        ]  # Use first 8 chars of hash for brevity
+
+                    binding_id = f"{resource}_{role}"
+                    if condition_hash:
+                        binding_id = f"{binding_id}_{condition_hash}"
+
+                    bindings[key] = {
+                        "id": binding_id,
+                        "role": role,
+                        "resource": resource,
+                        "resource_type": resource_type,
+                        "members": sorted(filtered_members),
+                        "has_condition": condition is not None,
+                        "condition_title": (
+                            condition.get("title") if condition else None
+                        ),
+                        "condition_expression": condition_expression,
+                    }
+
+    return list(bindings.values())
+
+
+@timeit
+def load_bindings(
+    neo4j_session: neo4j.Session,
+    bindings: list[dict[str, Any]],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPPolicyBindingSchema(),
+        bindings,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    logger.debug("Running GCP policy bindings cleanup job")
+
+    GraphJob.from_node_schema(
+        GCPPolicyBindingSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+    client: AssetServiceClient,
+) -> bool:
+    """
+    Sync GCP IAM policy bindings for a project.
+
+    Returns True if sync was successful, False if skipped due to permissions.
+    """
+    try:
+        bindings_data = get_policy_bindings(
+            project_id, common_job_parameters=common_job_parameters, client=client
+        )  # Why pass common_job_parameters here? Because we need to get the org_id for getting inherited policies.
+    except PermissionDenied as e:
+        logger.warning(
+            "Permission denied when fetching policy bindings for project %s. "
+            "Skipping policy bindings sync. To enable this feature, grant "
+            "roles/cloudasset.viewer at the organization level. Error: %s",
+            project_id,
+            e,
+        )
+        return False
+
+    transformed_bindings_data = transform_bindings(bindings_data)
+
+    load_bindings(neo4j_session, transformed_bindings_data, project_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+    return True

cartography/intel/gcp/storage.py

@@ -1,13 +1,17 @@
 import logging
 from typing import Dict
 from typing import List
+from typing import Tuple
 
 import neo4j
 from googleapiclient.discovery import HttpError
 from googleapiclient.discovery import Resource
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.gcp import compute
-from cartography.util import run_cleanup_job
+from cartography.models.gcp.storage.bucket import GCPBucketLabelSchema
+from cartography.models.gcp.storage.bucket import GCPBucketSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -58,165 +62,85 @@ def get_gcp_buckets(storage: Resource, project_id: str) -> Dict:
 
 
 @timeit
-def transform_gcp_buckets(bucket_res: Dict) -> List[Dict]:
+def transform_gcp_buckets_and_labels(bucket_res: Dict) -> Tuple[List[Dict], List[Dict]]:
     """
-    Transform the GCP Storage Bucket response object for Neo4j ingestion
+    Transform the GCP Storage Bucket response object for Neo4j ingestion.
 
-    :type bucket_res: The GCP storage resource object (https://cloud.google.com/storage/docs/json_api/v1/buckets)
-    :param bucket_res: The return data
-
-    :rtype: list
-    :return: List of buckets ready for ingestion to Neo4j
+    :param bucket_res: The raw GCP bucket response.
+    :return: A tuple of (buckets, bucket_labels) ready for ingestion to Neo4j.
     """
 
-    bucket_list = []
+    buckets: List[Dict] = []
+    labels: List[Dict] = []
     for b in bucket_res.get("items", []):
-        bucket = {}
-        bucket["etag"] = b.get("etag")
-        bucket["iam_config_bucket_policy_only"] = (
-            b.get("iamConfiguration", {})
-            .get("bucketPolicyOnly", {})
-            .get("enabled", None)
-        )
-        bucket["id"] = b["id"]
-        bucket["labels"] = [(key, val) for (key, val) in b.get("labels", {}).items()]
-        bucket["owner_entity"] = b.get("owner", {}).get("entity")
-        bucket["owner_entity_id"] = b.get("owner", {}).get("entityId")
-        bucket["kind"] = b.get("kind")
-        bucket["location"] = b.get("location")
-        bucket["location_type"] = b.get("locationType")
-        bucket["meta_generation"] = b.get("metageneration", None)
-        bucket["project_number"] = b["projectNumber"]
-        bucket["self_link"] = b.get("selfLink")
-        bucket["storage_class"] = b.get("storageClass")
-        bucket["time_created"] = b.get("timeCreated")
-        bucket["updated"] = b.get("updated")
-        bucket["versioning_enabled"] = b.get("versioning", {}).get("enabled", None)
-        bucket["default_event_based_hold"] = b.get("defaultEventBasedHold", None)
-        bucket["retention_period"] = b.get("retentionPolicy", {}).get(
-            "retentionPeriod",
-            None,
-        )
-        bucket["default_kms_key_name"] = b.get("encryption", {}).get(
-            "defaultKmsKeyName",
-        )
-        bucket["log_bucket"] = b.get("logging", {}).get("logBucket")
-        bucket["requester_pays"] = b.get("billing", {}).get("requesterPays", None)
-        bucket_list.append(bucket)
-    return bucket_list
+        bucket = {
+            "iam_config_bucket_policy_only": (
+                b.get("iamConfiguration", {}).get("bucketPolicyOnly", {}).get("enabled")
+            ),
+            "id": b["id"],
+            # Preserve legacy bucket_id field for compatibility
+            "bucket_id": b["id"],
+            "owner_entity": b.get("owner", {}).get("entity"),
+            "owner_entity_id": b.get("owner", {}).get("entityId"),
+            "kind": b.get("kind"),
+            "location": b.get("location"),
+            "location_type": b.get("locationType"),
+            "meta_generation": b.get("metageneration"),
+            "project_number": b.get("projectNumber"),
+            "self_link": b.get("selfLink"),
+            "storage_class": b.get("storageClass"),
+            "time_created": b.get("timeCreated"),
+            "versioning_enabled": b.get("versioning", {}).get("enabled"),
+            "retention_period": b.get("retentionPolicy", {}).get("retentionPeriod"),
+            "default_kms_key_name": b.get("encryption", {}).get("defaultKmsKeyName"),
+            "log_bucket": b.get("logging", {}).get("logBucket"),
+            "requester_pays": b.get("billing", {}).get("requesterPays"),
+        }
+        buckets.append(bucket)
+        for key, val in b.get("labels", {}).items():
+            labels.append(
+                {
+                    "id": f"GCPBucket_{key}",
+                    "key": key,
+                    "value": val,
+                    "bucket_id": b["id"],
+                }
+            )
+    return buckets, labels
 
 
 @timeit
 def load_gcp_buckets(
     neo4j_session: neo4j.Session,
     buckets: List[Dict],
+    project_id: str,
     gcp_update_tag: int,
 ) -> None:
-    """
-    Ingest GCP Storage Buckets to Neo4j
-
-    :type neo4j_session: Neo4j session object
-    :param neo4j session: The Neo4j session object
-
-    :type buckets: list
-    :param buckets: List of GCP Storage Buckets to injest
-
-    :type gcp_update_tag: timestamp
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
-
-    :rtype: NoneType
-    :return: Nothing
-    """
-
-    query = """
-    MERGE(p:GCPProject{projectnumber:$ProjectNumber})
-    ON CREATE SET p.firstseen = timestamp()
-    SET p.lastupdated = $gcp_update_tag
-
-    MERGE(bucket:GCPBucket{id:$BucketId})
-    ON CREATE SET bucket.firstseen = timestamp(),
-    bucket.bucket_id = $BucketId
-    SET bucket.self_link = $SelfLink,
-    bucket.project_number = $ProjectNumber,
-    bucket.kind = $Kind,
-    bucket.location = $Location,
-    bucket.location_type = $LocationType,
-    bucket.meta_generation = $MetaGeneration,
-    bucket.storage_class = $StorageClass,
-    bucket.time_created = $TimeCreated,
-    bucket.retention_period = $RetentionPeriod,
-    bucket.iam_config_bucket_policy_only = $IamConfigBucketPolicyOnly,
-    bucket.owner_entity = $OwnerEntity,
-    bucket.owner_entity_id = $OwnerEntityId,
-    bucket.lastupdated = $gcp_update_tag,
-    bucket.versioning_enabled = $VersioningEnabled,
-    bucket.log_bucket = $LogBucket,
-    bucket.requester_pays = $RequesterPays,
-    bucket.default_kms_key_name = $DefaultKmsKeyName
-
-    MERGE (p)-[r:RESOURCE]->(bucket)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $gcp_update_tag
-    """
-    for bucket in buckets:
-        neo4j_session.run(
-            query,
-            ProjectNumber=bucket["project_number"],
-            BucketId=bucket["id"],
-            SelfLink=bucket["self_link"],
-            Kind=bucket["kind"],
-            Location=bucket["location"],
-            LocationType=bucket["location_type"],
-            MetaGeneration=bucket["meta_generation"],
-            StorageClass=bucket["storage_class"],
-            TimeCreated=bucket["time_created"],
-            RetentionPeriod=bucket["retention_period"],
-            IamConfigBucketPolicyOnly=bucket["iam_config_bucket_policy_only"],
-            OwnerEntity=bucket["owner_entity"],
-            OwnerEntityId=bucket["owner_entity_id"],
-            VersioningEnabled=bucket["versioning_enabled"],
-            LogBucket=bucket["log_bucket"],
-            RequesterPays=bucket["requester_pays"],
-            DefaultKmsKeyName=bucket["default_kms_key_name"],
-            gcp_update_tag=gcp_update_tag,
-        )
-        _attach_gcp_bucket_labels(neo4j_session, bucket, gcp_update_tag)
+    """Ingest GCP Storage Buckets to Neo4j."""
+    load(
+        neo4j_session,
+        GCPBucketSchema(),
+        buckets,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
 
 @timeit
-def _attach_gcp_bucket_labels(
+def load_gcp_bucket_labels(
     neo4j_session: neo4j.Session,
-    bucket: Resource,
+    bucket_labels: List[Dict],
+    project_id: str,
     gcp_update_tag: int,
 ) -> None:
-    """
-    Attach GCP bucket labels to the bucket.
-    :param neo4j_session: The neo4j session
-    :param bucket: The GCP bucket object
-    :param gcp_update_tag: The update tag for this sync
-    :return: Nothing
-    """
-    query = """
-    MERGE (l:Label:GCPBucketLabel{id: $BucketLabelId})
-    ON CREATE SET l.firstseen = timestamp(),
-    l.key = $Key
-    SET l.value = $Value,
-    l.lastupdated = $gcp_update_tag
-    WITH l
-    MATCH (bucket:GCPBucket{id:$BucketId})
-    MERGE (l)<-[r:LABELED]-(bucket)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $gcp_update_tag
-    """
-    for key, val in bucket.get("labels", []):
-        neo4j_session.run(
-            query,
-            BucketLabelId=f"GCPBucket_{key}",
-            Key=key,
-            Value=val,
-            BucketId=bucket["id"],
-            gcp_update_tag=gcp_update_tag,
-        )
+    """Ingest GCP Storage Bucket labels and attach them to buckets."""
+    load(
+        neo4j_session,
+        GCPBucketLabelSchema(),
+        bucket_labels,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
 
 @timeit
@@ -224,22 +148,14 @@ def cleanup_gcp_buckets(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    """
-    Delete out-of-date GCP Storage Bucket nodes and relationships
-
-    :type neo4j_session: The Neo4j session object
-    :param neo4j_session: The Neo4j session
-
-    :type common_job_parameters: dict
-    :param common_job_parameters: Dictionary of other job parameters to pass to Neo4j
-
-    :rtype: NoneType
-    :return: Nothing
-    """
-    run_cleanup_job(
-        "gcp_storage_bucket_cleanup.json",
+    """Delete out-of-date GCP Storage Bucket nodes and relationships."""
+    # Bucket labels depend on buckets, so we must remove labels first to avoid
+    # dangling references before deleting the buckets themselves.
+    GraphJob.from_node_schema(GCPBucketLabelSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(GCPBucketSchema(), common_job_parameters).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 
@@ -274,7 +190,7 @@ def sync_gcp_buckets(
     """
     logger.info("Syncing Storage objects for project %s.", project_id)
     storage_res = get_gcp_buckets(storage, project_id)
-    bucket_list = transform_gcp_buckets(storage_res)
-    load_gcp_buckets(neo4j_session, bucket_list, gcp_update_tag)
-    # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
+    buckets, bucket_labels = transform_gcp_buckets_and_labels(storage_res)
+    load_gcp_buckets(neo4j_session, buckets, project_id, gcp_update_tag)
+    load_gcp_bucket_labels(neo4j_session, bucket_labels, project_id, gcp_update_tag)
     cleanup_gcp_buckets(neo4j_session, common_job_parameters)

cartography/intel/github/__init__.py

@@ -1,19 +1,45 @@
 import base64
 import json
 import logging
+from typing import cast
 
 import neo4j
-from requests import exceptions
 
+import cartography.intel.github.commits
 import cartography.intel.github.repos
 import cartography.intel.github.teams
 import cartography.intel.github.users
+from cartography.client.core.tx import read_list_of_values_tx
 from cartography.config import Config
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
+def _get_repos_from_graph(neo4j_session: neo4j.Session, organization: str) -> list[str]:
+    """
+    Get repository names for an organization from the graph instead of making an API call.
+
+    :param neo4j_session: Neo4j session for database interface
+    :param organization: GitHub organization name
+    :return: List of repository names
+    """
+    org_url = f"https://github.com/{organization}"
+    query = """
+    MATCH (org:GitHubOrganization {id: $org_url})<-[:OWNER]-(repo:GitHubRepository)
+    RETURN repo.name
+    ORDER BY repo.name
+    """
+    return cast(
+        list[str],
+        neo4j_session.execute_read(
+            read_list_of_values_tx,
+            query,
+            org_url=org_url,
+        ),
+    )
+
+
 @timeit
 def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     """
@@ -34,27 +60,38 @@ def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     }
     # run sync for the provided github tokens
     for auth_data in auth_tokens["organization"]:
-        try:
-            cartography.intel.github.users.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.repos.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.teams.sync_github_teams(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-        except exceptions.RequestException as e:
-            logger.error("Could not complete request to the GitHub API: %s", e)
+        cartography.intel.github.users.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.repos.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.teams.sync_github_teams(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+
+        # Sync commit relationships for the configured lookback period
+        # Get repo names from the graph instead of making another API call
+        repo_names = _get_repos_from_graph(neo4j_session, auth_data["name"])
+
+        cartography.intel.github.commits.sync_github_commits(
+            neo4j_session,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+            repo_names,
+            common_job_parameters["UPDATE_TAG"],
+            config.github_commit_lookback_days,
+        )