cartography 0.109.0rc2__py3-none-any.whl → 0.110.0__py3-none-any.whl

cartography/intel/kubernetes/rbac.py

@@ -0,0 +1,464 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from kubernetes.client import V1ClusterRole
+from kubernetes.client import V1ClusterRoleBinding
+from kubernetes.client import V1Role
+from kubernetes.client import V1RoleBinding
+from kubernetes.client import V1ServiceAccount
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
+from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.clusterrolebindings import (
+    KubernetesClusterRoleBindingSchema,
+)
+from cartography.models.kubernetes.clusterroles import KubernetesClusterRoleSchema
+from cartography.models.kubernetes.rolebindings import KubernetesRoleBindingSchema
+from cartography.models.kubernetes.roles import KubernetesRoleSchema
+from cartography.models.kubernetes.serviceaccounts import KubernetesServiceAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_service_accounts(k8s_client: K8sClient) -> List[V1ServiceAccount]:
+
+    return k8s_paginate(k8s_client.core.list_service_account_for_all_namespaces)
+
+
+@timeit
+def get_roles(k8s_client: K8sClient) -> List[V1Role]:
+
+    return k8s_paginate(k8s_client.rbac.list_role_for_all_namespaces)
+
+
+@timeit
+def get_role_bindings(k8s_client: K8sClient) -> List[V1RoleBinding]:
+
+    return k8s_paginate(k8s_client.rbac.list_role_binding_for_all_namespaces)
+
+
+@timeit
+def get_cluster_roles(k8s_client: K8sClient) -> List[V1ClusterRole]:
+
+    return k8s_paginate(k8s_client.rbac.list_cluster_role)
+
+
+@timeit
+def get_cluster_role_bindings(k8s_client: K8sClient) -> List[V1ClusterRoleBinding]:
+
+    return k8s_paginate(k8s_client.rbac.list_cluster_role_binding)
+
+
+def transform_service_accounts(
+    service_accounts: List[V1ServiceAccount], cluster_name: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Kubernetes ServiceAccounts into a list of dictionaries.
+    Uses cluster-scoped IDs to prevent collisions across multiple clusters.
+    """
+    result = []
+    for sa in service_accounts:
+        result.append(
+            {
+                "id": f"{cluster_name}/{sa.metadata.namespace}/{sa.metadata.name}",
+                "name": sa.metadata.name,
+                "namespace": sa.metadata.namespace,
+                "uid": sa.metadata.uid,
+                "creation_timestamp": get_epoch(sa.metadata.creation_timestamp),
+                "resource_version": sa.metadata.resource_version,
+            }
+        )
+    return result
+
+
+def transform_roles(roles: List[V1Role], cluster_name: str) -> List[Dict[str, Any]]:
+    """
+    Transform Kubernetes Roles into a list of dictionaries.
+    Flattens rules into separate api_groups, resources, and verbs lists.
+    """
+    result = []
+    for role in roles:
+        # Flatten all rules into combined sets
+        all_api_groups: set[str] = set()
+        all_resources: set[str] = set()
+        all_verbs: set[str] = set()
+
+        for rule in role.rules or []:
+            # Update api_groups, handling None and empty string cases
+            all_api_groups.update(
+                {
+                    "core" if api_group == "" else api_group
+                    for api_group in rule.api_groups or []
+                }
+            )
+            all_resources.update(rule.resources or [])
+            all_verbs.update(rule.verbs or [])
+
+        result.append(
+            {
+                "id": f"{cluster_name}/{role.metadata.namespace}/{role.metadata.name}",
+                "name": role.metadata.name,
+                "namespace": role.metadata.namespace,
+                "uid": role.metadata.uid,
+                "creation_timestamp": get_epoch(role.metadata.creation_timestamp),
+                "resource_version": role.metadata.resource_version,
+                "api_groups": sorted(
+                    all_api_groups
+                ),  # sorts to keep consistent ordering and converts to list to appease neo4j
+                "resources": sorted(all_resources),
+                "verbs": sorted(all_verbs),
+            }
+        )
+    return result
+
+
+def transform_role_bindings(
+    role_bindings: List[V1RoleBinding], cluster_name: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Kubernetes RoleBindings into a list of dictionaries.
+    Creates one RoleBinding node per Kubernetes RoleBinding with lists of subject IDs.
+    """
+    result = []
+    for rb in role_bindings:
+        # Collect all subjects by type
+        service_account_subjects = [
+            subject
+            for subject in (rb.subjects or [])
+            if subject.kind == "ServiceAccount"
+        ]
+        user_subjects = [
+            subject for subject in (rb.subjects or []) if subject.kind == "User"
+        ]
+        group_subjects = [
+            subject for subject in (rb.subjects or []) if subject.kind == "Group"
+        ]
+
+        # Only create a RoleBinding node if it has at least one subject
+        if rb.subjects:
+            result.append(
+                {
+                    "id": f"{cluster_name}/{rb.metadata.namespace}/{rb.metadata.name}",
+                    "name": rb.metadata.name,
+                    "namespace": rb.metadata.namespace,
+                    "uid": rb.metadata.uid,
+                    "creation_timestamp": get_epoch(rb.metadata.creation_timestamp),
+                    "resource_version": rb.metadata.resource_version,
+                    "role_name": rb.role_ref.name,
+                    "role_kind": rb.role_ref.kind,
+                    "service_account_ids": [
+                        f"{cluster_name}/{subject.namespace}/{subject.name}"
+                        for subject in service_account_subjects
+                    ],
+                    "user_ids": [
+                        f"{cluster_name}/{subject.name}" for subject in user_subjects
+                    ],
+                    "group_ids": [
+                        f"{cluster_name}/{subject.name}" for subject in group_subjects
+                    ],
+                    "role_id": f"{cluster_name}/{rb.metadata.namespace}/{rb.role_ref.name}",
+                }
+            )
+    return result
+
+
+def transform_cluster_roles(
+    cluster_roles: List[V1ClusterRole], cluster_name: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Kubernetes ClusterRoles into a list of dictionaries.
+    Flattens rules into separate api_groups, resources, and verbs lists.
+    """
+    result = []
+    for cluster_role in cluster_roles:
+        # Flatten all rules into combined sets
+        all_api_groups: set[str] = set()
+        all_resources: set[str] = set()
+        all_verbs: set[str] = set()
+
+        for rule in cluster_role.rules or []:
+            # Update api_groups, handling None and empty string cases
+            all_api_groups.update(
+                {
+                    "core" if api_group == "" else api_group
+                    for api_group in rule.api_groups or []
+                }
+            )
+            all_resources.update(rule.resources or [])
+            all_verbs.update(rule.verbs or [])
+
+        result.append(
+            {
+                "id": f"{cluster_name}/{cluster_role.metadata.name}",
+                "name": cluster_role.metadata.name,
+                "uid": cluster_role.metadata.uid,
+                "creation_timestamp": get_epoch(
+                    cluster_role.metadata.creation_timestamp
+                ),
+                "resource_version": cluster_role.metadata.resource_version,
+                "api_groups": sorted(
+                    all_api_groups
+                ),  # sorts to keep consistent ordering and converts to list to appease neo4j
+                "resources": sorted(all_resources),
+                "verbs": sorted(all_verbs),
+            }
+        )
+    return result
+
+
+def transform_cluster_role_bindings(
+    cluster_role_bindings: List[V1ClusterRoleBinding], cluster_name: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Kubernetes ClusterRoleBindings into a list of dictionaries.
+    Creates one ClusterRoleBinding node per Kubernetes ClusterRoleBinding with lists of subject IDs.
+    """
+    result = []
+    for crb in cluster_role_bindings:
+        # Collect all subjects by type
+        service_account_subjects = [
+            subject
+            for subject in (crb.subjects or [])
+            if subject.kind == "ServiceAccount"
+        ]
+        user_subjects = [
+            subject for subject in (crb.subjects or []) if subject.kind == "User"
+        ]
+        group_subjects = [
+            subject for subject in (crb.subjects or []) if subject.kind == "Group"
+        ]
+
+        # Only create a ClusterRoleBinding node if it has at least one subject
+        if crb.subjects:
+            result.append(
+                {
+                    "id": f"{cluster_name}/{crb.metadata.name}",
+                    "name": crb.metadata.name,
+                    "uid": crb.metadata.uid,
+                    "creation_timestamp": get_epoch(crb.metadata.creation_timestamp),
+                    "resource_version": crb.metadata.resource_version,
+                    "role_name": crb.role_ref.name,
+                    "role_kind": crb.role_ref.kind,
+                    "service_account_ids": [
+                        f"{cluster_name}/{subject.namespace}/{subject.name}"
+                        for subject in service_account_subjects
+                    ],
+                    "user_ids": [
+                        f"{cluster_name}/{subject.name}" for subject in user_subjects
+                    ],
+                    "group_ids": [
+                        f"{cluster_name}/{subject.name}" for subject in group_subjects
+                    ],
+                    "role_id": f"{cluster_name}/{crb.role_ref.name}",
+                }
+            )
+    return result
+
+
+@timeit
+def load_service_accounts(
+    session: neo4j.Session,
+    service_accounts: List[Dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(service_accounts)} KubernetesServiceAccounts")
+    load(
+        session,
+        KubernetesServiceAccountSchema(),
+        service_accounts,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def load_roles(
+    session: neo4j.Session,
+    roles: List[Dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(roles)} KubernetesRoles")
+    load(
+        session,
+        KubernetesRoleSchema(),
+        roles,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def load_role_bindings(
+    session: neo4j.Session,
+    role_bindings: List[Dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(role_bindings)} KubernetesRoleBindings")
+    load(
+        session,
+        KubernetesRoleBindingSchema(),
+        role_bindings,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def load_cluster_roles(
+    session: neo4j.Session,
+    cluster_roles: List[Dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(cluster_roles)} KubernetesClusterRoles")
+    load(
+        session,
+        KubernetesClusterRoleSchema(),
+        cluster_roles,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def load_cluster_role_bindings(
+    session: neo4j.Session,
+    cluster_role_bindings: List[Dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(cluster_role_bindings)} KubernetesClusterRoleBindings")
+    load(
+        session,
+        KubernetesClusterRoleBindingSchema(),
+        cluster_role_bindings,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def cleanup(session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    logger.debug("Running cleanup job for Kubernetes RBAC resources")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesServiceAccountSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesRoleSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesRoleBindingSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesClusterRoleSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesClusterRoleBindingSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+
+@timeit
+def sync_kubernetes_rbac(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.info(f"Syncing Kubernetes RBAC resources for cluster {client.name}")
+
+    # Get namespace-scoped resources
+    service_accounts = get_service_accounts(client)
+    roles = get_roles(client)
+    role_bindings = get_role_bindings(client)
+
+    # Get cluster-scoped resources
+    cluster_roles = get_cluster_roles(client)
+    cluster_role_bindings = get_cluster_role_bindings(client)
+
+    # Transform namespace-scoped resources
+    transformed_service_accounts = transform_service_accounts(
+        service_accounts, client.name
+    )
+    transformed_roles = transform_roles(roles, client.name)
+    transformed_role_bindings = transform_role_bindings(role_bindings, client.name)
+
+    # Transform cluster-scoped resources
+    transformed_cluster_roles = transform_cluster_roles(cluster_roles, client.name)
+    transformed_cluster_role_bindings = transform_cluster_role_bindings(
+        cluster_role_bindings, client.name
+    )
+
+    cluster_id = common_job_parameters["CLUSTER_ID"]
+    cluster_name = client.name
+
+    load_service_accounts(
+        session=session,
+        service_accounts=transformed_service_accounts,
+        update_tag=update_tag,
+        cluster_id=cluster_id,
+        cluster_name=cluster_name,
+    )
+
+    load_roles(
+        session=session,
+        roles=transformed_roles,
+        update_tag=update_tag,
+        cluster_id=cluster_id,
+        cluster_name=cluster_name,
+    )
+
+    load_cluster_roles(
+        session=session,
+        cluster_roles=transformed_cluster_roles,
+        update_tag=update_tag,
+        cluster_id=cluster_id,
+        cluster_name=cluster_name,
+    )
+
+    load_role_bindings(
+        session=session,
+        role_bindings=transformed_role_bindings,
+        update_tag=update_tag,
+        cluster_id=cluster_id,
+        cluster_name=cluster_name,
+    )
+
+    load_cluster_role_bindings(
+        session=session,
+        cluster_role_bindings=transformed_cluster_role_bindings,
+        update_tag=update_tag,
+        cluster_id=cluster_id,
+        cluster_name=cluster_name,
+    )
+
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/util.py

@@ -7,6 +7,7 @@ from kubernetes import config
 from kubernetes.client import ApiClient
 from kubernetes.client import CoreV1Api
 from kubernetes.client import NetworkingV1Api
+from kubernetes.client import RbacAuthorizationV1Api
 from kubernetes.client import VersionApi
 from kubernetes.client.exceptions import ApiException
 
@@ -62,6 +63,21 @@ class K8VersionApiClient(VersionApi):
         super().__init__(api_client=api_client)
 
 
+class K8RbacApiClient(RbacAuthorizationV1Api):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
 class K8sClient:
     def __init__(
         self,
@@ -75,6 +91,7 @@ class K8sClient:
         self.core = K8CoreApiClient(self.name, self.config_file)
         self.networking = K8NetworkingApiClient(self.name, self.config_file)
         self.version = K8VersionApiClient(self.name, self.config_file)
+        self.rbac = K8RbacApiClient(self.name, self.config_file)
 
 
 def get_k8s_clients(kubeconfig: str) -> list[K8sClient]:

cartography/intel/trivy/__init__.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any
 
@@ -8,7 +9,9 @@ from cartography.client.aws import list_accounts
 from cartography.client.aws.ecr import get_ecr_images
 from cartography.config import Config
 from cartography.intel.trivy.scanner import cleanup
+from cartography.intel.trivy.scanner import get_json_files_in_dir
 from cartography.intel.trivy.scanner import get_json_files_in_s3
+from cartography.intel.trivy.scanner import sync_single_image_from_file
 from cartography.intel.trivy.scanner import sync_single_image_from_s3
 from cartography.stats import get_stats_client
 from cartography.util import timeit
@@ -39,13 +42,13 @@ def get_scan_targets(
 
 
 def _get_intersection(
-    images_in_graph: set[str], json_files: set[str], trivy_s3_prefix: str
+    image_uris: set[str], json_files: set[str], trivy_s3_prefix: str
 ) -> list[tuple[str, str]]:
     """
     Get the intersection of ECR images in the graph and S3 scan results.
 
     Args:
-        images_in_graph: Set of ECR images in the graph
+        image_uris: Set of ECR images in the graph
         json_files: Set of S3 object keys for JSON files
         trivy_s3_prefix: S3 prefix path containing scan results
 
@@ -60,7 +63,7 @@ def _get_intersection(
         # Remove the prefix and the .json suffix
         image_uri = s3_object_key[prefix_len:-5]
 
-        if image_uri in images_in_graph:
+        if image_uri in image_uris:
             intersection.append((image_uri, s3_object_key))
 
     return intersection
@@ -90,12 +93,12 @@ def sync_trivy_aws_ecr_from_s3(
         f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
     )
 
-    images_in_graph: set[str] = get_scan_targets(neo4j_session)
+    image_uris: set[str] = get_scan_targets(neo4j_session)
     json_files: set[str] = get_json_files_in_s3(
         trivy_s3_bucket, trivy_s3_prefix, boto3_session
     )
     intersection: list[tuple[str, str]] = _get_intersection(
-        images_in_graph, json_files, trivy_s3_prefix
+        image_uris, json_files, trivy_s3_prefix
     )
 
     if len(intersection) == 0:
@@ -124,21 +127,79 @@ def sync_trivy_aws_ecr_from_s3(
     cleanup(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_trivy_aws_ecr_from_dir(
+    neo4j_session: Session,
+    results_dir: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """Sync Trivy scan results from local files for AWS ECR images."""
+    logger.info(f"Using Trivy scan results from {results_dir}")
+
+    image_uris: set[str] = get_scan_targets(neo4j_session)
+    json_files: set[str] = get_json_files_in_dir(results_dir)
+
+    if not json_files:
+        logger.error(
+            f"Trivy sync was configured, but no json files were found in {results_dir}."
+        )
+        raise ValueError("No Trivy json results found on disk")
+
+    logger.info(f"Processing {len(json_files)} local Trivy result files")
+
+    for file_path in json_files:
+        # First, check if the image exists in the graph before syncing
+        try:
+            # Peek at the artifact name without processing the file
+            with open(file_path, encoding="utf-8") as f:
+                trivy_data = json.load(f)
+                artifact_name = trivy_data.get("ArtifactName")
+
+            if artifact_name and artifact_name not in image_uris:
+                logger.debug(
+                    f"Skipping results for {artifact_name} since the image is not present in the graph"
+                )
+                continue
+
+        except (json.JSONDecodeError, KeyError) as e:
+            logger.error(f"Failed to read artifact name from {file_path}: {e}")
+            continue
+
+        # Now sync the file since we know the image exists in the graph
+        sync_single_image_from_file(
+            neo4j_session,
+            file_path,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)
+
+
 @timeit
 def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
-    """
-    Start Trivy scan ingestion from S3.
+    """Start Trivy scan ingestion from S3 or local files.
 
     Args:
         neo4j_session: Neo4j session for database operations
-        config: Configuration object containing S3 settings
+        config: Configuration object containing S3 or directory paths
     """
-    # Check if S3 configuration is provided
-    if not config.trivy_s3_bucket:
-        logger.info("Trivy S3 configuration not provided. Skipping Trivy ingestion.")
+    if not config.trivy_s3_bucket and not config.trivy_results_dir:
+        logger.info("Trivy configuration not provided. Skipping Trivy ingestion.")
+        return
+
+    if config.trivy_results_dir:
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+        sync_trivy_aws_ecr_from_dir(
+            neo4j_session,
+            config.trivy_results_dir,
+            config.update_tag,
+            common_job_parameters,
+        )
         return
 
-    # Default to empty string if s3 prefix is not provided
     if config.trivy_s3_prefix is None:
         config.trivy_s3_prefix = ""
 
@@ -146,7 +207,6 @@ def start_trivy_ingestion(neo4j_session: Session, config: Config) -> None:
         "UPDATE_TAG": config.update_tag,
     }
 
-    # Get ECR images to scan
     boto3_session = boto3.Session()
 
     sync_trivy_aws_ecr_from_s3(