cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/kubernetes/secrets.py

@@ -1,60 +1,110 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1OwnerReference
+from kubernetes.client.models import V1Secret
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.secrets import KubernetesSecretSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_secrets(
-    session: Session,
-    client: K8sClient,
+def get_secrets(client: K8sClient) -> list[V1Secret]:
+    items = k8s_paginate(client.core.list_secret_for_all_namespaces)
+    return items
+
+
+def _get_owner_references(
+    owner_references: list[V1OwnerReference] | None,
+) -> str | None:
+    if owner_references:
+        owner_references_list = []
+        for owner_reference in owner_references:
+            owner_references_list.append(
+                {
+                    "kind": owner_reference.kind,
+                    "name": owner_reference.name,
+                    "uid": owner_reference.uid,
+                    "apiVersion": owner_reference.api_version,
+                    "controller": owner_reference.controller,
+                }
+            )
+        return json.dumps(owner_references_list)
+    return None
+
+
+def transform_secrets(secrets: list[V1Secret]) -> list[dict[str, Any]]:
+    secrets_list = []
+    for secret in secrets:
+        secrets_list.append(
+            {
+                "uid": secret.metadata.uid,
+                "name": secret.metadata.name,
+                "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
+                "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
+                "owner_references": _get_owner_references(
+                    secret.metadata.owner_references
+                ),
+                "namespace": secret.metadata.namespace,
+                "type": secret.type,
+            }
+        )
+
+    return secrets_list
+
+
+@timeit
+def load_secrets(
+    session: neo4j.Session,
+    secrets: list[dict[str, Any]],
     update_tag: int,
-    cluster: Dict,
-) -> List[Dict]:
-    secrets = get_secrets(client, cluster)
-    load_secrets(session, secrets, update_tag)
-    return secrets
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(secrets)} KubernetesSecrets")
+    load(
+        session,
+        KubernetesSecretSchema(),
+        secrets,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup for KubernetesSecrets")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesSecretSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(session)
 
 
 @timeit
-def get_secrets(client: K8sClient, cluster: Dict) -> List[Dict]:
-    return [
-        {
-            "uid": secret.metadata.uid,
-            "name": secret.metadata.name,
-            "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
-            "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
-            "namespace": secret.metadata.namespace,
-            "cluster_uid": cluster["uid"],
-            "labels": secret.metadata.labels,
-            "type": secret.type,
-        }
-        for secret in client.core.list_secret_for_all_namespaces().items
-    ]
-
-
-def load_secrets(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $secrets as k8secret
-        MERGE (secret:KubernetesSecret {id: k8secret.uid})
-        ON CREATE SET secret.firstseen = timestamp()
-        SET secret.lastupdated = $update_tag,
-            secret.name = k8secret.name,
-            secret.created_at = k8secret.creation_timestamp,
-            secret.deleted_at = k8secret.deletion_timestamp,
-            secret.type = k8secret.type
-        WITH secret, k8secret.namespace as ns, k8secret.cluster_uid as cuid
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SECRET]->(secret)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes secrets.")
-    session.run(ingestion_cypher_query, secrets=data, update_tag=update_tag)
+def sync_secrets(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    secrets = get_secrets(client)
+    transformed_secrets = transform_secrets(secrets)
+    load_secrets(
+        session=session,
+        secrets=transformed_secrets,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/services.py

@@ -1,90 +1,154 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1LoadBalancerIngress
+from kubernetes.client.models import V1PortStatus
+from kubernetes.client.models import V1Service
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.services import KubernetesServiceSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_services(
-    session: Session,
-    client: K8sClient,
-    update_tag: int,
-    cluster: Dict,
-    pods: List[Dict],
-) -> None:
-    services = get_services(client, cluster, pods)
-    load_services(session, services, update_tag)
+def get_services(client: K8sClient) -> list[V1Service]:
+    items = k8s_paginate(client.core.list_service_for_all_namespaces)
+    return items
 
 
-@timeit
-def get_services(client: K8sClient, cluster: Dict, pods: List[Dict]) -> List[Dict]:
-    services = list()
-    for service in client.core.list_service_for_all_namespaces().items:
+def _format_service_selector(selector: dict[str, str]) -> str:
+    return json.dumps(selector)
+
+
+def _format_load_balancer_ingress(ingress: list[V1LoadBalancerIngress] | None) -> str:
+
+    def _format_ingress_ports(
+        ports: list[V1PortStatus] | None,
+    ) -> list[dict[str, Any]] | None:
+        if ports is None:
+            return None
+
+        ingress_ports = []
+        for port in ports:
+            ingress_ports.append(
+                {
+                    "error": port.port,
+                    "port": port.protocol,
+                    "protocol": port.ip,
+                }
+            )
+        return ingress_ports
+
+    if ingress is None:
+        return json.dumps(None)
+
+    loadbalancer_ingress = []
+    for item in ingress:
+        loadbalancer_ingress.append(
+            {
+                "hostname": item.hostname,
+                "ip": item.ip,
+                "ip_mode": item.ip_mode,
+                "ports": _format_ingress_ports(item.ports),
+            }
+        )
+    return json.dumps(loadbalancer_ingress)
+
+
+def transform_services(
+    services: list[V1Service], all_pods: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    services_list = []
+    for service in services:
         item = {
             "uid": service.metadata.uid,
             "name": service.metadata.name,
             "creation_timestamp": get_epoch(service.metadata.creation_timestamp),
             "deletion_timestamp": get_epoch(service.metadata.deletion_timestamp),
             "namespace": service.metadata.namespace,
-            "cluster_uid": cluster["uid"],
             "type": service.spec.type,
-            "selector": service.spec.selector,
+            "selector": _format_service_selector(service.spec.selector),
+            "cluster_ip": service.spec.cluster_ip,
             "load_balancer_ip": service.spec.load_balancer_ip,
         }
 
-        ingresses = service.status.load_balancer.ingress
-        for ingress in ingresses or list():
-            item.update({"ingress_host": ingress.hostname, "ingress_ip": ingress.ip})
-
-        service_pods = list()
-        for pod in pods:
-            is_service_pod = True if service.spec.selector else False
-            for selector in service.spec.selector or dict():
-                if (
-                    not pod.get("labels")
-                    or selector not in pod["labels"]
-                    or service.spec.selector[selector] != pod["labels"][selector]
-                ):
-                    is_service_pod = False
-                    break
-            if is_service_pod:
-                service_pods.append(pod)
-        item["pods"] = service_pods
-        services.append(item)
-    return services
-
-
-def load_services(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $services as k8service
-        MERGE (service:KubernetesService {id: k8service.uid})
-        ON CREATE SET service.firstseen = timestamp()
-        SET service.lastupdated = $update_tag,
-            service.name = k8service.name,
-            service.created_at = k8service.creation_timestamp,
-            service.deleted_at = k8service.deletion_timestamp,
-            service.type = k8service.type,
-            service.load_balancer_ip = k8service.load_balancer_ip,
-            service.ingress_host = k8service.ingress_host,
-            service.ingress_ip = k8service.ingress_ip
-        WITH service, k8service.namespace as ns, k8service.cluster_uid as cuid, k8service.pods as k8pods
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SERVICE]->(service)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-        WITH service, k8pods
-        UNWIND k8pods as k8pod
-            MATCH (pod:KubernetesPod {id: k8pod.uid})
-            MERGE (service)-[rel2:SERVES_POD]->(pod)
-            ON CREATE SET rel2.firstseen = timestamp()
-            SET rel2.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes services.")
-    session.run(ingestion_cypher_query, services=data, update_tag=update_tag)
+        # TODO: instead of storing a json string, we should probably create seperate nodes for each ingress
+        if service.spec.type == "LoadBalancer":
+            if service.status.load_balancer:
+                item["load_balancer_ingress"] = _format_load_balancer_ingress(
+                    service.status.load_balancer.ingress
+                )
+
+        # check if pod labels match service selector and add pod_ids to item
+        pod_ids = []
+        for pod in all_pods:
+            if pod["namespace"] == service.metadata.namespace:
+                service_selector: dict[str, str] | None = service.spec.selector
+                pod_labels: dict[str, str] | None = json.loads(pod["labels"])
+
+                # check if pod labels match service selector
+                if pod_labels and service_selector:
+                    if all(
+                        service_selector[key] == pod_labels.get(key)
+                        for key in service_selector
+                    ):
+                        pod_ids.append(pod["uid"])
+
+        item["pod_ids"] = pod_ids
+
+        services_list.append(item)
+    return services_list
+
+
+def load_services(
+    session: neo4j.Session,
+    services: list[dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(services)} KubernetesServices")
+    load(
+        session,
+        KubernetesServiceSchema(),
+        services,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup job for KubernetesService")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesServiceSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+
+@timeit
+def sync_services(
+    session: neo4j.Session,
+    client: K8sClient,
+    all_pods: list[dict[str, Any]],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    services = get_services(client)
+    transformed_services = transform_services(services, all_pods)
+    load_services(
+        session=session,
+        services=transformed_services,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/util.py

@@ -1,11 +1,17 @@
+import logging
 from datetime import datetime
-from typing import List
-from typing import Union
+from typing import Any
+from typing import Callable
 
 from kubernetes import config
 from kubernetes.client import ApiClient
 from kubernetes.client import CoreV1Api
 from kubernetes.client import NetworkingV1Api
+from kubernetes.client import RbacAuthorizationV1Api
+from kubernetes.client import VersionApi
+from kubernetes.client.exceptions import ApiException
+
+logger = logging.getLogger(__name__)
 
 
 class KubernetesContextNotFound(Exception):
@@ -13,39 +19,161 @@ class KubernetesContextNotFound(Exception):
 
 
 class K8CoreApiClient(CoreV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8NetworkingApiClient(NetworkingV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
+class K8VersionApiClient(VersionApi):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
+class K8RbacApiClient(RbacAuthorizationV1Api):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8sClient:
-    def __init__(self, name: str) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        external_id: str | None = None,
+    ) -> None:
         self.name = name
-        self.core = K8CoreApiClient(self.name)
-        self.networking = K8NetworkingApiClient(self.name)
+        self.config_file = config_file
+        self.external_id = external_id
+        self.core = K8CoreApiClient(self.name, self.config_file)
+        self.networking = K8NetworkingApiClient(self.name, self.config_file)
+        self.version = K8VersionApiClient(self.name, self.config_file)
+        self.rbac = K8RbacApiClient(self.name, self.config_file)
 
 
-def get_k8s_clients(kubeconfig: str) -> List[K8sClient]:
+def get_k8s_clients(kubeconfig: str) -> list[K8sClient]:
+    # returns a tuple of (all contexts, current context)
     contexts, _ = config.list_kube_config_contexts(kubeconfig)
     if not contexts:
         raise KubernetesContextNotFound("No context found in kubeconfig.")
-    clients = list()
+
+    clients = []
     for context in contexts:
-        clients.append(K8sClient(context["name"]))
+        clients.append(
+            K8sClient(
+                context["name"],
+                kubeconfig,
+                external_id=context["context"].get("cluster"),
+            ),
+        )
     return clients
 
 
-def get_epoch(date: datetime) -> Union[int, None]:
+def get_epoch(date: datetime | None) -> int | None:
     if date:
-        return int(date.strftime("%s"))
+        return int(date.timestamp())
     return None
+
+
+def k8s_paginate(
+    list_func: Callable,
+    **kwargs: Any,
+) -> list[dict[str, Any]]:
+    """
+    Handles pagination for a Kubernetes API call.
+
+    :param list_func: The list function to call (e.g. client.core.list_pod_for_all_namespaces)
+    :param kwargs: Keyword arguments to pass to the list function (e.g. limit=100)
+    :return: A list of all resources returned by the list function
+    """
+    all_resources = []
+    continue_token = None
+    limit = kwargs.pop("limit", 100)
+    function_name = list_func.__name__
+
+    logger.debug(f"Starting pagination for {function_name} with limit {limit}.")
+
+    while True:
+        try:
+            if continue_token:
+                response = list_func(limit=limit, _continue=continue_token, **kwargs)
+            else:
+                response = list_func(limit=limit, **kwargs)
+
+            # Check if items exists on the response
+            if not hasattr(response, "items"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'items' attribute."
+                )
+                break
+
+            items_count = len(response.items)
+            all_resources.extend(response.items)
+
+            logger.debug(f"Retrieved {items_count} {function_name} resources")
+
+            # Check if metadata exists on the response
+            if not hasattr(response, "metadata"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'metadata' attribute."
+                )
+                break
+
+            continue_token = response.metadata._continue
+            if not continue_token:
+                logger.debug(f"No more {function_name} resources to retrieve.")
+                break
+
+        except ApiException as e:
+            logger.error(
+                f"Kubernetes API error retrieving {function_name} resources. {e}: {e.status} - {e.reason}"
+            )
+            break
+
+    logger.debug(
+        f"Completed pagination for {function_name}: retrieved {len(all_resources)} resources"
+    )
+    return all_resources

cartography/intel/oci/iam.py

@@ -10,6 +10,8 @@ from typing import List
 import neo4j
 import oci
 
+from cartography.client.core.tx import read_list_of_dicts_tx
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 
 from . import utils
@@ -89,7 +91,8 @@ def load_compartments(
     """
 
     for compartment in compartments:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_compartment,
             OCID=compartment["id"],
             COMPARTMENT_ID=compartment["compartment-id"],
@@ -126,7 +129,8 @@ def load_users(
     """
 
     for user in users:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_user,
             OCID=user["id"],
             CREATE_DATE=str(user["time-created"]),
@@ -206,7 +210,8 @@ def load_groups(
     """
 
     for group in groups:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_group,
             OCID=group["id"],
             CREATE_DATE=str(group["time-created"]),
@@ -260,7 +265,11 @@ def sync_group_memberships(
         "MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) "
         "return group.name as name, group.ocid as ocid;"
     )
-    groups = neo4j_session.run(query, OCI_TENANCY_ID=current_tenancy_id)
+    groups = neo4j_session.execute_read(
+        read_list_of_dicts_tx,
+        query,
+        OCI_TENANCY_ID=current_tenancy_id,
+    )
     groups_membership = {
         group["ocid"]: get_group_membership_data(iam, group["ocid"], current_tenancy_id)
         for group in groups
@@ -288,7 +297,8 @@ def load_group_memberships(
     """
     for group_ocid, membership_data in group_memberships.items():
         for info in membership_data["GroupMemberships"]:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_membership,
                 COMPARTMENT_ID=info["compartment-id"],
                 GROUP_OCID=info["group-id"],
@@ -317,7 +327,8 @@ def load_policies(
     """
 
     for policy in policies:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_policy,
             OCID=policy["id"],
             POLICY_NAME=policy["name"],
@@ -386,7 +397,8 @@ def load_oci_policy_group_reference(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $oci_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_group_reference,
         POLICY_ID=policy_id,
         GROUP_ID=group_id,
@@ -408,7 +420,8 @@ def load_oci_policy_compartment_reference(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $oci_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_compartment_reference,
         POLICY_ID=policy_id,
         COMPARTMENT_ID=compartment_id,
@@ -487,7 +500,8 @@ def load_region_subscriptions(
     SET r.lastupdated = $oci_update_tag
     """
     for region in regions:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             REGION_KEY=region["region-key"],
             REGION_NAME=region["region-name"],