cartography 0.105.0__py3-none-any.whl → 0.106.0rc2__py3-none-any.whl

cartography/intel/kubernetes/pods.py

@@ -1,39 +1,35 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1Container
+from kubernetes.client.models import V1Pod
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.containers import KubernetesContainerSchema
+from cartography.models.kubernetes.pods import KubernetesPodSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
-@timeit
-def sync_pods(
-    session: Session,
-    client: K8sClient,
-    update_tag: int,
-    cluster: Dict,
-) -> List[Dict]:
-    pods = get_pods(client, cluster)
-    load_pods(session, pods, update_tag)
-    return pods
-
-
-@timeit
-def get_pods(client: K8sClient, cluster: Dict) -> List[Dict]:
-    pods = list()
-    for pod in client.core.list_pod_for_all_namespaces().items:
-        containers = {}
-        for container in pod.spec.containers:
-            containers[container.name] = {
-                "name": container.name,
-                "image": container.image,
-                "uid": f"{pod.metadata.uid}-{container.name}",
-            }
+def _extract_pod_containers(pod: V1Pod) -> dict[str, Any]:
+    pod_containers: list[V1Container] = pod.spec.containers
+    containers = dict()
+    for container in pod_containers:
+        containers[container.name] = {
+            "uid": f"{pod.metadata.uid}-{container.name}",
+            "name": container.name,
+            "image": container.image,
+            "namespace": pod.metadata.namespace,
+            "pod_id": pod.metadata.uid,
+            "imagePullPolicy": container.image_pull_policy,
+        }
         if pod.status and pod.status.container_statuses:
             for status in pod.status.container_statuses:
                 if status.name in containers:
@@ -46,14 +42,31 @@ def get_pods(client: K8sClient, cluster: Dict) -> List[Dict]:
                         image_sha = status.image_id.split("@")[1]
                     except IndexError:
                         image_sha = None
-                    containers[status.name]["status"] = {
-                        "image_id": status.image_id,
-                        "image_sha": image_sha,
-                        "ready": status.ready,
-                        "started": status.started,
-                        "state": _state,
-                    }
-        pods.append(
+
+                    containers[status.name]["status_image_id"] = status.image_id
+                    containers[status.name]["status_image_sha"] = image_sha
+                    containers[status.name]["status_ready"] = status.ready
+                    containers[status.name]["status_started"] = status.started
+                    containers[status.name]["status_state"] = _state
+    return containers
+
+
+@timeit
+def get_pods(client: K8sClient) -> list[V1Pod]:
+    items = k8s_paginate(client.core.list_pod_for_all_namespaces)
+    return items
+
+
+def _format_pod_labels(labels: dict[str, str]) -> str:
+    return json.dumps(labels)
+
+
+def transform_pods(pods: list[V1Pod]) -> list[dict[str, Any]]:
+    transformed_pods = []
+
+    for pod in pods:
+        containers = _extract_pod_containers(pod)
+        transformed_pods.append(
             {
                 "uid": pod.metadata.uid,
                 "name": pod.metadata.name,
@@ -62,49 +75,99 @@ def get_pods(client: K8sClient, cluster: Dict) -> List[Dict]:
                 "deletion_timestamp": get_epoch(pod.metadata.deletion_timestamp),
                 "namespace": pod.metadata.namespace,
                 "node": pod.spec.node_name,
-                "cluster_uid": cluster["uid"],
-                "labels": pod.metadata.labels,
+                "labels": _format_pod_labels(pod.metadata.labels),
                 "containers": list(containers.values()),
             },
         )
-    return pods
-
-
-def load_pods(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $pods as k8pod
-        MERGE (pod:KubernetesPod {id: k8pod.uid})
-        ON CREATE SET pod.firstseen = timestamp()
-        SET pod.lastupdated = $update_tag,
-            pod.name = k8pod.name,
-            pod.status_phase = k8pod.status_phase,
-            pod.created_at = k8pod.creation_timestamp,
-            pod.deleted_at = k8pod.deletion_timestamp
-        WITH pod, k8pod.namespace as ns, k8pod.cluster_uid as cuid, k8pod.containers as k8containers
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_POD]->(pod)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-        WITH pod, space, cluster, k8containers
-        UNWIND k8containers as k8container
-            MERGE (container: KubernetesContainer {id: k8container.uid})
-            ON CREATE SET container.firstseen = timestamp()
-            SET container.image = k8container.image,
-                container.status_image_id = k8container.status.image_id,
-                container.status_image_sha = k8container.status.image_sha,
-                container.status_ready = k8container.status.ready,
-                container.status_started = k8container.status.started,
-                container.status_state = k8container.status.state,
-                container.name = k8container.name,
-                container.lastupdated = $update_tag
-            WITH pod, space, cluster, container
-            MERGE (pod)-[rel2:HAS_CONTAINER]->(container)
-            ON CREATE SET rel2.firstseen = timestamp()
-            SET rel2.lastupdated = $update_tag
-            WITH pod, space, container
-            MERGE (cluster)-[rel3:HAS_POD]->(pod)
-            ON CREATE SET rel3.firstseen = timestamp()
-            SET rel3.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes pods.")
-    session.run(ingestion_cypher_query, pods=data, update_tag=update_tag)
+    return transformed_pods
+
+
+@timeit
+def load_pods(
+    session: neo4j.Session,
+    pods: list[dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(pods)} kubernetes pods.")
+    load(
+        session,
+        KubernetesPodSchema(),
+        pods,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+def transform_containers(pods: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    containers = []
+    for pod in pods:
+        containers.extend(pod.get("containers", []))
+    return containers
+
+
+@timeit
+def load_containers(
+    session: neo4j.Session,
+    containers: list[dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(containers)} kubernetes containers.")
+    load(
+        session,
+        KubernetesContainerSchema(),
+        containers,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup job for KubernetesContainer")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesContainerSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+    logger.debug("Running cleanup job for KubernetesPod")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesPodSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+
+@timeit
+def sync_pods(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[dict[str, Any]]:
+    pods = get_pods(client)
+
+    transformed_pods = transform_pods(pods)
+    load_pods(
+        session=session,
+        pods=transformed_pods,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+
+    transformed_containers = transform_containers(transformed_pods)
+    load_containers(
+        session=session,
+        containers=transformed_containers,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+
+    cleanup(session, common_job_parameters)
+    return transformed_pods

cartography/intel/kubernetes/secrets.py

@@ -1,60 +1,110 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1OwnerReference
+from kubernetes.client.models import V1Secret
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.secrets import KubernetesSecretSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_secrets(
-    session: Session,
-    client: K8sClient,
+def get_secrets(client: K8sClient) -> list[V1Secret]:
+    items = k8s_paginate(client.core.list_secret_for_all_namespaces)
+    return items
+
+
+def _get_owner_references(
+    owner_references: list[V1OwnerReference] | None,
+) -> str | None:
+    if owner_references:
+        owner_references_list = []
+        for owner_reference in owner_references:
+            owner_references_list.append(
+                {
+                    "kind": owner_reference.kind,
+                    "name": owner_reference.name,
+                    "uid": owner_reference.uid,
+                    "apiVersion": owner_reference.api_version,
+                    "controller": owner_reference.controller,
+                }
+            )
+        return json.dumps(owner_references_list)
+    return None
+
+
+def transform_secrets(secrets: list[V1Secret]) -> list[dict[str, Any]]:
+    secrets_list = []
+    for secret in secrets:
+        secrets_list.append(
+            {
+                "uid": secret.metadata.uid,
+                "name": secret.metadata.name,
+                "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
+                "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
+                "owner_references": _get_owner_references(
+                    secret.metadata.owner_references
+                ),
+                "namespace": secret.metadata.namespace,
+                "type": secret.type,
+            }
+        )
+
+    return secrets_list
+
+
+@timeit
+def load_secrets(
+    session: neo4j.Session,
+    secrets: list[dict[str, Any]],
     update_tag: int,
-    cluster: Dict,
-) -> List[Dict]:
-    secrets = get_secrets(client, cluster)
-    load_secrets(session, secrets, update_tag)
-    return secrets
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(secrets)} KubernetesSecrets")
+    load(
+        session,
+        KubernetesSecretSchema(),
+        secrets,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup for KubernetesSecrets")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesSecretSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(session)
 
 
 @timeit
-def get_secrets(client: K8sClient, cluster: Dict) -> List[Dict]:
-    return [
-        {
-            "uid": secret.metadata.uid,
-            "name": secret.metadata.name,
-            "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
-            "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
-            "namespace": secret.metadata.namespace,
-            "cluster_uid": cluster["uid"],
-            "labels": secret.metadata.labels,
-            "type": secret.type,
-        }
-        for secret in client.core.list_secret_for_all_namespaces().items
-    ]
-
-
-def load_secrets(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $secrets as k8secret
-        MERGE (secret:KubernetesSecret {id: k8secret.uid})
-        ON CREATE SET secret.firstseen = timestamp()
-        SET secret.lastupdated = $update_tag,
-            secret.name = k8secret.name,
-            secret.created_at = k8secret.creation_timestamp,
-            secret.deleted_at = k8secret.deletion_timestamp,
-            secret.type = k8secret.type
-        WITH secret, k8secret.namespace as ns, k8secret.cluster_uid as cuid
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SECRET]->(secret)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes secrets.")
-    session.run(ingestion_cypher_query, secrets=data, update_tag=update_tag)
+def sync_secrets(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    secrets = get_secrets(client)
+    transformed_secrets = transform_secrets(secrets)
+    load_secrets(
+        session=session,
+        secrets=transformed_secrets,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/services.py

@@ -1,90 +1,154 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1LoadBalancerIngress
+from kubernetes.client.models import V1PortStatus
+from kubernetes.client.models import V1Service
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.services import KubernetesServiceSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_services(
-    session: Session,
-    client: K8sClient,
-    update_tag: int,
-    cluster: Dict,
-    pods: List[Dict],
-) -> None:
-    services = get_services(client, cluster, pods)
-    load_services(session, services, update_tag)
+def get_services(client: K8sClient) -> list[V1Service]:
+    items = k8s_paginate(client.core.list_service_for_all_namespaces)
+    return items
 
 
-@timeit
-def get_services(client: K8sClient, cluster: Dict, pods: List[Dict]) -> List[Dict]:
-    services = list()
-    for service in client.core.list_service_for_all_namespaces().items:
+def _format_service_selector(selector: dict[str, str]) -> str:
+    return json.dumps(selector)
+
+
+def _format_load_balancer_ingress(ingress: list[V1LoadBalancerIngress] | None) -> str:
+
+    def _format_ingress_ports(
+        ports: list[V1PortStatus] | None,
+    ) -> list[dict[str, Any]] | None:
+        if ports is None:
+            return None
+
+        ingress_ports = []
+        for port in ports:
+            ingress_ports.append(
+                {
+                    "error": port.port,
+                    "port": port.protocol,
+                    "protocol": port.ip,
+                }
+            )
+        return ingress_ports
+
+    if ingress is None:
+        return json.dumps(None)
+
+    loadbalancer_ingress = []
+    for item in ingress:
+        loadbalancer_ingress.append(
+            {
+                "hostname": item.hostname,
+                "ip": item.ip,
+                "ip_mode": item.ip_mode,
+                "ports": _format_ingress_ports(item.ports),
+            }
+        )
+    return json.dumps(loadbalancer_ingress)
+
+
+def transform_services(
+    services: list[V1Service], all_pods: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    services_list = []
+    for service in services:
         item = {
             "uid": service.metadata.uid,
             "name": service.metadata.name,
             "creation_timestamp": get_epoch(service.metadata.creation_timestamp),
             "deletion_timestamp": get_epoch(service.metadata.deletion_timestamp),
             "namespace": service.metadata.namespace,
-            "cluster_uid": cluster["uid"],
             "type": service.spec.type,
-            "selector": service.spec.selector,
+            "selector": _format_service_selector(service.spec.selector),
+            "cluster_ip": service.spec.cluster_ip,
             "load_balancer_ip": service.spec.load_balancer_ip,
         }
 
-        ingresses = service.status.load_balancer.ingress
-        for ingress in ingresses or list():
-            item.update({"ingress_host": ingress.hostname, "ingress_ip": ingress.ip})
-
-        service_pods = list()
-        for pod in pods:
-            is_service_pod = True if service.spec.selector else False
-            for selector in service.spec.selector or dict():
-                if (
-                    not pod.get("labels")
-                    or selector not in pod["labels"]
-                    or service.spec.selector[selector] != pod["labels"][selector]
-                ):
-                    is_service_pod = False
-                    break
-            if is_service_pod:
-                service_pods.append(pod)
-        item["pods"] = service_pods
-        services.append(item)
-    return services
-
-
-def load_services(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $services as k8service
-        MERGE (service:KubernetesService {id: k8service.uid})
-        ON CREATE SET service.firstseen = timestamp()
-        SET service.lastupdated = $update_tag,
-            service.name = k8service.name,
-            service.created_at = k8service.creation_timestamp,
-            service.deleted_at = k8service.deletion_timestamp,
-            service.type = k8service.type,
-            service.load_balancer_ip = k8service.load_balancer_ip,
-            service.ingress_host = k8service.ingress_host,
-            service.ingress_ip = k8service.ingress_ip
-        WITH service, k8service.namespace as ns, k8service.cluster_uid as cuid, k8service.pods as k8pods
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SERVICE]->(service)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-        WITH service, k8pods
-        UNWIND k8pods as k8pod
-            MATCH (pod:KubernetesPod {id: k8pod.uid})
-            MERGE (service)-[rel2:SERVES_POD]->(pod)
-            ON CREATE SET rel2.firstseen = timestamp()
-            SET rel2.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes services.")
-    session.run(ingestion_cypher_query, services=data, update_tag=update_tag)
+        # TODO: instead of storing a json string, we should probably create seperate nodes for each ingress
+        if service.spec.type == "LoadBalancer":
+            if service.status.load_balancer:
+                item["load_balancer_ingress"] = _format_load_balancer_ingress(
+                    service.status.load_balancer.ingress
+                )
+
+        # check if pod labels match service selector and add pod_ids to item
+        pod_ids = []
+        for pod in all_pods:
+            if pod["namespace"] == service.metadata.namespace:
+                service_selector: dict[str, str] | None = service.spec.selector
+                pod_labels: dict[str, str] | None = json.loads(pod["labels"])
+
+                # check if pod labels match service selector
+                if pod_labels and service_selector:
+                    if all(
+                        service_selector[key] == pod_labels.get(key)
+                        for key in service_selector
+                    ):
+                        pod_ids.append(pod["uid"])
+
+        item["pod_ids"] = pod_ids
+
+        services_list.append(item)
+    return services_list
+
+
+def load_services(
+    session: neo4j.Session,
+    services: list[dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(services)} KubernetesServices")
+    load(
+        session,
+        KubernetesServiceSchema(),
+        services,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup job for KubernetesService")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesServiceSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+
+@timeit
+def sync_services(
+    session: neo4j.Session,
+    client: K8sClient,
+    all_pods: list[dict[str, Any]],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    services = get_services(client)
+    transformed_services = transform_services(services, all_pods)
+    load_services(
+        session=session,
+        services=transformed_services,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)