cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/kubernetes/secrets.py

@@ -1,60 +1,110 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1OwnerReference
+from kubernetes.client.models import V1Secret
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.secrets import KubernetesSecretSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_secrets(
-    session: Session,
-    client: K8sClient,
+def get_secrets(client: K8sClient) -> list[V1Secret]:
+    items = k8s_paginate(client.core.list_secret_for_all_namespaces)
+    return items
+
+
+def _get_owner_references(
+    owner_references: list[V1OwnerReference] | None,
+) -> str | None:
+    if owner_references:
+        owner_references_list = []
+        for owner_reference in owner_references:
+            owner_references_list.append(
+                {
+                    "kind": owner_reference.kind,
+                    "name": owner_reference.name,
+                    "uid": owner_reference.uid,
+                    "apiVersion": owner_reference.api_version,
+                    "controller": owner_reference.controller,
+                }
+            )
+        return json.dumps(owner_references_list)
+    return None
+
+
+def transform_secrets(secrets: list[V1Secret]) -> list[dict[str, Any]]:
+    secrets_list = []
+    for secret in secrets:
+        secrets_list.append(
+            {
+                "uid": secret.metadata.uid,
+                "name": secret.metadata.name,
+                "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
+                "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
+                "owner_references": _get_owner_references(
+                    secret.metadata.owner_references
+                ),
+                "namespace": secret.metadata.namespace,
+                "type": secret.type,
+            }
+        )
+
+    return secrets_list
+
+
+@timeit
+def load_secrets(
+    session: neo4j.Session,
+    secrets: list[dict[str, Any]],
     update_tag: int,
-    cluster: Dict,
-) -> List[Dict]:
-    secrets = get_secrets(client, cluster)
-    load_secrets(session, secrets, update_tag)
-    return secrets
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(secrets)} KubernetesSecrets")
+    load(
+        session,
+        KubernetesSecretSchema(),
+        secrets,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+@timeit
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup for KubernetesSecrets")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesSecretSchema(),
+        common_job_parameters,
+    )
+    cleanup_job.run(session)
 
 
 @timeit
-def get_secrets(client: K8sClient, cluster: Dict) -> List[Dict]:
-    return [
-        {
-            "uid": secret.metadata.uid,
-            "name": secret.metadata.name,
-            "creation_timestamp": get_epoch(secret.metadata.creation_timestamp),
-            "deletion_timestamp": get_epoch(secret.metadata.deletion_timestamp),
-            "namespace": secret.metadata.namespace,
-            "cluster_uid": cluster["uid"],
-            "labels": secret.metadata.labels,
-            "type": secret.type,
-        }
-        for secret in client.core.list_secret_for_all_namespaces().items
-    ]
-
-
-def load_secrets(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $secrets as k8secret
-        MERGE (secret:KubernetesSecret {id: k8secret.uid})
-        ON CREATE SET secret.firstseen = timestamp()
-        SET secret.lastupdated = $update_tag,
-            secret.name = k8secret.name,
-            secret.created_at = k8secret.creation_timestamp,
-            secret.deleted_at = k8secret.deletion_timestamp,
-            secret.type = k8secret.type
-        WITH secret, k8secret.namespace as ns, k8secret.cluster_uid as cuid
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SECRET]->(secret)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes secrets.")
-    session.run(ingestion_cypher_query, secrets=data, update_tag=update_tag)
+def sync_secrets(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    secrets = get_secrets(client)
+    transformed_secrets = transform_secrets(secrets)
+    load_secrets(
+        session=session,
+        secrets=transformed_secrets,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/services.py

@@ -1,86 +1,154 @@
+import json
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1LoadBalancerIngress
+from kubernetes.client.models import V1PortStatus
+from kubernetes.client.models import V1Service
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.services import KubernetesServiceSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def sync_services(
-    session: Session, client: K8sClient, update_tag: int, cluster: Dict, pods: List[Dict],
-) -> None:
-    services = get_services(client, cluster, pods)
-    load_services(session, services, update_tag)
+def get_services(client: K8sClient) -> list[V1Service]:
+    items = k8s_paginate(client.core.list_service_for_all_namespaces)
+    return items
 
 
-@timeit
-def get_services(client: K8sClient, cluster: Dict, pods: List[Dict]) -> List[Dict]:
-    services = list()
-    for service in client.core.list_service_for_all_namespaces().items:
+def _format_service_selector(selector: dict[str, str]) -> str:
+    return json.dumps(selector)
+
+
+def _format_load_balancer_ingress(ingress: list[V1LoadBalancerIngress] | None) -> str:
+
+    def _format_ingress_ports(
+        ports: list[V1PortStatus] | None,
+    ) -> list[dict[str, Any]] | None:
+        if ports is None:
+            return None
+
+        ingress_ports = []
+        for port in ports:
+            ingress_ports.append(
+                {
+                    "error": port.port,
+                    "port": port.protocol,
+                    "protocol": port.ip,
+                }
+            )
+        return ingress_ports
+
+    if ingress is None:
+        return json.dumps(None)
+
+    loadbalancer_ingress = []
+    for item in ingress:
+        loadbalancer_ingress.append(
+            {
+                "hostname": item.hostname,
+                "ip": item.ip,
+                "ip_mode": item.ip_mode,
+                "ports": _format_ingress_ports(item.ports),
+            }
+        )
+    return json.dumps(loadbalancer_ingress)
+
+
+def transform_services(
+    services: list[V1Service], all_pods: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    services_list = []
+    for service in services:
         item = {
             "uid": service.metadata.uid,
             "name": service.metadata.name,
             "creation_timestamp": get_epoch(service.metadata.creation_timestamp),
             "deletion_timestamp": get_epoch(service.metadata.deletion_timestamp),
             "namespace": service.metadata.namespace,
-            "cluster_uid": cluster["uid"],
             "type": service.spec.type,
-            "selector": service.spec.selector,
+            "selector": _format_service_selector(service.spec.selector),
+            "cluster_ip": service.spec.cluster_ip,
             "load_balancer_ip": service.spec.load_balancer_ip,
         }
 
-        ingresses = service.status.load_balancer.ingress
-        for ingress in ingresses or list():
-            item.update({"ingress_host": ingress.hostname, "ingress_ip": ingress.ip})
-
-        service_pods = list()
-        for pod in pods:
-            is_service_pod = True if service.spec.selector else False
-            for selector in service.spec.selector or dict():
-                if (
-                    not pod.get("labels") or
-                    selector not in pod["labels"] or
-                    service.spec.selector[selector] != pod["labels"][selector]
-                ):
-                    is_service_pod = False
-                    break
-            if is_service_pod:
-                service_pods.append(pod)
-        item["pods"] = service_pods
-        services.append(item)
-    return services
-
-
-def load_services(session: Session, data: List[Dict], update_tag: int) -> None:
-    ingestion_cypher_query = """
-    UNWIND $services as k8service
-        MERGE (service:KubernetesService {id: k8service.uid})
-        ON CREATE SET service.firstseen = timestamp()
-        SET service.lastupdated = $update_tag,
-            service.name = k8service.name,
-            service.created_at = k8service.creation_timestamp,
-            service.deleted_at = k8service.deletion_timestamp,
-            service.type = k8service.type,
-            service.load_balancer_ip = k8service.load_balancer_ip,
-            service.ingress_host = k8service.ingress_host,
-            service.ingress_ip = k8service.ingress_ip
-        WITH service, k8service.namespace as ns, k8service.cluster_uid as cuid, k8service.pods as k8pods
-        MATCH (cluster:KubernetesCluster {id: cuid})-[:HAS_NAMESPACE]->(space:KubernetesNamespace {name: ns})
-        MERGE (space)-[rel1:HAS_SERVICE]->(service)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-        WITH service, k8pods
-        UNWIND k8pods as k8pod
-            MATCH (pod:KubernetesPod {id: k8pod.uid})
-            MERGE (service)-[rel2:SERVES_POD]->(pod)
-            ON CREATE SET rel2.firstseen = timestamp()
-            SET rel2.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes services.")
-    session.run(ingestion_cypher_query, services=data, update_tag=update_tag)
+        # TODO: instead of storing a json string, we should probably create seperate nodes for each ingress
+        if service.spec.type == "LoadBalancer":
+            if service.status.load_balancer:
+                item["load_balancer_ingress"] = _format_load_balancer_ingress(
+                    service.status.load_balancer.ingress
+                )
+
+        # check if pod labels match service selector and add pod_ids to item
+        pod_ids = []
+        for pod in all_pods:
+            if pod["namespace"] == service.metadata.namespace:
+                service_selector: dict[str, str] | None = service.spec.selector
+                pod_labels: dict[str, str] | None = json.loads(pod["labels"])
+
+                # check if pod labels match service selector
+                if pod_labels and service_selector:
+                    if all(
+                        service_selector[key] == pod_labels.get(key)
+                        for key in service_selector
+                    ):
+                        pod_ids.append(pod["uid"])
+
+        item["pod_ids"] = pod_ids
+
+        services_list.append(item)
+    return services_list
+
+
+def load_services(
+    session: neo4j.Session,
+    services: list[dict[str, Any]],
+    update_tag: int,
+    cluster_id: str,
+    cluster_name: str,
+) -> None:
+    logger.info(f"Loading {len(services)} KubernetesServices")
+    load(
+        session,
+        KubernetesServiceSchema(),
+        services,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+        CLUSTER_NAME=cluster_name,
+    )
+
+
+def cleanup(session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running cleanup job for KubernetesService")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesServiceSchema(), common_job_parameters
+    )
+    cleanup_job.run(session)
+
+
+@timeit
+def sync_services(
+    session: neo4j.Session,
+    client: K8sClient,
+    all_pods: list[dict[str, Any]],
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    services = get_services(client)
+    transformed_services = transform_services(services, all_pods)
+    load_services(
+        session=session,
+        services=transformed_services,
+        update_tag=update_tag,
+        cluster_id=common_job_parameters["CLUSTER_ID"],
+        cluster_name=client.name,
+    )
+    cleanup(session, common_job_parameters)

cartography/intel/kubernetes/util.py

@@ -1,11 +1,17 @@
+import logging
 from datetime import datetime
-from typing import List
-from typing import Union
+from typing import Any
+from typing import Callable
 
 from kubernetes import config
 from kubernetes.client import ApiClient
 from kubernetes.client import CoreV1Api
 from kubernetes.client import NetworkingV1Api
+from kubernetes.client import RbacAuthorizationV1Api
+from kubernetes.client import VersionApi
+from kubernetes.client.exceptions import ApiException
+
+logger = logging.getLogger(__name__)
 
 
 class KubernetesContextNotFound(Exception):
@@ -13,39 +19,161 @@ class KubernetesContextNotFound(Exception):
 
 
 class K8CoreApiClient(CoreV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8NetworkingApiClient(NetworkingV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
+class K8VersionApiClient(VersionApi):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
+class K8RbacApiClient(RbacAuthorizationV1Api):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8sClient:
-    def __init__(self, name: str) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        external_id: str | None = None,
+    ) -> None:
         self.name = name
-        self.core = K8CoreApiClient(self.name)
-        self.networking = K8NetworkingApiClient(self.name)
+        self.config_file = config_file
+        self.external_id = external_id
+        self.core = K8CoreApiClient(self.name, self.config_file)
+        self.networking = K8NetworkingApiClient(self.name, self.config_file)
+        self.version = K8VersionApiClient(self.name, self.config_file)
+        self.rbac = K8RbacApiClient(self.name, self.config_file)
 
 
-def get_k8s_clients(kubeconfig: str) -> List[K8sClient]:
+def get_k8s_clients(kubeconfig: str) -> list[K8sClient]:
+    # returns a tuple of (all contexts, current context)
     contexts, _ = config.list_kube_config_contexts(kubeconfig)
     if not contexts:
         raise KubernetesContextNotFound("No context found in kubeconfig.")
-    clients = list()
+
+    clients = []
     for context in contexts:
-        clients.append(K8sClient(context["name"]))
+        clients.append(
+            K8sClient(
+                context["name"],
+                kubeconfig,
+                external_id=context["context"].get("cluster"),
+            ),
+        )
     return clients
 
 
-def get_epoch(date: datetime) -> Union[int, None]:
+def get_epoch(date: datetime | None) -> int | None:
     if date:
-        return int(date.strftime("%s"))
+        return int(date.timestamp())
     return None
+
+
+def k8s_paginate(
+    list_func: Callable,
+    **kwargs: Any,
+) -> list[dict[str, Any]]:
+    """
+    Handles pagination for a Kubernetes API call.
+
+    :param list_func: The list function to call (e.g. client.core.list_pod_for_all_namespaces)
+    :param kwargs: Keyword arguments to pass to the list function (e.g. limit=100)
+    :return: A list of all resources returned by the list function
+    """
+    all_resources = []
+    continue_token = None
+    limit = kwargs.pop("limit", 100)
+    function_name = list_func.__name__
+
+    logger.debug(f"Starting pagination for {function_name} with limit {limit}.")
+
+    while True:
+        try:
+            if continue_token:
+                response = list_func(limit=limit, _continue=continue_token, **kwargs)
+            else:
+                response = list_func(limit=limit, **kwargs)
+
+            # Check if items exists on the response
+            if not hasattr(response, "items"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'items' attribute."
+                )
+                break
+
+            items_count = len(response.items)
+            all_resources.extend(response.items)
+
+            logger.debug(f"Retrieved {items_count} {function_name} resources")
+
+            # Check if metadata exists on the response
+            if not hasattr(response, "metadata"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'metadata' attribute."
+                )
+                break
+
+            continue_token = response.metadata._continue
+            if not continue_token:
+                logger.debug(f"No more {function_name} resources to retrieve.")
+                break
+
+        except ApiException as e:
+            logger.error(
+                f"Kubernetes API error retrieving {function_name} resources. {e}: {e.status} - {e.reason}"
+            )
+            break
+
+    logger.debug(
+        f"Completed pagination for {function_name}: retrieved {len(all_resources)} resources"
+    )
+    return all_resources

cartography/intel/lastpass/__init__.py

@@ -20,8 +20,8 @@ def start_lastpass_ingestion(neo4j_session: neo4j.Session, config: Config) -> No
 
     if not config.lastpass_cid or not config.lastpass_provhash:
         logger.info(
-            'Lastpass import is not configured - skipping this module. '
-            'See docs to configure.',
+            "Lastpass import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return
 

cartography/intel/lastpass/users.py

@@ -35,13 +35,17 @@ def sync(
 @timeit
 def get(lastpass_provhash: str, tenant_id: int) -> Dict[str, Any]:
     payload = {
-        'cid': tenant_id,
-        'provhash': lastpass_provhash,
-        'cmd': 'getuserdata',
-        'data': None,
+        "cid": tenant_id,
+        "provhash": lastpass_provhash,
+        "cmd": "getuserdata",
+        "data": None,
     }
     session = Session()
-    req = session.post('https://lastpass.com/enterpriseapi.php', data=payload, timeout=_TIMEOUT)
+    req = session.post(
+        "https://lastpass.com/enterpriseapi.php",
+        data=payload,
+        timeout=_TIMEOUT,
+    )
     req.raise_for_status()
     return req.json()
 
@@ -49,11 +53,13 @@ def get(lastpass_provhash: str, tenant_id: int) -> Dict[str, Any]:
 @timeit
 def transform(api_result: Dict[str, Any]) -> List[Dict[str, Any]]:
     result: List[Dict[str, Any]] = []
-    for uid, user in api_result['Users'].items():
+    for uid, user in api_result["Users"].items():
         n_user = user.copy()
-        n_user['id'] = int(uid)
-        for k in ('created', 'last_pw_change', 'last_login'):
-            n_user[k] = int(dt_parse.parse(user[k]).timestamp() * 1000) if user[k] else None
+        n_user["id"] = int(uid)
+        for k in ("created", "last_pw_change", "last_login"):
+            n_user[k] = (
+                int(dt_parse.parse(user[k]).timestamp() * 1000) if user[k] else None
+            )
         result.append(n_user)
     return result
 
@@ -68,7 +74,7 @@ def load_users(
     load(
         neo4j_session,
         LastpassTenantSchema(),
-        [{'id': tenant_id}],
+        [{"id": tenant_id}],
         lastupdated=update_tag,
     )
 
@@ -81,5 +87,10 @@ def load_users(
     )
 
 
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    GraphJob.from_node_schema(LastpassUserSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(LastpassUserSchema(), common_job_parameters).run(
+        neo4j_session,
+    )