cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/intel/kubernetes/util.py

@@ -1,11 +1,16 @@
+import logging
 from datetime import datetime
-from typing import List
-from typing import Union
+from typing import Any
+from typing import Callable
 
 from kubernetes import config
 from kubernetes.client import ApiClient
 from kubernetes.client import CoreV1Api
 from kubernetes.client import NetworkingV1Api
+from kubernetes.client import VersionApi
+from kubernetes.client.exceptions import ApiException
+
+logger = logging.getLogger(__name__)
 
 
 class KubernetesContextNotFound(Exception):
@@ -13,39 +18,145 @@ class KubernetesContextNotFound(Exception):
 
 
 class K8CoreApiClient(CoreV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8NetworkingApiClient(NetworkingV1Api):
-    def __init__(self, name: str, api_client: ApiClient = None) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
+        self.name = name
+        if not api_client:
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
+        super().__init__(api_client=api_client)
+
+
+class K8VersionApiClient(VersionApi):
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        api_client: ApiClient | None = None,
+    ) -> None:
         self.name = name
         if not api_client:
-            api_client = config.new_client_from_config(context=name)
+            api_client = config.new_client_from_config(
+                context=name, config_file=config_file
+            )
         super().__init__(api_client=api_client)
 
 
 class K8sClient:
-    def __init__(self, name: str) -> None:
+    def __init__(
+        self,
+        name: str,
+        config_file: str,
+        external_id: str | None = None,
+    ) -> None:
         self.name = name
-        self.core = K8CoreApiClient(self.name)
-        self.networking = K8NetworkingApiClient(self.name)
+        self.config_file = config_file
+        self.external_id = external_id
+        self.core = K8CoreApiClient(self.name, self.config_file)
+        self.networking = K8NetworkingApiClient(self.name, self.config_file)
+        self.version = K8VersionApiClient(self.name, self.config_file)
 
 
-def get_k8s_clients(kubeconfig: str) -> List[K8sClient]:
+def get_k8s_clients(kubeconfig: str) -> list[K8sClient]:
+    # returns a tuple of (all contexts, current context)
     contexts, _ = config.list_kube_config_contexts(kubeconfig)
     if not contexts:
         raise KubernetesContextNotFound("No context found in kubeconfig.")
-    clients = list()
+
+    clients = []
     for context in contexts:
-        clients.append(K8sClient(context["name"]))
+        clients.append(
+            K8sClient(
+                context["name"],
+                kubeconfig,
+                external_id=context["context"].get("cluster"),
+            ),
+        )
     return clients
 
 
-def get_epoch(date: datetime) -> Union[int, None]:
+def get_epoch(date: datetime | None) -> int | None:
     if date:
-        return int(date.strftime("%s"))
+        return int(date.timestamp())
     return None
+
+
+def k8s_paginate(
+    list_func: Callable,
+    **kwargs: Any,
+) -> list[dict[str, Any]]:
+    """
+    Handles pagination for a Kubernetes API call.
+
+    :param list_func: The list function to call (e.g. client.core.list_pod_for_all_namespaces)
+    :param kwargs: Keyword arguments to pass to the list function (e.g. limit=100)
+    :return: A list of all resources returned by the list function
+    """
+    all_resources = []
+    continue_token = None
+    limit = kwargs.pop("limit", 100)
+    function_name = list_func.__name__
+
+    logger.debug(f"Starting pagination for {function_name} with limit {limit}.")
+
+    while True:
+        try:
+            if continue_token:
+                response = list_func(limit=limit, _continue=continue_token, **kwargs)
+            else:
+                response = list_func(limit=limit, **kwargs)
+
+            # Check if items exists on the response
+            if not hasattr(response, "items"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'items' attribute."
+                )
+                break
+
+            items_count = len(response.items)
+            all_resources.extend(response.items)
+
+            logger.debug(f"Retrieved {items_count} {function_name} resources")
+
+            # Check if metadata exists on the response
+            if not hasattr(response, "metadata"):
+                logger.warning(
+                    f"Response from {function_name} does not contain 'metadata' attribute."
+                )
+                break
+
+            continue_token = response.metadata._continue
+            if not continue_token:
+                logger.debug(f"No more {function_name} resources to retrieve.")
+                break
+
+        except ApiException as e:
+            logger.error(
+                f"Kubernetes API error retrieving {function_name} resources. {e}: {e.status} - {e.reason}"
+            )
+            break
+
+    logger.debug(
+        f"Completed pagination for {function_name}: retrieved {len(all_resources)} resources"
+    )
+    return all_resources

cartography/intel/scaleway/__init__.py

@@ -0,0 +1,127 @@
+import logging
+
+import neo4j
+import scaleway
+
+import cartography.intel.scaleway.iam.apikeys
+import cartography.intel.scaleway.iam.applications
+import cartography.intel.scaleway.iam.groups
+import cartography.intel.scaleway.iam.users
+import cartography.intel.scaleway.instances.flexibleips
+import cartography.intel.scaleway.instances.instances
+import cartography.intel.scaleway.projects
+import cartography.intel.scaleway.storage.snapshots
+import cartography.intel.scaleway.storage.volumes
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_scaleway_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Scaleway data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if (
+        not config.scaleway_access_key
+        or not config.scaleway_secret_key
+        or not config.scaleway_org
+    ):
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = scaleway.Client(
+        access_key=config.scaleway_access_key,
+        secret_key=config.scaleway_secret_key,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "ORG_ID": config.scaleway_org,
+    }
+
+    # Organization level
+    projects = cartography.intel.scaleway.projects.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    projects_id = [project["id"] for project in projects]
+    cartography.intel.scaleway.iam.users.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.applications.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.groups.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.apikeys.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+
+    # Storage
+    cartography.intel.scaleway.storage.volumes.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.storage.snapshots.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+
+    # Instances
+    # DISABLED due to https://github.com/scaleway/scaleway-sdk-python/issues/1040
+    """
+    cartography.intel.scaleway.instances.flexibleips.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    """
+    cartography.intel.scaleway.instances.instances.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )

cartography/intel/scaleway/iam/apikeys.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import APIKey
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.apikey import ScalewayApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    apikeys = get(client, org_id)
+    formatted_apikeys = transform_apikeys(apikeys)
+    load_apikeys(neo4j_session, formatted_apikeys, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[APIKey]:
+    api = IamV1Alpha1API(client)
+    return api.list_api_keys_all(organization_id=org_id)
+
+
+def transform_apikeys(apikeys: list[APIKey]) -> list[dict[str, Any]]:
+    formatted_apikeys = []
+    for apikey in apikeys:
+        formatted_apikeys.append(scaleway_obj_to_dict(apikey))
+    return formatted_apikeys
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway ApiKeys into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/applications.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Application
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.application import ScalewayApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    applications = get(client, org_id)
+    formatted_applications = transform_applications(applications)
+    load_applications(neo4j_session, formatted_applications, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Application]:
+    api = IamV1Alpha1API(client)
+    return api.list_applications_all(organization_id=org_id)
+
+
+def transform_applications(applications: list[Application]) -> list[dict[str, Any]]:
+    formatted_applications = []
+    for application in applications:
+        formatted_applications.append(scaleway_obj_to_dict(application))
+    return formatted_applications
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Applications into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/groups.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Group
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.group import ScalewayGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    groups = get(client, org_id)
+    formatted_groups = transform_groups(groups)
+    load_groups(neo4j_session, formatted_groups, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Group]:
+    api = IamV1Alpha1API(client)
+    return api.list_groups_all(organization_id=org_id)
+
+
+def transform_groups(groups: list[Group]) -> list[dict[str, Any]]:
+    formatted_groups = []
+    for group in groups:
+        formatted_groups.append(scaleway_obj_to_dict(group))
+    return formatted_groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Groups into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/users.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+from scaleway.iam.v1alpha1 import User
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.user import ScalewayUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    users = get(client, org_id)
+    formatted_users = transform_users(users)
+    load_users(neo4j_session, formatted_users, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[User]:
+    api = IamV1Alpha1API(client)
+    return api.list_users_all(organization_id=org_id)
+
+
+def transform_users(users: list[User]) -> list[dict[str, Any]]:
+    formatted_users = []
+    for user in users:
+        formatted_users.append(scaleway_obj_to_dict(user))
+    return formatted_users
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Users into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/instances/flexibleips.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Ip
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.flexibleip import ScalewayFlexibleIpSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    flexibleips = get(client, org_id)
+    flexibleips_by_project = transform_flexibleips(flexibleips)
+    load_flexibleips(neo4j_session, flexibleips_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Ip]:
+    api = InstanceV1API(client)
+    return api.list_ips_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_flexibleips(
+    flexibleips: list[Ip],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for flexibleip in flexibleips:
+        project_id = flexibleip.project
+        formatted_flexibleip = scaleway_obj_to_dict(flexibleip)
+        result.setdefault(project_id, []).append(formatted_flexibleip)
+    return result
+
+
+@timeit
+def load_flexibleips(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, flexibleips in data.items():
+        logger.info(
+            "Loading %d Scaleway Flexible IPs in project '%s' into Neo4j.",
+            len(flexibleips),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayFlexibleIpSchema(),
+            flexibleips,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayFlexibleIpSchema(), scopped_job_parameters
+        ).run(neo4j_session)