vantage6 5.0.0a35__py3-none-any.whl → 5.0.0a37__py3-none-any.whl

vantage6/cli/node/task_cleanup/__init__.py

@@ -0,0 +1,153 @@
+import logging
+
+from kubernetes import client as k8s_client
+from kubernetes.client.rest import ApiException
+
+from vantage6.common import logger_name
+
+log = logging.getLogger(logger_name(__name__))
+
+
+def delete_job_related_pods(
+    run_id: int,
+    container_name: str,
+    namespace: str,
+    core_api: k8s_client.CoreV1Api,
+    batch_api: k8s_client.BatchV1Api,
+) -> None:
+    """
+    Deletes all the PODs created by a Kubernetes job in a given namespace
+
+    Parameters
+    ----------
+    run_id: int
+        Server run identifier
+    container_name: str
+        Name of the container
+    namespace: str
+        Namespace where the container is located
+    core_api: k8s_client.CoreV1Api
+        Kubernetes Core API instance
+    batch_api: k8s_client.BatchV1Api
+        Kubernetes Batch API instance
+    """
+    log.info(
+        "Cleaning up kubernetes Job %s (run_id = %s) and related PODs",
+        container_name,
+        run_id,
+    )
+
+    __delete_job(container_name, namespace, batch_api)
+
+    job_selector = f"job-name={container_name}"
+    job_pods_list = core_api.list_namespaced_pod(namespace, label_selector=job_selector)
+    for job_pod in job_pods_list.items:
+        __delete_pod(job_pod.metadata.name, namespace, core_api)
+
+    __delete_secret(container_name, namespace, core_api)
+
+
+def __delete_secret(
+    secret_name: str, namespace: str, core_api: k8s_client.CoreV1Api
+) -> None:
+    """
+    Deletes a secret in a given namespace
+
+    Parameters
+    ----------
+    secret_name: str
+        Name of the secret
+    namespace: str
+        Namespace where the secret is located
+    core_api: k8s_client.CoreV1Api
+        Kubernetes Core API instance
+    """
+    try:
+        core_api.delete_namespaced_secret(name=secret_name, namespace=namespace)
+        log.info(
+            "Removed kubernetes Secret %s in namespace %s",
+            secret_name,
+            namespace,
+        )
+    except ApiException as exc:
+        if exc.status == 404:
+            log.debug("No secret %s to remove in namespace %s", secret_name, namespace)
+        else:
+            log.error("Exception when deleting namespaced secret: %s", exc)
+
+
+def __delete_job(
+    job_name: str, namespace: str, batch_api: k8s_client.BatchV1Api
+) -> None:
+    """
+    Deletes a job in a given namespace
+
+    Parameters
+    ----------
+    job_name: str
+        Name of the job
+    namespace: str
+        Namespace where the job is located
+    batch_api: k8s_client.BatchV1Api
+        Kubernetes Batch API instance
+    """
+    log.info(
+        "Cleaning up kubernetes Job %s and related PODs",
+        job_name,
+    )
+    try:
+        # Check if the job exists before attempting to delete it
+        job = batch_api.read_namespaced_job(name=job_name, namespace=namespace)
+        if job:
+            batch_api.delete_namespaced_job(name=job_name, namespace=namespace)
+        else:
+            log.warning(
+                "Job %s not found in namespace %s, skipping deletion",
+                job_name,
+                namespace,
+            )
+    except ApiException as exc:
+        if exc.status == 404:
+            log.warning(
+                "Job %s not found in namespace %s, skipping deletion",
+                job_name,
+                namespace,
+            )
+        else:
+            log.error("Exception when deleting namespaced job: %s", exc)
+
+
+def __delete_pod(pod_name: str, namespace: str, core_api: k8s_client.CoreV1Api) -> None:
+    """
+    Deletes a job in a given namespace
+
+    Parameters
+    ----------
+    pod_name: str
+        Name of the job
+    namespace: str
+        Namespace where the job is located
+    core_api: k8s_client.CoreV1Api
+        Kubernetes Core API instance
+    """
+    log.info("Cleaning up kubernetes pod %s in namespace %s", pod_name, namespace)
+    try:
+        # Check if the job exists before attempting to delete it
+        job = core_api.read_namespaced_pod(name=pod_name, namespace=namespace)
+        if job:
+            core_api.delete_namespaced_pod(name=pod_name, namespace=namespace)
+        else:
+            log.warning(
+                "Pod %s not found in namespace %s, skipping deletion",
+                pod_name,
+                namespace,
+            )
+    except ApiException as exc:
+        if exc.status == 404:
+            log.warning(
+                "Pod %s not found in namespace %s, skipping deletion",
+                pod_name,
+                namespace,
+            )
+        else:
+            log.error("Exception when deleting namespaced job: %s", exc)

vantage6/cli/node/version.py

@@ -1,12 +1,13 @@
 import click
-import questionary as q
 import docker
+import questionary as q
 
 from vantage6.common import error
-from vantage6.common.globals import APPNAME
 from vantage6.common.docker.addons import check_docker_running
-from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+from vantage6.common.globals import APPNAME
+
 from vantage6.cli import __version__
+from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
 from vantage6.cli.node.common import find_running_node_names
 
 
@@ -16,7 +17,7 @@ from vantage6.cli.node.common import find_running_node_names
     "--system",
     "system_folders",
     flag_value=True,
-    help="Search for configuration in system folders rather than " "user folders",
+    help="Search for configuration in system folders rather than user folders",
 )
 @click.option(
     "--user",

vantage6/cli/prometheus/monitoring_manager.py

@@ -1,10 +1,12 @@
-import yaml
-import docker
 from pathlib import Path
 
-from vantage6.common import info, error
+import docker
+import yaml
+
+from vantage6.common import error, info
 from vantage6.common.docker.network_manager import NetworkManager
 from vantage6.common.globals import DEFAULT_PROMETHEUS_EXPORTER_PORT
+
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.globals import (
     DEFAULT_PROMETHEUS_IMAGE,

vantage6/cli/sandbox/config/base.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+import yaml
+
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.context import select_context_class
+from vantage6.cli.sandbox.populate.helpers.utils import replace_wsl_path
+
+
+class BaseSandboxConfigManager:
+    """
+    Base class for sandbox configuration managers.
+
+    Parameters
+    ----------
+    server_name : str
+        Name of the server.
+    custom_data_dir : Path | None
+        Path to the custom data directory. Useful on WSL because of mount issues for
+        default directories.
+    """
+
+    def __init__(self, server_name: str, custom_data_dir: Path | None) -> None:
+        self.server_name = server_name
+        self.custom_data_dir = Path(custom_data_dir) if custom_data_dir else None
+
+    @staticmethod
+    def _read_extra_config_file(extra_config_file: Path | None) -> dict:
+        """Reads extra configuration file.
+
+        Parameters
+        ----------
+        extra_config_file : Path | None
+            Path to file with additional configuration.
+
+        Returns
+        -------
+        dict
+            Extra configuration parsed from YAML. Empty dict if none provided.
+        """
+        if extra_config_file:
+            with open(extra_config_file, "r", encoding="utf-8") as f:
+                loaded = yaml.safe_load(f) or {}
+                if not isinstance(loaded, dict):
+                    # Ensure we always return a dictionary
+                    return {"value": loaded}
+                return loaded
+        return {}
+
+    def _create_and_get_data_dir(
+        self, instance_type: InstanceType, is_data_folder: bool = False
+    ) -> Path:
+        """
+        Create and get the data directory.
+
+        Parameters
+        ----------
+        instance_type: InstanceType
+            Type of vantage6 component
+        is_data_folder: bool
+            Whether or not to create the data folder or a config folder. This is only
+            used for node databases. Default is False.
+
+        Returns
+        -------
+        Path
+            Path to the data directory
+        """
+        ctx_class = select_context_class(instance_type)
+        folders = ctx_class.instance_folders(
+            instance_type=InstanceType.SERVER,
+            instance_name=self.server_name,
+            system_folders=False,
+        )
+        main_data_dir = (
+            Path(folders["dev"]) if not self.custom_data_dir else self.custom_data_dir
+        )
+
+        if instance_type == InstanceType.SERVER:
+            data_dir = main_data_dir / self.server_name / "server"
+        elif instance_type == InstanceType.ALGORITHM_STORE:
+            data_dir = main_data_dir / self.server_name / "store"
+        elif instance_type == InstanceType.NODE:
+            if is_data_folder:
+                last_subfolder = "data"
+            else:
+                last_subfolder = "node"
+            data_dir = main_data_dir / self.server_name / last_subfolder
+        else:
+            raise ValueError(f"Invalid instance type to get data dir: {instance_type}")
+
+        # For the directory to be created, ensure that if a WSL path is used, the path
+        # is converted to /mnt/wsl to create the directory on the host (not
+        # /run/desktop/mnt/host/wsl as will raise non-existent directory errors)
+        data_dir = replace_wsl_path(data_dir, to_mnt_wsl=True)
+        data_dir.mkdir(parents=True, exist_ok=True)
+        # now ensure that the wsl path is properly replaced to /run/desktop/mnt/host/wsl
+        # if it is a WSL path, because that path will be used in the node configuration
+        # files and is required to successfully mount the volumes.
+        return replace_wsl_path(data_dir, to_mnt_wsl=False)

vantage6/cli/sandbox/config/core.py

@@ -0,0 +1,300 @@
+from pathlib import Path
+
+from vantage6.common.globals import HTTP_LOCALHOST, InstanceType, Ports
+
+from vantage6.cli.common.new import new
+from vantage6.cli.sandbox.config.base import BaseSandboxConfigManager
+
+
+class CoreSandboxConfigManager(BaseSandboxConfigManager):
+    """
+    Class to store the sandbox configurations.
+
+    Parameters
+    ----------
+    server_name : str
+        Name of the server.
+    server_port : int
+        Port of the server.
+    ui_port : int
+        Port of the UI.
+    algorithm_store_port : int
+        Port of the algorithm store.
+    server_image : str | None
+        Image of the server.
+    store_image : str | None
+        Image of the algorithm store.
+    ui_image : str | None
+        Image of the UI.
+    extra_server_config : Path | None
+        Path to the extra server configuration file.
+    extra_store_config : Path | None
+        Path to the extra algorithm store configuration file.
+    context : str | None
+        Kubernetes context.
+    namespace : str | None
+        Kubernetes namespace.
+    k8s_node_name : str
+        Kubernetes node name.
+    custom_data_dir : Path | None
+        Path to the custom data directory. Useful on WSL because of mount issues for
+        default directories.
+    """
+
+    def __init__(
+        self,
+        server_name: str,
+        server_port: int,
+        ui_port: int,
+        algorithm_store_port: int,
+        server_image: str | None,
+        store_image: str | None,
+        ui_image: str | None,
+        extra_server_config: Path | None,
+        extra_store_config: Path | None,
+        extra_auth_config: Path | None,
+        context: str,
+        namespace: str,
+        k8s_node_name: str,
+        custom_data_dir: Path | None = None,
+    ) -> None:
+        super().__init__(server_name, custom_data_dir)
+
+        self.server_port = server_port
+        self.ui_port = ui_port
+        self.algorithm_store_port = algorithm_store_port
+        self.server_image = server_image
+        self.store_image = store_image
+        self.ui_image = ui_image
+        self.extra_server_config = extra_server_config
+        self.extra_store_config = extra_store_config
+        self.extra_auth_config = extra_auth_config
+        self.context = context
+        self.namespace = namespace
+
+        self.server_config_file = None
+        self.store_config_file = None
+        self.auth_config_file = None
+        self.k8s_node_name = k8s_node_name
+
+    def generate_server_configs(self) -> None:
+        """Generates the demo network."""
+
+        self._create_auth_config()
+
+        self._create_vserver_config()
+
+        self._create_algo_store_config()
+
+    def __server_config_return_func(self, extra_config: dict, data_dir: Path) -> dict:
+        """
+        Return a dict with server configuration values to be used in creating the
+        config file.
+
+        Parameters
+        ----------
+        extra_config : dict
+            Extra configuration (parsed from YAML) to be added to the server
+            configuration.
+        data_dir : Path
+            Path to the data directory.
+
+        Returns
+        -------
+        dict
+            Dictionary with server configuration values.
+        """
+        store_service = (
+            f"vantage6-{self.server_name}-store-user-algorithm-store-store-service"
+        )
+        store_address = (
+            f"http://{store_service}.{self.namespace}.svc.cluster.local:"
+            f"{Ports.DEV_ALGO_STORE}"
+        )
+        config = {
+            "server": {
+                "baseUrl": f"{HTTP_LOCALHOST}:{self.server_port}",
+                # TODO: v5+ set to latest v5 image
+                # TODO make this configurable
+                "image": (
+                    self.server_image
+                    or "harbor2.vantage6.ai/infrastructure/server:5.0.0a36"
+                ),
+                "algorithm_stores": [
+                    {
+                        "name": "Local store",
+                        "url": store_address,
+                        "api_path": "/store",
+                    }
+                ],
+                "logging": {
+                    "level": "DEBUG",
+                },
+                "jwt": {
+                    "secret": "development-constant-secret!",
+                },
+                "dev": {
+                    "host_uri": (
+                        "host.docker.internal"
+                        if self.k8s_node_name == "docker-desktop"
+                        else "172.17.0.1"
+                    ),
+                    "store_address": store_address,
+                },
+                "keycloakUrl": (
+                    f"http://vantage6-{self.server_name}-auth-user-auth-keycloak."
+                    f"{self.namespace}.svc.cluster.local"
+                ),
+            },
+            "rabbitmq": {},
+            "database": {
+                "volumePath": str(data_dir),
+                "k8sNodeName": self.k8s_node_name,
+            },
+            "ui": {
+                "port": self.ui_port,
+                # TODO: v5+ set to latest v5 image
+                # TODO: make this configurable
+                "image": (
+                    self.ui_image or "harbor2.vantage6.ai/infrastructure/ui:5.0.0a36"
+                ),
+            },
+        }
+
+        # merge the extra config with the server config
+        if extra_config is not None:
+            config.update(extra_config)
+
+        return config
+
+    def _create_vserver_config(self) -> None:
+        """Creates server configuration file (YAML)."""
+
+        data_dir = self._create_and_get_data_dir(instance_type=InstanceType.SERVER)
+
+        extra_config = self._read_extra_config_file(self.extra_server_config)
+        if self.ui_image is not None:
+            ui_config = extra_config.get("ui", {}) if extra_config is not None else {}
+            ui_config["image"] = self.ui_image
+            extra_config["ui"] = ui_config
+
+        # Create the server config file
+        self.server_config_file = new(
+            config_producing_func=self.__server_config_return_func,
+            config_producing_func_args=(extra_config, data_dir),
+            name=self.server_name,
+            system_folders=False,
+            namespace=self.namespace,
+            context=self.context,
+            type_=InstanceType.SERVER,
+            is_sandbox=True,
+        )
+
+    def _create_algo_store_config(self) -> None:
+        """Create algorithm store configuration file (YAML)."""
+
+        extra_config = self._read_extra_config_file(self.extra_store_config)
+
+        data_dir = self._create_and_get_data_dir(InstanceType.ALGORITHM_STORE)
+
+        self.store_config_file = new(
+            config_producing_func=self.__algo_store_config_return_func,
+            config_producing_func_args=(extra_config, data_dir),
+            name=f"{self.server_name}-store",
+            system_folders=False,
+            namespace=self.namespace,
+            context=self.context,
+            type_=InstanceType.ALGORITHM_STORE,
+            is_sandbox=True,
+        )
+
+    def __algo_store_config_return_func(
+        self, extra_config: dict, data_dir: Path
+    ) -> dict:
+        """
+        Return a dict with algorithm store configuration values to be used in creating
+        the config file.
+
+        Returns
+        -------
+        dict
+            Dictionary with algorithm store configuration values.
+        """
+        config = {
+            "store": {
+                "internal": {
+                    "port": self.algorithm_store_port,
+                },
+                "logging": {
+                    "level": "DEBUG",
+                },
+                "vantage6ServerUri": f"{HTTP_LOCALHOST}:{self.server_port}",
+                "image": (
+                    self.store_image
+                    or "harbor2.vantage6.ai/infrastructure/algorithm-store:5.0.0a36"
+                ),
+                "keycloakUrl": (
+                    f"http://vantage6-{self.server_name}-auth-user-auth-keycloak."
+                    f"{self.namespace}.svc.cluster.local"
+                ),
+                "policies": {
+                    "allowLocalhost": True,
+                    "assignReviewOwnAlgorithm": True,
+                },
+                "dev": {
+                    "host_uri": (
+                        "host.docker.internal"
+                        if self.k8s_node_name == "docker-desktop"
+                        else "172.17.0.1"
+                    ),
+                    "disable_review": True,
+                    "review_own_algorithm": True,
+                },
+            },
+            "database": {
+                "volumePath": str(data_dir),
+                "k8sNodeName": self.k8s_node_name,
+            },
+        }
+
+        # merge the extra config with the algorithm store config
+        if extra_config is not None:
+            config.update(extra_config)
+
+        return config
+
+    def _create_auth_config(self) -> None:
+        """Create auth configuration file (YAML)."""
+        self.auth_config_file = new(
+            config_producing_func=self.__auth_config_return_func,
+            config_producing_func_args=(self.extra_auth_config,),
+            name=f"{self.server_name}-auth",
+            system_folders=False,
+            namespace=self.namespace,
+            context=self.context,
+            type_=InstanceType.AUTH,
+            is_sandbox=True,
+        )
+
+    def __auth_config_return_func(self, extra_config: dict) -> dict:
+        """
+        Return a dict with auth configuration values to be used in creating the
+        config file.
+        """
+
+        config = {
+            "keycloak": {
+                "production": False,
+                "no_password_update_required": True,
+                "redirectUris": [
+                    f"{HTTP_LOCALHOST}:7600",
+                    f"{HTTP_LOCALHOST}:7681",
+                ],
+            },
+        }
+
+        # merge the extra config with the auth config
+        if extra_config is not None:
+            config.update(extra_config)
+
+        return config