vantage6 5.0.0a37__py3-none-any.whl → 5.0.0a38__py3-none-any.whl

vantage6/cli/sandbox/config/node.py

@@ -214,7 +214,7 @@ class NodeSandboxConfigManager(BaseSandboxConfigManager):
         config_name = f"{self.server_name}-{node_name}"
 
         path_to_data_dir = self._create_and_get_data_dir(
-            InstanceType.NODE, is_data_folder=True
+            InstanceType.NODE, is_data_folder=True, node_name=node_name
         )
 
         # delete old node config if it exists
@@ -262,7 +262,7 @@ class NodeSandboxConfigManager(BaseSandboxConfigManager):
                 "image": (
                     self.node_image
                     # TODO v5+ update
-                    or "harbor2.vantage6.ai/infrastructure/node:5.0.0a36"
+                    or "harbor2.vantage6.ai/infrastructure/node:5.0.0a37"
                 ),
                 "logging": {
                     "level": "DEBUG",
@@ -272,9 +272,6 @@ class NodeSandboxConfigManager(BaseSandboxConfigManager):
                     f"http://vantage6-{self.server_name}-auth-user-auth-keycloak."
                     f"{self.namespace}.svc.cluster.local"
                 ),
-                "dev": {
-                    "task_dir_extension": str(path_to_data_dir),
-                },
                 "persistence": {
                     "tasks": {
                         "hostPath": str(path_to_data_dir),

vantage6/cli/sandbox/remove.py

@@ -4,19 +4,18 @@ from shutil import rmtree
 
 import click
 
-from vantage6.common import error, info
+from vantage6.common import error, warning
 from vantage6.common.globals import InstanceType
 
+from vantage6.cli.auth.remove import auth_remove
 from vantage6.cli.common.remove import execute_remove
 from vantage6.cli.configuration_create import select_configuration_questionnaire
 from vantage6.cli.context import get_context
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
 from vantage6.cli.context.auth import AuthContext
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.context.server import ServerContext
 from vantage6.cli.globals import InfraComponentName
 from vantage6.cli.server.remove import cli_server_remove
-from vantage6.cli.utils import remove_file
 
 
 @click.command()
@@ -60,28 +59,6 @@ def cli_sandbox_remove(
 
     ctx = get_context(InstanceType.SERVER, name, system_folders=False, is_sandbox=True)
 
-    # remove the server
-    # Note that this also checks if the server is running. Therefore, it is prevented
-    # that a running sandbox is removed.
-    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-        handler.close()
-    click_ctx.invoke(
-        cli_server_remove, ctx=ctx, name=name, system_folders=False, force=True
-    )
-
-    # removing the server import config
-    info("Deleting demo import config file")
-    server_configs = ServerContext.instance_folders(
-        InstanceType.SERVER, ctx.name, system_folders=False
-    )
-    import_config_to_del = Path(server_configs["dev"]) / f"{ctx.name}.yaml"
-    remove_file(import_config_to_del, "import_configuration")
-
-    # also remove the server folder
-    server_folder = server_configs["data"]
-    if server_folder.is_dir():
-        rmtree(server_folder)
-
     # remove the store folder
     store_configs = AlgorithmStoreContext.instance_folders(
         InstanceType.ALGORITHM_STORE,
@@ -117,20 +94,13 @@ def cli_sandbox_remove(
     if auth_folder.is_dir():
         rmtree(auth_folder)
 
-    # remove the auth config file
+    # remove the auth service
     auth_ctx = AuthContext(
         instance_name=f"{ctx.name}-auth",
         system_folders=False,
         is_sandbox=True,
     )
-    execute_remove(
-        auth_ctx,
-        InstanceType.AUTH,
-        InfraComponentName.AUTH,
-        f"{ctx.name}-auth",
-        system_folders=False,
-        force=True,
-    )
+    auth_remove(auth_ctx, f"{ctx.name}-auth", system_folders=False, force=True)
 
     # remove the nodes
     NodeContext.LOGGING_ENABLED = False
@@ -168,6 +138,18 @@ def cli_sandbox_remove(
 
     # remove data files attached to the network
     data_dirs_nodes = NodeContext.instance_folders("node", "", False)["dev"]
-    rmtree(Path(data_dirs_nodes / ctx.name))
+    try:
+        rmtree(Path(data_dirs_nodes / ctx.name))
+    except Exception as e:
+        warning(f"Failed to delete data directory {data_dirs_nodes / ctx.name}: {e}")
 
+    # remove the server last - if anything goes wrong, the server is still there so the
+    # user can still retry the removal.
+    # Note that this also checks if the server is running. Therefore, it is prevented
+    # that a running sandbox is removed.
+    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
+        handler.close()
+    click_ctx.invoke(
+        cli_server_remove, ctx=ctx, name=name, system_folders=False, force=True
+    )
     # TODO remove the right data in the custom data directory if it is provided

vantage6/cli/sandbox/start.py

@@ -44,6 +44,12 @@ from vantage6.cli.server.start import cli_server_start
     help="Generate this number of nodes in the development network. Only used if "
     "--re-initialize flag is provided.",
 )
+@click.option(
+    "--node-image",
+    type=str,
+    default=None,
+    help="Node image to use. Only used if --re-initialize flag is provided.",
+)
 @click.option(
     "--extra-node-config",
     type=click.Path("rb"),
@@ -80,6 +86,7 @@ def cli_sandbox_start(
     local_chart_dir: Path | None,
     re_initialize: bool,
     num_nodes: int,
+    node_image: str | None,
     extra_node_config: Path | None,
     add_dataset: tuple[str, Path] | None,
     custom_data_dir: Path | None,
@@ -101,6 +108,7 @@ def cli_sandbox_start(
         namespace=namespace,
         num_nodes=num_nodes,
         initialize=re_initialize,
+        node_image=node_image,
         extra_node_config=extra_node_config,
         add_dataset=add_dataset,
         custom_data_dir=custom_data_dir,

vantage6/cli/sandbox/stop.py

@@ -83,7 +83,7 @@ def cli_sandbox_stop(
         is_sandbox=True,
     )
 
-    # TODO: stop the auth service
+    # stop the auth service
     cmd = [
         "v6",
         "auth",

vantage6/cli/server/attach.py

@@ -2,13 +2,38 @@ import click
 
 from vantage6.common import info
 
-from vantage6.cli.common.utils import attach_logs
+from vantage6.cli.common.attach import attach_logs
+from vantage6.cli.context import InstanceType
+from vantage6.cli.globals import InfraComponentName
 
 
 @click.command()
-def cli_server_attach() -> None:
+@click.option("-n", "--name", default=None, help="Name of the configuration")
+@click.option("--system", "system_folders", flag_value=True, help="Use system folders")
+@click.option("--user", "system_folders", flag_value=False, help="Use user folders")
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click.option(
+    "--sandbox", "is_sandbox", flag_value=True, help="Attach to a sandbox environment"
+)
+def cli_server_attach(
+    name: str | None,
+    system_folders: bool,
+    context: str,
+    namespace: str,
+    is_sandbox: bool,
+) -> None:
     """
     Show the server logs in the current console.
     """
     info("Attaching to server logs...")
-    attach_logs("app=vantage6-server", "component=vantage6-server")
+    attach_logs(
+        name,
+        instance_type=InstanceType.SERVER,
+        infra_component=InfraComponentName.SERVER,
+        system_folders=system_folders,
+        context=context,
+        namespace=namespace,
+        is_sandbox=is_sandbox,
+        additional_labels="component=vantage6-server",
+    )

vantage6/cli/server/list.py

@@ -2,7 +2,7 @@ import click
 
 from vantage6.common.globals import InstanceType
 
-from vantage6.cli.common.utils import get_server_configuration_list
+from vantage6.cli.common.list import get_configuration_list
 
 
 @click.command()
@@ -10,4 +10,4 @@ def cli_server_configuration_list() -> None:
     """
     Print the available server configurations.
     """
-    get_server_configuration_list(InstanceType.SERVER)
+    get_configuration_list(InstanceType.SERVER)

vantage6/cli/server/start.py

@@ -3,6 +3,7 @@ import click
 from vantage6.common import info
 from vantage6.common.globals import InstanceType, Ports
 
+from vantage6.cli.common.attach import attach_logs
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.common.start import (
     helm_install,
@@ -10,12 +11,11 @@ from vantage6.cli.common.start import (
     start_port_forward,
 )
 from vantage6.cli.common.utils import (
-    attach_logs,
     create_directory_if_not_exists,
     select_context_and_namespace,
 )
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.globals import ChartName
+from vantage6.cli.globals import ChartName, InfraComponentName
 
 
 @click.command()
@@ -95,4 +95,12 @@ def cli_server_start(
     )
 
     if attach:
-        attach_logs("app=vantage6-server", "component=vantage6-server")
+        attach_logs(
+            name,
+            instance_type=InstanceType.SERVER,
+            infra_component=InfraComponentName.SERVER,
+            system_folders=system_folders,
+            context=context,
+            namespace=namespace,
+            is_sandbox=ctx.is_sandbox,
+        )

vantage6/cli/server/version.py

@@ -1,12 +1,11 @@
 import click
-import docker
+import requests
 
-from vantage6.common import error
-from vantage6.common.docker.addons import check_docker_running
+from vantage6.common import error, info
 from vantage6.common.globals import InstanceType
 
 from vantage6.cli import __version__
-from vantage6.cli.common.utils import get_running_servers, get_server_name
+from vantage6.cli.common.version import get_and_select_ctx
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 
 
@@ -16,22 +15,36 @@ from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 @click.option(
     "--user", "system_folders", flag_value=False, default=DEFAULT_SERVER_SYSTEM_FOLDERS
 )
-def cli_server_version(name: str, system_folders: bool) -> None:
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click.option(
+    "--sandbox", "is_sandbox", flag_value=True, help="Is this a sandbox environment?"
+)
+def cli_server_version(
+    name: str, system_folders: bool, context: str, namespace: str, is_sandbox: bool
+) -> None:
     """
     Print the version of the vantage6 server.
     """
-    check_docker_running()
-    client = docker.from_env()
-
-    running_server_names = get_running_servers(client, InstanceType.SERVER)
-
-    name = get_server_name(
-        name, system_folders, running_server_names, InstanceType.SERVER
+    ctx = get_and_select_ctx(
+        InstanceType.SERVER, name, system_folders, context, namespace, is_sandbox
     )
+    server_config = ctx.config.get("server", {})
+    base_url = server_config.get("baseUrl", "")
+    api_path = server_config.get("apiPath", "")
+    if not base_url:
+        error("No base URL found in server configuration.")
+        return
+    if not api_path:
+        error("No API path found in server configuration.")
+        return
+
+    response = requests.get(f"{base_url}{api_path}/version")
+    if response.status_code != 200:
+        error("Failed to get server version.")
+        return
+    server_version = response.json().get("version", "")
 
-    if name in running_server_names:
-        container = client.containers.get(name)
-        version = container.exec_run(cmd="vserver-local version", stdout=True)
-        click.echo({"server": version.output.decode("utf-8"), "cli": __version__})
-    else:
-        error(f"Server {name} is not running! Cannot provide version...")
+    info("")
+    info(f"Server version: {server_version}")
+    info(f"CLI version: {__version__}")

vantage6/cli/template/algo_store_config.j2

@@ -102,6 +102,9 @@ store:
     port: 7602
   {% endif %}
 
+  # The port to expose the store on in the cluster
+  port: {{ store.port | default(7602) }}
+
   logging:
     # Controls the logging output level. Could be one of the following
     # levels: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET

vantage6/cli/template/node_config.j2

@@ -79,6 +79,8 @@ node:
       name: engineio.server
     - level: warning
       name: sqlalchemy.engine
+    - level: warning
+      name: kubernetes.client.rest
     {% endif %}
 
   {% if node.encryption is defined %}

vantage6/cli/use/context.py

@@ -1,11 +1,12 @@
 import click
 import questionary
-from kubernetes import config
+from kubernetes import client, config
 
 from vantage6.common import error
 
 from vantage6.cli.config import CliConfig
 from vantage6.cli.utils import switch_context_and_namespace
+from vantage6.cli.utils_kubernetes import configure_kubernetes_client_for_microk8s
 
 
 @click.command()
@@ -14,6 +15,12 @@ def cli_use_context(context: str):
     """
     Set which Kubernetes context to use.
     """
+    # Configure for MicroK8s if needed
+    config.load_kube_config()
+    cfg = client.Configuration.get_default_copy()
+    configure_kubernetes_client_for_microk8s(cfg)
+    client.Configuration.set_default(cfg)
+
     # Get available contexts
     contexts, active_context = config.list_kube_config_contexts()
     context_names = [ctx["name"] for ctx in contexts]

vantage6/cli/use/namespace.py

@@ -1,11 +1,14 @@
 import click
 import questionary
-from kubernetes import client, config
+from kubernetes import client
 
 from vantage6.common import error
 
 from vantage6.cli.config import CliConfig
 from vantage6.cli.utils import switch_context_and_namespace
+from vantage6.cli.utils_kubernetes import (
+    get_core_api_with_ssl_handling,
+)
 
 
 @click.command()
@@ -16,15 +19,15 @@ def cli_use_namespace(namespace: str):
 
     The namespace will be created if it does not exist.
     """
-    # Load the active context configuration
-    config.load_kube_config()
+    # Configure for MicroK8s if needed
+    core_api, _ = get_core_api_with_ssl_handling()
 
     try:
-        v1 = client.CoreV1Api()
-        namespace_list = v1.list_namespace()
+        namespace_list = core_api.list_namespace()
     except Exception:
         error(
-            "Failed to connect to Kubernetes cluster. Check if the cluster is running and reachable."
+            "Failed to connect to Kubernetes cluster. Check if the cluster is running "
+            "and reachable."
         )
         return
 
@@ -46,7 +49,7 @@ def cli_use_namespace(namespace: str):
         namespace_body = client.V1Namespace(
             metadata=client.V1ObjectMeta(name=namespace)
         )
-        v1.create_namespace(namespace_body)
+        core_api.create_namespace(namespace_body)
 
     # Switch to the selected namespace for current context
     switch_context_and_namespace(namespace=namespace)

vantage6/cli/utils_kubernetes.py

@@ -0,0 +1,270 @@
+"""
+Kubernetes utility functions for Vantage6 CLI.
+
+This module provides utilities for handling Kubernetes client configuration,
+especially for MicroK8s environments that may have SSL certificate issues.
+
+The issue is that the python kubernetes client does not automatically use the
+certificate from microk8s, so it is manually configured in this module. Note that this
+is only an issue for the CLI and not for running nodes/servers/stores/etc., because
+those are started using the `kubectl` command, which includes the microk8scertificate
+automatically.
+"""
+
+import base64
+import ssl
+from pathlib import Path
+
+from kubernetes import client, config
+from kubernetes.config.config_exception import ConfigException
+
+from vantage6.common import warning
+
+
+def configure_kubernetes_client_for_microk8s(cfg: client.Configuration) -> None:
+    """
+    Configure the Kubernetes client to handle MicroK8s SSL certificate issues.
+
+    This function detects if we're using MicroK8s and configures the client
+    to handle SSL certificates appropriately for both development and production.
+
+    Parameters
+    ----------
+    cfg : client.Configuration
+        The Kubernetes client configuration to configure
+    """
+    # Check if we're using MicroK8s by looking at the current context
+    if is_microk8s_context():
+        _configure_microk8s_ssl(cfg)
+
+
+def _configure_microk8s_ssl(cfg: client.Configuration) -> None:
+    """
+    Configure SSL settings for MicroK8s.
+
+    This function handles MicroK8s SSL certificates in a secure way by:
+    1. First trying to use the MicroK8s certificate directly
+    2. If that fails, falling back to a more lenient but still secure approach
+    """
+    try:
+        # Try to get the MicroK8s certificate and use it properly
+        if cert_path := _get_microk8s_certificate_path():
+            _configure_with_certificate(cert_path, cfg)
+        else:
+            warning(
+                "MicroK8s certificate not found. You may run into errors when using "
+                "the CLI."
+            )
+
+    except Exception as e:
+        warning(f"Could not configure MicroK8s SSL settings: {e}")
+        warning("You may run into errors when using the CLI.")
+
+
+def _get_microk8s_certificate_path() -> Path | None:
+    """
+    Get the path to the MicroK8s certificate.
+
+    Returns
+    -------
+    Path | None
+        Path to the MicroK8s certificate if found, None otherwise
+    """
+    # Common MicroK8s certificate locations
+    possible_paths = [
+        Path.home() / ".kube" / "microk8s.crt",
+        Path("/var/snap/microk8s/current/certs/ca.crt"),
+        Path("/var/snap/microk8s/current/certs/server.crt"),
+    ]
+
+    for cert_path in possible_paths:
+        if cert_path.exists():
+            return cert_path
+
+    return None
+
+
+def _configure_with_certificate(cert_path: Path, cfg: client.Configuration) -> None:
+    """
+    Configure the Kubernetes client to use a specific certificate.
+
+    Parameters
+    ----------
+    cert_path : Path
+        Path to the certificate file
+    """
+    try:
+        # Validate the certificate before using it
+        if not _validate_certificate(cert_path):
+            warning(
+                f"Certificate {cert_path} appears to be invalid. You may run into "
+                "errors when using the CLI."
+            )
+            return
+
+        cfg.verify_ssl = True
+        cfg.ssl_ca_cert = str(cert_path)
+
+        # Apply the configuration to the default client
+        client.Configuration.set_default(cfg)
+
+    except Exception as e:
+        warning(f"Failed to configure with certificate {cert_path}: {e}")
+        warning("You may run into errors when using the CLI.")
+
+
+def _validate_certificate(cert_path: Path) -> bool:
+    """
+    Validate that a certificate file is readable and appears to be a valid certificate.
+
+    Parameters
+    ----------
+    cert_path : Path
+        Path to the certificate file
+
+    Returns
+    -------
+    bool
+        True if the certificate appears valid, False otherwise
+    """
+    try:
+        # Check if the file exists and is readable
+        if not cert_path.exists() or not cert_path.is_file():
+            return False
+
+        # Try to read the certificate content
+        with open(cert_path, "rb") as f:
+            cert_data = f.read()
+
+        # Basic validation: check if it looks like a PEM certificate
+        if b"-----BEGIN CERTIFICATE-----" not in cert_data:
+            return False
+
+        # Try to parse the certificate with Python's ssl module
+        if _validate_certificate_with_ssl(cert_data):
+            return True
+
+        # If we can't parse it with ssl module, at least check it has the right
+        # structure
+        return (
+            b"-----BEGIN CERTIFICATE-----" in cert_data
+            and b"-----END CERTIFICATE-----" in cert_data
+        )
+
+    except Exception as e:
+        warning(f"Certificate validation failed: {e}")
+        return False
+
+
+def _validate_certificate_with_ssl(cert_data: bytes) -> bool:
+    """
+    Validate a certificate with the ssl module.
+
+    Parameters
+    ----------
+    cert_data : bytes
+        The certificate data
+
+    Returns
+    -------
+    bool
+        True if the certificate appears valid, False otherwise
+    """
+    try:
+        # Extract the base64 part between BEGIN and END
+        start_text = b"-----BEGIN CERTIFICATE-----"
+        len_start_text = len(start_text)
+        start = cert_data.find(start_text)
+        end = cert_data.find(b"-----END CERTIFICATE-----")
+        if start != -1 and end != -1:
+            cert_b64 = (
+                cert_data[start + len_start_text : end]
+                .replace(b"\n", b"")
+                .replace(b"\r", b"")
+            )
+            cert_der = base64.b64decode(cert_b64)
+            ssl.DER_cert_to_PEM_cert(cert_der)
+            return True
+    except Exception:
+        pass
+    return False
+
+
+def load_kubernetes_config_with_fallback() -> bool:
+    """
+    Load Kubernetes configuration with fallback for development environments.
+
+    This function tries to load the Kubernetes configuration and handles
+    common issues like SSL certificate problems in development environments.
+
+    Returns
+    -------
+    bool
+        True if configuration was loaded successfully, False otherwise
+    """
+    # Try to load in-cluster config first (for when running inside Kubernetes)
+    try:
+        config.load_incluster_config()
+        return True
+    except ConfigException:
+        pass
+
+    # Fallback to kubeconfig
+    try:
+        # Load kubeconfig into default config, then adjust CA if MicroK8s
+        config.load_kube_config()
+        cfg = client.Configuration.get_default_copy()
+        configure_kubernetes_client_for_microk8s(cfg)
+        client.Configuration.set_default(cfg)
+
+        return True
+    except ConfigException as exc:
+        warning(f"Failed to load Kubernetes configuration: {exc}")
+        return False
+
+
+def create_kubernetes_apis_with_ssl_handling() -> tuple[
+    client.CoreV1Api, client.BatchV1Api
+]:
+    """
+    Create Kubernetes API clients with SSL handling for development environments.
+
+    Returns
+    -------
+    tuple[client.CoreV1Api, client.BatchV1Api]
+        Tuple of CoreV1Api and BatchV1Api clients
+    """
+    # Load configuration with fallback handling
+    if not load_kubernetes_config_with_fallback():
+        raise RuntimeError("Failed to load Kubernetes configuration")
+
+    # Create API clients
+    core_api = client.CoreV1Api()
+    batch_api = client.BatchV1Api()
+
+    return core_api, batch_api
+
+
+def get_core_api_with_ssl_handling() -> client.CoreV1Api:
+    """
+    Get the CoreV1Api client with SSL handling for development environments.
+    """
+    core_api, _ = create_kubernetes_apis_with_ssl_handling()
+    return core_api
+
+
+def is_microk8s_context() -> bool:
+    """
+    Check if the current Kubernetes context is MicroK8s.
+
+    Returns
+    -------
+    bool
+        True if using MicroK8s context, False otherwise
+    """
+    try:
+        _, active_context = config.list_kube_config_contexts()
+        current_context = active_context.get("name", "") if active_context else ""
+        return "microk8s" in current_context.lower()
+    except Exception:
+        return False