vantage6 5.0.0a35__py3-none-any.whl → 5.0.0a37__py3-none-any.whl

vantage6/cli/sandbox/start.py

@@ -0,0 +1,341 @@
+import subprocess
+import time
+from pathlib import Path
+
+import click
+from colorama import Fore, Style
+
+from vantage6.common import error, info
+from vantage6.common.globals import HTTP_LOCALHOST, InstanceType, Ports
+
+from vantage6.client import Client
+from vantage6.client.utils import LogLevel
+
+from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.common.utils import select_context_and_namespace
+from vantage6.cli.context.auth import AuthContext
+from vantage6.cli.context.node import NodeContext
+from vantage6.cli.context.server import ServerContext
+from vantage6.cli.sandbox.config.node import NodeDataset, NodeSandboxConfigManager
+from vantage6.cli.sandbox.populate import populate_server_sandbox
+from vantage6.cli.server.start import cli_server_start
+
+
+@click.command()
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click.option(
+    "--local-chart-dir",
+    type=click.Path(exists=True),
+    default=None,
+    help="Local chart repository to use.",
+)
+@click.option(
+    "--re-initialize",
+    is_flag=True,
+    default=False,
+    help="Re-initialize the sandbox. This will repopulate the server and create new "
+    "node configurations.",
+)
+@click.option(
+    "--num-nodes",
+    type=int,
+    default=3,
+    help="Generate this number of nodes in the development network. Only used if "
+    "--re-initialize flag is provided.",
+)
+@click.option(
+    "--extra-node-config",
+    type=click.Path("rb"),
+    default=None,
+    help="YAML File with additional node configuration. This will be "
+    "appended to each of the node configuration files. Only used if --re-initialize "
+    "flag is provided",
+)
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=None,
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file. Only used if the --re-initialize flag "
+    "is provided.",
+)
+@click.option(
+    "--data-dir",
+    "custom_data_dir",
+    type=click.Path(exists=True),
+    default=None,
+    help="Path to a custom data directory to use. This option is especially useful "
+    "on WSL because of mount issues for default directories. Only used if the "
+    "--re-initialize flag is provided.",
+)
+@click_insert_context(type_=InstanceType.SERVER, is_sandbox=True)
+@click.pass_context
+def cli_sandbox_start(
+    click_ctx: click.Context,
+    ctx: ServerContext,
+    context: str | None,
+    namespace: str | None,
+    local_chart_dir: Path | None,
+    re_initialize: bool,
+    num_nodes: int,
+    extra_node_config: Path | None,
+    add_dataset: tuple[str, Path] | None,
+    custom_data_dir: Path | None,
+) -> None:
+    """
+    Start a sandbox environment.
+    """
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
+
+    # TODO if re-initalize is specified, we must remove the existing node configs
+    execute_sandbox_start(
+        click_ctx=click_ctx,
+        ctx=ctx,
+        server_name=ctx.name,
+        context=context,
+        namespace=namespace,
+        num_nodes=num_nodes,
+        initialize=re_initialize,
+        extra_node_config=extra_node_config,
+        add_dataset=add_dataset,
+        custom_data_dir=custom_data_dir,
+        local_chart_dir=local_chart_dir,
+    )
+
+
+def execute_sandbox_start(
+    click_ctx: click.Context,
+    ctx: ServerContext,
+    server_name: str,
+    context: str,
+    namespace: str,
+    num_nodes: int,
+    initialize: bool,
+    node_image: str | None = None,
+    k8s_node_name: str | None = None,
+    extra_node_config: Path | None = None,
+    add_dataset: tuple[str, Path] | None = None,
+    custom_data_dir: Path | None = None,
+    local_chart_dir: str | None = None,
+) -> None:
+    if k8s_node_name is None:
+        k8s_node_name = ctx.config["database"]["k8sNodeName"]
+
+    # First we need to start the keycloak service
+    cmd = [
+        "v6",
+        "auth",
+        "start",
+        "--name",
+        f"{server_name}-auth.sandbox",
+        "--user",
+        "--context",
+        context,
+        "--namespace",
+        namespace,
+        "--sandbox",
+    ]
+    if local_chart_dir:
+        cmd.extend(["--local-chart-dir", local_chart_dir])
+    subprocess.run(cmd, check=True)
+    # Note: the CLI auth start function is blocking until the auth service is ready,
+    # so no need to wait for it to be ready here.
+
+    # run the store. The store is started before the server so that the server can
+    # couple to the store on startup.
+    info("Starting algorithm store...")
+    cmd = [
+        "v6",
+        "algorithm-store",
+        "start",
+        "--name",
+        f"{ctx.name}-store.sandbox",
+        "--user",
+        "--context",
+        context,
+        "--namespace",
+        namespace,
+        "--sandbox",
+    ]
+    if local_chart_dir:
+        cmd.extend(["--local-chart-dir", local_chart_dir])
+    subprocess.run(cmd, check=True)
+
+    # Then we need to start the server
+    info("Starting vantage6 server")
+    click_ctx.invoke(
+        cli_server_start,
+        ctx=ctx,
+        name=ctx.name,
+        system_folders=False,
+        namespace=namespace,
+        context=context,
+        attach=False,
+        local_chart_dir=local_chart_dir,
+    )
+
+    server_url = f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}"
+    _wait_for_server_to_be_ready(server_url)
+
+    # Then we need to populate the server
+    if initialize:
+        node_config_names = _initialize_sandbox(
+            server_url=server_url,
+            server_name=server_name,
+            num_nodes=num_nodes,
+            ctx=ctx,
+            node_image=node_image,
+            extra_node_config=extra_node_config,
+            add_dataset=add_dataset,
+            context=context,
+            namespace=namespace,
+            k8s_node_name=k8s_node_name,
+            custom_data_dir=custom_data_dir,
+        )
+    else:
+        node_configs, _ = NodeContext.available_configurations(
+            system_folders=False, is_sandbox=True
+        )
+        node_config_names = [
+            config.name
+            for config in node_configs
+            if config.name.startswith(f"{server_name}-node-")
+        ]
+
+    # Then start the nodes
+    info("Starting nodes")
+    for node_config_name in node_config_names:
+        cmd = [
+            "v6",
+            "node",
+            "start",
+            "--name",
+            node_config_name,
+            "--sandbox",
+        ]
+        if local_chart_dir:
+            cmd.extend(["--local-chart-dir", local_chart_dir])
+        subprocess.run(cmd, check=True)
+
+    # Print the authentication credentials
+    _print_auth_credentials(server_name)
+
+
+def _initialize_sandbox(
+    server_url: str,
+    server_name: str,
+    num_nodes: int,
+    ctx: ServerContext,
+    node_image: str | None,
+    extra_node_config: Path | None,
+    add_dataset: tuple[str, Path] | None,
+    context: str,
+    namespace: str,
+    k8s_node_name: str,
+    custom_data_dir: Path | None,
+) -> list[str]:
+    info("Populating server")
+    node_details = populate_server_sandbox(
+        server_url=server_url,
+        auth_url=f"{HTTP_LOCALHOST}:{Ports.DEV_AUTH}",
+        number_of_nodes=num_nodes,
+    )
+
+    api_keys = [node["api_key"] for node in node_details]
+    node_names = [node["name"] for node in node_details]
+
+    extra_dataset = (
+        NodeDataset(
+            label=add_dataset[0],
+            path=add_dataset[1],
+        )
+        if add_dataset is not None and add_dataset != ()
+        else None
+    )
+
+    # Create node config files from the nodes that were just registered in the server
+    node_config_manager = NodeSandboxConfigManager(
+        server_name=server_name,
+        api_keys=api_keys,
+        node_names=node_names,
+        server_port=ctx.config["server"]["port"],
+        node_image=node_image,
+        extra_node_config=extra_node_config,
+        extra_dataset=extra_dataset,
+        context=context,
+        namespace=namespace,
+        k8s_node_name=k8s_node_name,
+        custom_data_dir=custom_data_dir,
+    )
+    node_config_manager.generate_node_configs()
+
+    return node_config_manager.node_config_names
+
+
+def _print_auth_credentials(server_name: str) -> None:
+    """
+    Find user credentials to print, from the auth config file
+
+    Parameters
+    ----------
+    server_name : str
+        Name of the server.
+    """
+    auth_ctx = AuthContext(
+        instance_name=f"{server_name}-auth",
+        system_folders=False,
+        is_sandbox=True,
+    )
+    auth_config = auth_ctx.config
+
+    try:
+        admin_user = auth_config["keycloak"]["keycloakConfigCli"]["configuration"][
+            "realm"
+        ]["users"][0]
+        username = admin_user["username"]
+        password = admin_user["credentials"][0]["value"]
+        info("You can login with the following credentials:")
+        info(f"Username: {Fore.GREEN}{username}{Style.RESET_ALL}")
+        info(f"Password: {Fore.GREEN}{password}{Style.RESET_ALL}")
+    except KeyError:
+        # No user found, skip printing credentials
+        pass
+
+
+def _wait_for_server_to_be_ready(server_url: str) -> None:
+    """
+    Wait for the server to be initialized.
+
+    Parameters
+    ----------
+    server_url : str
+        URL of the server.
+    """
+    client = Client(
+        # TODO replace default API path global
+        server_url=server_url,
+        auth_url=f"{HTTP_LOCALHOST}:{Ports.DEV_AUTH}",
+        log_level=LogLevel.ERROR,
+    )
+    max_retries = 100
+    wait_time = 3
+    ready = False
+    for _ in range(max_retries):
+        try:
+            result = client.util.get_server_health()
+            if result and result.get("api"):
+                info("Server is ready.")
+                ready = True
+                break
+        except Exception:
+            info("Waiting for server to be ready...")
+            time.sleep(wait_time)
+
+    if not ready:
+        error("Server did not become ready in time. Exiting...")
+        exit(1)

vantage6/cli/sandbox/stop.py

@@ -0,0 +1,106 @@
+import subprocess
+
+import click
+
+from vantage6.common import error
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.common.utils import (
+    find_running_service_names,
+    select_context_and_namespace,
+    select_running_service,
+)
+from vantage6.cli.context import get_context
+from vantage6.cli.context.node import NodeContext
+from vantage6.cli.server.stop import cli_server_stop
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="Name of the configuration.")
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click.pass_context
+def cli_sandbox_stop(
+    click_ctx: click.Context,
+    name: str | None,
+    context: str | None,
+    namespace: str | None,
+) -> None:
+    """
+    Stop a sandbox environment.
+    """
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
+
+    running_services = find_running_service_names(
+        instance_type=InstanceType.SERVER,
+        only_system_folders=False,
+        only_user_folders=False,
+        context=context,
+        namespace=namespace,
+        sandbox=True,
+    )
+
+    if not running_services:
+        error("No running sandbox services found.")
+        return
+
+    if not name:
+        selected_service = select_running_service(running_services, InstanceType.SERVER)
+        name = selected_service.split("-")[-3]
+    else:
+        ctx = get_context(InstanceType.SERVER, name, False, is_sandbox=True)
+        name = ctx.name
+
+    # stop the sandbox nodes
+    nodes_user_folder, _ = NodeContext.available_configurations(False, is_sandbox=True)
+    for node in nodes_user_folder:
+        if node.name.startswith(f"{name}-node-"):
+            cmd = [
+                "v6",
+                "node",
+                "stop",
+                "--name",
+                node.name,
+                "--sandbox",
+                "--context",
+                context,
+                "--namespace",
+                namespace,
+            ]
+            subprocess.run(cmd, check=True)
+
+    # # Stop all server services
+    click_ctx.invoke(
+        cli_server_stop,
+        name=name,
+        context=context,
+        namespace=namespace,
+        system_folders=False,
+        all_servers=False,
+        is_sandbox=True,
+    )
+
+    # TODO: stop the auth service
+    cmd = [
+        "v6",
+        "auth",
+        "stop",
+        "--name",
+        f"{name}-auth.sandbox",
+        "--sandbox",
+    ]
+    subprocess.run(cmd, check=True)
+
+    # stop the algorithm store
+    cmd = [
+        "v6",
+        "algorithm-store",
+        "stop",
+        "--name",
+        f"{name}-store.sandbox",
+        "--sandbox",
+    ]
+    subprocess.run(cmd, check=True)

vantage6/cli/server/attach.py

@@ -1,6 +1,7 @@
 import click
 
 from vantage6.common import info
+
 from vantage6.cli.common.utils import attach_logs
 
 

vantage6/cli/server/common/__init__.py

@@ -1,16 +1,11 @@
-from docker.client import DockerClient
 from colorama import Fore, Style
 
-from vantage6.common import error, info
-from vantage6.common.globals import APPNAME
-from vantage6.common.docker.addons import remove_container, get_container
+from vantage6.common import error
 from vantage6.common.context import AppContext
 
-from vantage6.cli.context.server import ServerContext
-
 
 def get_server_context(
-    name: str, system_folders: bool, ctx_class: AppContext
+    name: str, system_folders: bool, ctx_class: AppContext, is_sandbox: bool = False
 ) -> AppContext:
     """
     Load the server context from the configuration file.
@@ -23,13 +18,14 @@ def get_server_context(
         Wether to use system folders or if False, the user folders
     ctx_class : AppContext
         Context class to be used. Derivative of AppContext class
-
+    is_sandbox : bool
+        Whether the configuration is a sandbox configuration, by default False
     Returns
     -------
     ServerContext
         Server context object
     """
-    if not ctx_class.config_exists(name, system_folders):
+    if not ctx_class.config_exists(name, system_folders, is_sandbox=is_sandbox):
         scope = "system" if system_folders else "user"
         error(
             f"Configuration {Fore.RED}{name}{Style.RESET_ALL} does not "
@@ -40,29 +36,6 @@ def get_server_context(
     # We do not want to log this here, we do this in the container and not on
     # the host. We only want CLI logging here.
     ctx_class.LOGGING_ENABLED = False
-
     # create server context, and initialize db
-    ctx = ctx_class(name, system_folders=system_folders)
-
+    ctx = ctx_class(name, system_folders=system_folders, is_sandbox=is_sandbox)
     return ctx
-
-
-def stop_ui(client: DockerClient, ctx: ServerContext) -> None:
-    """
-    Check if the UI container is running, and if so, stop and remove it.
-
-    Parameters
-    ----------
-    client : DockerClient
-        Docker client
-    ctx : ServerContext
-        Server context object
-    """
-    ui_container_name = f"{APPNAME}-{ctx.name}-{ctx.scope}-ui"
-    ui_container = get_container(client, name=ui_container_name)
-    if ui_container:
-        remove_container(ui_container, kill=True)
-        info(
-            f"Stopped the {Fore.GREEN}{ui_container_name}"
-            f"{Style.RESET_ALL} User Interface container."
-        )