vantage6 5.0.0a33__py3-none-any.whl → 5.0.0a35__py3-none-any.whl

vantage6/cli/node/new.py

@@ -1,13 +1,22 @@
+import os
+from pathlib import Path
+
 import click
-from colorama import Fore, Style
+import questionary as q
 
-from vantage6.common import ensure_config_dir_writable, error, info
-from vantage6.common.globals import InstanceType
+from vantage6.common import error, info, warning
+from vantage6.common.client.node_client import NodeClient
+from vantage6.common.globals import (
+    FILE_BASED_DATABASE_TYPES,
+    SERVICE_BASED_DATABASE_TYPES,
+    InstanceType,
+    NodePolicy,
+    Ports,
+    RequiredNodeEnvVars,
+)
 
-from vantage6.cli.configuration_wizard import configuration_wizard
-from vantage6.cli.context.node import NodeContext
+from vantage6.cli.common.new import new
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli.utils import check_config_name_allowed, prompt_config_name
 
 
 @click.command()
@@ -23,38 +32,349 @@ from vantage6.cli.utils import check_config_name_allowed, prompt_config_name
     "system_folders",
     flag_value=False,
     default=N_FOL,
-    help="Store this configuration in the user folders. This is the " "default",
+    help="Store this configuration in the user folders. This is the default.",
+)
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option(
+    "--namespace",
+    default=None,
+    help="Kubernetes namespace to use",
 )
-def cli_node_new_configuration(name: str, system_folders: bool) -> None:
+def cli_node_new_configuration(
+    name: str,
+    system_folders: bool,
+    namespace: str,
+    context: str,
+) -> None:
     """
     Create a new node configuration.
 
     Checks if the configuration already exists. If this is not the case
     a questionnaire is invoked to create a new configuration file.
     """
-    name = prompt_config_name(name)
-    # check if config name is allowed docker name
-    check_config_name_allowed(name)
+    new(
+        questionnaire_function=node_configuration_questionaire,
+        name=name,
+        system_folders=system_folders,
+        namespace=namespace,
+        context=context,
+        type_=InstanceType.NODE,
+    )
+
+
+def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
+    """
+    Questionary to generate a config file for the node instance.
+
+    Parameters
+    ----------
+    dirs : dict
+        Dictionary with the directories of the node instance.
+    instance_name : str
+        Name of the node instance.
+
+    Returns
+    -------
+    dict
+        Dictionary with the new node configuration
+    """
+    config = q.unsafe_prompt(
+        [
+            {"type": "text", "name": "api_key", "message": "Enter given api-key:"},
+            {
+                "type": "text",
+                "name": "server_url",
+                "message": "The base-URL of the server:",
+                "default": "http://localhost",
+            },
+        ]
+    )
+    # remove trailing slash from server_url if entered by user
+    config["server_url"] = config["server_url"].rstrip("/")
+
+    # set default port to the https port if server_url is https
+    default_port = (
+        str(Ports.HTTPS)
+        if config["server_url"].startswith("https")
+        else str(Ports.DEV_SERVER)
+    )
+
+    config = config | q.unsafe_prompt(
+        [
+            {
+                "type": "text",
+                "name": "port",
+                "message": "Enter port to which the server listens:",
+                "default": default_port,
+            },
+            {
+                "type": "text",
+                "name": "api_path",
+                "message": "Path of the api:",
+                "default": "/api",
+            },
+            {
+                "type": "text",
+                "name": "task_dir",
+                "message": "Task directory path:",
+                "default": str(dirs["data"]),
+            },
+        ]
+    )
+
+    config["databases"] = {"fileBased": [], "serviceBased": []}
+    while q.confirm("Do you want to add a database?").unsafe_ask():
+        db_label = q.select(
+            "What type of database do you want to add?",
+            choices=["File database", "Database reachable by URI"],
+        ).unsafe_ask()
+
+        if db_label == "File database":
+            config["databases"]["fileBased"].append(_get_file_based_database_config())
+        else:
+            config["databases"]["serviceBased"].append(
+                _get_service_based_database_config()
+            )
+
+    is_policies = q.confirm(
+        "Do you want to limit the algorithms allowed to run on your node? This "
+        "should always be done for production scenarios.",
+        default=True,
+    ).unsafe_ask()
+    policies = {}
+    if is_policies:
+        info(
+            "You can limit the algorithms that can run on your node in two ways: by "
+            "allowing specific algorithms or by allowing all algorithms in a given "
+            "algorithm store."
+        )
+        ask_single_algorithms = q.confirm(
+            "Do you want to enter a list of allowed algorithms?"
+        ).unsafe_ask()
+        if ask_single_algorithms:
+            policies[NodePolicy.ALLOWED_ALGORITHMS.value] = _get_allowed_algorithms()
+        ask_algorithm_stores = q.confirm(
+            "Do you want to allow algorithms from specific algorithm stores?"
+        ).unsafe_ask()
+        if ask_algorithm_stores:
+            policies[NodePolicy.ALLOWED_ALGORITHM_STORES.value] = (
+                _get_allowed_algorithm_stores()
+            )
+        if ask_single_algorithms and ask_algorithm_stores:
+            require_both_whitelists = q.confirm(
+                "Do you want to allow only algorithms that are both in the list of "
+                "allowed algorithms *AND* are part of one of the allowed algorithm "
+                "stores? If not, algorithms will be allowed if they are in either the "
+                "list of allowed algorithms or one of the allowed algorithm stores.",
+                default=True,
+            ).unsafe_ask()
+            policies["allow_either_whitelist_or_store"] = not require_both_whitelists
+    if policies:
+        config["policies"] = policies
 
-    # check that this config does not exist
-    if NodeContext.config_exists(name, system_folders):
-        error(f"Configuration {name} already exists!")
-        exit(1)
+    res = q.select(
+        "Which level of logging would you like?",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "NOTSET"],
+    ).unsafe_ask()
 
-    # Check that we can write in this folder
-    if not ensure_config_dir_writable(system_folders):
-        error("Cannot write configuration file. Exiting...")
-        exit(1)
+    config["logging"] = {
+        "level": res,
+        "use_console": True,
+        "backup_count": 5,
+        "max_size": 1024,
+        "format": "%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s",
+        "datefmt": "%Y-%m-%d %H:%M:%S",
+        "loggers": [
+            {"name": "urllib3", "level": "warning"},
+            {"name": "requests", "level": "warning"},
+            {"name": "engineio.client", "level": "warning"},
+            {"name": "docker.utils.config", "level": "warning"},
+            {"name": "docker.auth", "level": "warning"},
+        ],
+    }
 
-    # create config in ctx location
-    flag = "--system" if system_folders else ""
+    # Check if we can login to the server to retrieve collaboration settings
+    client = NodeClient(
+        instance_name,
+        config["api_key"],
+        server_url=f"{config['server_url']}:{config['port']}{config['api_path']}",
+        auth_url=os.environ.get(RequiredNodeEnvVars.KEYCLOAK_URL.value),
+    )
     try:
-        cfg_file = configuration_wizard(InstanceType.NODE, name, system_folders)
-    except KeyboardInterrupt:
-        error("Configuration creation aborted.")
-        exit(1)
-    info(f"New configuration created: {Fore.GREEN}{cfg_file}{Style.RESET_ALL}")
+        client.authenticate()
+    except Exception as e:
+        error(f"Could not authenticate with server: {e}")
+        error("Please check (1) your API key and (2) if your server is online")
+        warning(
+            "If you continue, you should provide your collaboration settings manually."
+        )
+        if q.confirm("Do you want to abort?", default=True).unsafe_ask():
+            exit(0)
+
+    if client.whoami is not None:
+        encryption = client.is_encrypted_collaboration()
+        # TODO when we build collaboration policies, update this to provide
+        # the node admin with a list of all policies, and whether or not
+        # to accept them
+        q.confirm(
+            f"Encryption is {'enabled' if encryption else 'disabled'}"
+            f" for this collaboration. Accept?",
+            default=True,
+        ).unsafe_ask()
+    else:
+        encryption = q.confirm("Enable encryption?", default=True).unsafe_ask()
+
+    private_key = (
+        "" if not encryption else q.text("Path to private key file:").unsafe_ask()
+    )
+
+    config["encryption"] = {
+        "enabled": encryption is True or encryption == "true",
+        "private_key": private_key,
+    }
+
+    # pack the entire config in a dict with the 'node' key at top level
+    return {"node": config}
+
+
+def _get_file_based_database_config() -> dict:
+    """
+    Prompt the user for the file-based database configuration
+    """
+    db_label = _get_database_label()
+    while True:
+        db_path = q.text(
+            "Path to the database file:",
+        ).unsafe_ask()
+        if Path(db_path).exists():
+            break
+        else:
+            error("The path to the database file does not exist. Please try again.")
+    db_path_resolved = Path(db_path).resolve()
+    db_dir = db_path_resolved.parent
+    db_filename = db_path_resolved.name
+    db_type = q.select("Database type:", choices=FILE_BASED_DATABASE_TYPES).unsafe_ask()
+    return {
+        "name": db_label,
+        "uri": db_path,
+        "type": db_type,
+        "volumePath": db_dir,
+        "originalName": db_filename,
+    }
+
+
+def _get_service_based_database_config() -> dict:
+    """
+    Prompt the user for the service-based database configuration
+
+    Returns
+    -------
+    dict
+        Dictionary with the service-based database configuration
+    """
+    db_label = _get_database_label()
+    db_uri = q.text(
+        "Database URI:",
+    ).unsafe_ask()
+    db_type = q.select(
+        "Database type:", choices=SERVICE_BASED_DATABASE_TYPES
+    ).unsafe_ask()
+
+    env_vars = {}
+    info("You can add environment variables to the database configuration.")
+    info("These variables will be available to the algorithms on your node.")
+    info("Example: MY_POSTGRES_USER=vantage6, MY_POSTGRES_PASSWORD=vantage6")
+    while q.confirm("Do you want to add an environment variable?").unsafe_ask():
+        env_var_name = q.text(
+            "Enter the name of the environment variable:"
+        ).unsafe_ask()
+        env_var_value = q.text(
+            "Enter the value of the environment variable:"
+        ).unsafe_ask()
+        env_vars[env_var_name] = env_var_value
+
+    return {
+        "name": db_label,
+        "uri": db_uri,
+        "type": db_type,
+        "env": env_vars,
+    }
+
+
+def _get_database_label() -> str:
+    """
+    Prompt the user for the label of the database
+    """
+    return q.text(
+        "Enter unique label for the database:",
+        default="default",
+    ).unsafe_ask()
+
+
+def _get_allowed_algorithms() -> list[str]:
+    """
+    Prompt the user for the allowed algorithms on their node
+
+    Returns
+    -------
+    list[str]
+        List of allowed algorithms or regular expressions to match them
+    """
+    info("Below you can add algorithms that are allowed to run on your node.")
+    info(
+        "You can use regular expressions to match multiple algorithms, or you can "
+        "use strings to provide one algorithm at a time."
+    )
+    info("Examples:")
+    info(r"^harbor2\.vantage6\.ai/demo/average$    Allow the demo average algorithm")
+    info(
+        r"^harbor2\.vantage6\.ai/algorithms/.*   Allow all algorithms from "
+        "harbor2.vantage6.ai/algorithms"
+    )
+    info(
+        r"^harbor2\.vantage6\.ai/demo/average@sha256:82becede...$    Allow a "
+        "specific hash of average algorithm"
+    )
+    allowed_algorithms = []
+    while True:
+        algo = q.text(message="Enter your algorithm expression:").unsafe_ask()
+        allowed_algorithms.append(algo)
+        if not q.confirm(
+            "Do you want to add another algorithm expression?", default=True
+        ).unsafe_ask():
+            break
+    return allowed_algorithms
+
+
+def _get_allowed_algorithm_stores() -> list[str]:
+    """
+    Prompt the user for the allowed algorithm stores on their node
+
+    Returns
+    -------
+    list[str]
+        List of allowed algorithm stores
+    """
+    info("Below you can add algorithm stores that are allowed to run on your node.")
+    info(
+        "You can use regular expressions to match multiple algorithm stores, or you can"
+        " use strings to provide one algorithm store at a time."
+    )
+    info("Examples:")
+    info(
+        "https://store.cotopaxi.vantage6.ai    Allow all algorithms from the "
+        "community store"
+    )
     info(
-        f"You can start the node by running "
-        f"{Fore.GREEN}v6 node start {flag}{Style.RESET_ALL}"
+        r"^https://*\.vantage6\.ai$               Allow all algorithms from any "
+        "store hosted on vantage6.ai"
     )
+    allowed_algorithm_stores = []
+    while True:
+        store = q.text(message="Enter the URL of the algorithm store:").unsafe_ask()
+        allowed_algorithm_stores.append(store)
+        if not q.confirm(
+            "Do you want to add another algorithm store?", default=True
+        ).unsafe_ask():
+            break
+    return allowed_algorithm_stores

vantage6/cli/node/remove.py

@@ -1,112 +1,36 @@
-import click
-import questionary as q
-import docker
-import os.path
-import itertools
-
 from pathlib import Path
 from shutil import rmtree
 
-from vantage6.common import (
-    error,
-    info,
-    debug,
-)
-from vantage6.common.globals import APPNAME
+import click
 
+from vantage6.common.globals import InstanceType
 
+from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.common.remove import execute_remove
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli.utils import check_if_docker_daemon_is_running, remove_file
-from vantage6.cli.node.common import select_node, find_running_node_names
-
-# TODO v5+ remove this - just a dummy to prevent import issues from v4 CLI
-# from vantage6.common.globals import VPN_CONFIG_FILE
-VPN_CONFIG_FILE = "vpn.conf"
+from vantage6.cli.globals import InfraComponentName
 
 
 @click.command()
-@click.option("-n", "--name", default=None, help="Configuration name")
-@click.option(
-    "--system",
-    "system_folders",
-    flag_value=True,
-    help="Search for configuration in system folders rather than " "user folders",
-)
-@click.option(
-    "--user",
-    "system_folders",
-    flag_value=False,
-    default=N_FOL,
-    help="Search for configuration in user folders rather than "
-    "system folders. This is the default",
-)
 @click.option(
     "-f", "--force", type=bool, flag_value=True, help="Don't ask for confirmation"
 )
-def cli_node_remove(name: str, system_folders: bool, force: bool) -> None:
+@click_insert_context(
+    type_=InstanceType.NODE, include_name=True, include_system_folders=True
+)
+def cli_node_remove(
+    ctx: NodeContext, name: str, system_folders: bool, force: bool
+) -> None:
     """
     Delete a node permanently.
 
     Remove the configuration file, log file, and docker volumes attached to
     the node.
     """
-    # select configuration name if none supplied
-    name = select_node(name, system_folders)
-
-    client = docker.from_env()
-    check_if_docker_daemon_is_running(client)
-
-    # check if node is still running, otherwise don't allow deleting it
-    running_node_names = find_running_node_names(client)
-
-    post_fix = "system" if system_folders else "user"
-    node_container_name = f"{APPNAME}-{name}-{post_fix}"
-    if node_container_name in running_node_names:
-        error(
-            f"Node {name} is still running! Please stop the node before " "deleting it."
-        )
-        exit(1)
-
-    if not force:
-        if not q.confirm(
-            "This node will be deleted permanently including its "
-            "configuration. Are you sure?",
-            default=False,
-        ).ask():
-            info("Node will not be deleted")
-            exit(0)
-
-    # create node context
-    ctx = NodeContext(name, system_folders=system_folders)
-
-    # remove the docker volume and any temporary volumes
-    debug("Deleting docker volumes")
-    volumes = client.volumes.list()
-    for vol in volumes:
-        if vol.name.startswith(ctx.docker_volume_name):  # includes tmp volumes
-            info(f"Deleting docker volume {vol.name}")
-            vol.remove()
-        # remove docker vpn volume
-        if vol.name == ctx.docker_vpn_volume_name:
-            info(f"Deleting VPN docker volume {vol.name}")
-            vol.remove()
-
-    # remove the VPN configuration file
-    vpn_config_file = os.path.join(ctx.data_dir, "vpn", VPN_CONFIG_FILE)
-    remove_file(vpn_config_file, "VPN configuration")
-
-    # remove the config file
-    remove_file(ctx.config_file, "configuration")
 
-    # remove the log file. As this process opens the log file above, the log
-    # handlers need to be closed before deleting
-    log_dir = Path(ctx.log_file.parent)
-    info(f"Removing log file {log_dir}")
-    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-        handler.close()
-    # remove the whole folder with all the log files
-    rmtree(log_dir)
+    execute_remove(
+        ctx, InstanceType.NODE, InfraComponentName.NODE, name, system_folders, force
+    )
 
     # remove the folder: if it hasn't been started yet this won't exist...
     if Path.exists(ctx.config_dir / name):

vantage6/cli/node/restart.py

@@ -1,23 +1,29 @@
 import subprocess
+
 import click
 import questionary as q
-import docker
 
-from vantage6.common import warning, error
-from vantage6.common.docker.addons import check_docker_running
-from vantage6.cli.common.utils import get_name_from_container_name
-from vantage6.cli.node.stop import cli_node_stop
-from vantage6.cli.node.common import find_running_node_names
+from vantage6.common import error, warning
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.common.utils import (
+    find_running_service_names,
+    get_config_name_from_service_name,
+    select_context_and_namespace,
+)
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+from vantage6.cli.node.stop import cli_node_stop
 
 
 @click.command()
 @click.option("-n", "--name", default=None, help="Configuration name")
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option(
     "--system",
     "system_folders",
     flag_value=True,
-    help="Search for configuration in system folders instead of " "user folders",
+    help="Search for configuration in system folders instead of user folders",
 )
 @click.option(
     "--user",
@@ -27,53 +33,35 @@ from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
     help="Search for configuration in the user folders instead of "
     "system folders. This is the default.",
 )
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep node container after finishing. Useful for debugging",
-)
-@click.option(
-    "--force-db-mount",
-    is_flag=True,
-    help="Always mount node databases; skip the check if they are existing files.",
-)
 @click.option(
     "--attach/--detach",
     default=False,
-    help="Show node logs on the current console after starting the " "node",
-)
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source"
-    " code in this path",
+    help="Show node logs on the current console after starting the node",
 )
 @click.option("--all", "all_nodes", flag_value=True, help="Stop all running nodes")
-@click.option(
-    "--force",
-    "force",
-    flag_value=True,
-    help="Kill nodes instantly; don't wait for them to shut down",
-)
 @click.pass_context
 def cli_node_restart(
     click_ctx: click.Context,
     name: str,
+    context: str,
+    namespace: str,
     system_folders: bool,
-    image: str,
-    keep: bool,
-    mount_src: str,
     attach: bool,
-    force_db_mount: bool,
     all_nodes: bool,
-    force: bool,
 ) -> None:
     """Restart the node"""
-    check_docker_running()
-    client = docker.from_env()
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
 
-    running_node_names = find_running_node_names(client)
+    running_node_names = find_running_service_names(
+        instance_type=InstanceType.NODE,
+        only_system_folders=system_folders,
+        only_user_folders=not system_folders,
+        context=context,
+        namespace=namespace,
+    )
     if not running_node_names:
         warning("No nodes are currently running. No action taken.")
         return
@@ -87,19 +75,19 @@ def cli_node_restart(
 
     if all_nodes:
         names = [
-            get_name_from_container_name(container_name)
+            get_config_name_from_service_name(container_name)
             for container_name in running_node_names
         ]
     else:
         if not name:
             try:
-                container_name = q.select(
+                helm_name = q.select(
                     "Select the node you wish to restart:", choices=running_node_names
                 ).unsafe_ask()
             except KeyboardInterrupt:
                 error("Aborted by user!")
                 return
-            names = [get_name_from_container_name(container_name)]
+            names = [get_config_name_from_service_name(helm_name)]
         else:
             names = [name]
 
@@ -109,20 +97,11 @@ def cli_node_restart(
             name=node_name,
             system_folders=system_folders,
             all_nodes=False,
-            force=force,
         )
 
         cmd = ["v6", "node", "start", "--name", node_name]
         if system_folders:
             cmd.append("--system")
-        if image:
-            cmd.extend(["--image", image])
-        if keep:
-            cmd.append("--keep")
-        if mount_src:
-            cmd.extend(["--mount-src", mount_src])
         if attach:
             cmd.append("--attach")
-        if force_db_mount:
-            cmd.append("--force-db-mount")
         subprocess.run(cmd, check=True)