vantage6 5.0.0a34__py3-none-any.whl → 5.0.0a36__py3-none-any.whl

vantage6/cli/node/remove.py

@@ -1,112 +1,36 @@
-import click
-import questionary as q
-import docker
-import os.path
-import itertools
-
 from pathlib import Path
 from shutil import rmtree
 
-from vantage6.common import (
-    error,
-    info,
-    debug,
-)
-from vantage6.common.globals import APPNAME
+import click
 
+from vantage6.common.globals import InstanceType
 
+from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.common.remove import execute_remove
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli.utils import check_if_docker_daemon_is_running, remove_file
-from vantage6.cli.node.common import select_node, find_running_node_names
-
-# TODO v5+ remove this - just a dummy to prevent import issues from v4 CLI
-# from vantage6.common.globals import VPN_CONFIG_FILE
-VPN_CONFIG_FILE = "vpn.conf"
+from vantage6.cli.globals import InfraComponentName
 
 
 @click.command()
-@click.option("-n", "--name", default=None, help="Configuration name")
-@click.option(
-    "--system",
-    "system_folders",
-    flag_value=True,
-    help="Search for configuration in system folders rather than " "user folders",
-)
-@click.option(
-    "--user",
-    "system_folders",
-    flag_value=False,
-    default=N_FOL,
-    help="Search for configuration in user folders rather than "
-    "system folders. This is the default",
-)
 @click.option(
     "-f", "--force", type=bool, flag_value=True, help="Don't ask for confirmation"
 )
-def cli_node_remove(name: str, system_folders: bool, force: bool) -> None:
+@click_insert_context(
+    type_=InstanceType.NODE, include_name=True, include_system_folders=True
+)
+def cli_node_remove(
+    ctx: NodeContext, name: str, system_folders: bool, force: bool
+) -> None:
     """
     Delete a node permanently.
 
     Remove the configuration file, log file, and docker volumes attached to
     the node.
     """
-    # select configuration name if none supplied
-    name = select_node(name, system_folders)
-
-    client = docker.from_env()
-    check_if_docker_daemon_is_running(client)
-
-    # check if node is still running, otherwise don't allow deleting it
-    running_node_names = find_running_node_names(client)
-
-    post_fix = "system" if system_folders else "user"
-    node_container_name = f"{APPNAME}-{name}-{post_fix}"
-    if node_container_name in running_node_names:
-        error(
-            f"Node {name} is still running! Please stop the node before " "deleting it."
-        )
-        exit(1)
-
-    if not force:
-        if not q.confirm(
-            "This node will be deleted permanently including its "
-            "configuration. Are you sure?",
-            default=False,
-        ).ask():
-            info("Node will not be deleted")
-            exit(0)
-
-    # create node context
-    ctx = NodeContext(name, system_folders=system_folders)
-
-    # remove the docker volume and any temporary volumes
-    debug("Deleting docker volumes")
-    volumes = client.volumes.list()
-    for vol in volumes:
-        if vol.name.startswith(ctx.docker_volume_name):  # includes tmp volumes
-            info(f"Deleting docker volume {vol.name}")
-            vol.remove()
-        # remove docker vpn volume
-        if vol.name == ctx.docker_vpn_volume_name:
-            info(f"Deleting VPN docker volume {vol.name}")
-            vol.remove()
-
-    # remove the VPN configuration file
-    vpn_config_file = os.path.join(ctx.data_dir, "vpn", VPN_CONFIG_FILE)
-    remove_file(vpn_config_file, "VPN configuration")
-
-    # remove the config file
-    remove_file(ctx.config_file, "configuration")
 
-    # remove the log file. As this process opens the log file above, the log
-    # handlers need to be closed before deleting
-    log_dir = Path(ctx.log_file.parent)
-    info(f"Removing log file {log_dir}")
-    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-        handler.close()
-    # remove the whole folder with all the log files
-    rmtree(log_dir)
+    execute_remove(
+        ctx, InstanceType.NODE, InfraComponentName.NODE, name, system_folders, force
+    )
 
     # remove the folder: if it hasn't been started yet this won't exist...
     if Path.exists(ctx.config_dir / name):

vantage6/cli/node/restart.py

@@ -1,23 +1,29 @@
 import subprocess
+
 import click
 import questionary as q
-import docker
 
-from vantage6.common import warning, error
-from vantage6.common.docker.addons import check_docker_running
-from vantage6.cli.common.utils import get_name_from_container_name
-from vantage6.cli.node.stop import cli_node_stop
-from vantage6.cli.node.common import find_running_node_names
+from vantage6.common import error, warning
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.common.utils import (
+    find_running_service_names,
+    get_config_name_from_service_name,
+    select_context_and_namespace,
+)
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+from vantage6.cli.node.stop import cli_node_stop
 
 
 @click.command()
 @click.option("-n", "--name", default=None, help="Configuration name")
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option(
     "--system",
     "system_folders",
     flag_value=True,
-    help="Search for configuration in system folders instead of " "user folders",
+    help="Search for configuration in system folders instead of user folders",
 )
 @click.option(
     "--user",
@@ -27,53 +33,35 @@ from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
     help="Search for configuration in the user folders instead of "
     "system folders. This is the default.",
 )
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep node container after finishing. Useful for debugging",
-)
-@click.option(
-    "--force-db-mount",
-    is_flag=True,
-    help="Always mount node databases; skip the check if they are existing files.",
-)
 @click.option(
     "--attach/--detach",
     default=False,
-    help="Show node logs on the current console after starting the " "node",
-)
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source"
-    " code in this path",
+    help="Show node logs on the current console after starting the node",
 )
 @click.option("--all", "all_nodes", flag_value=True, help="Stop all running nodes")
-@click.option(
-    "--force",
-    "force",
-    flag_value=True,
-    help="Kill nodes instantly; don't wait for them to shut down",
-)
 @click.pass_context
 def cli_node_restart(
     click_ctx: click.Context,
     name: str,
+    context: str,
+    namespace: str,
     system_folders: bool,
-    image: str,
-    keep: bool,
-    mount_src: str,
     attach: bool,
-    force_db_mount: bool,
     all_nodes: bool,
-    force: bool,
 ) -> None:
     """Restart the node"""
-    check_docker_running()
-    client = docker.from_env()
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
 
-    running_node_names = find_running_node_names(client)
+    running_node_names = find_running_service_names(
+        instance_type=InstanceType.NODE,
+        only_system_folders=system_folders,
+        only_user_folders=not system_folders,
+        context=context,
+        namespace=namespace,
+    )
     if not running_node_names:
         warning("No nodes are currently running. No action taken.")
         return
@@ -87,19 +75,19 @@ def cli_node_restart(
 
     if all_nodes:
         names = [
-            get_name_from_container_name(container_name)
+            get_config_name_from_service_name(container_name)
             for container_name in running_node_names
         ]
     else:
         if not name:
             try:
-                container_name = q.select(
+                helm_name = q.select(
                     "Select the node you wish to restart:", choices=running_node_names
                 ).unsafe_ask()
             except KeyboardInterrupt:
                 error("Aborted by user!")
                 return
-            names = [get_name_from_container_name(container_name)]
+            names = [get_config_name_from_service_name(helm_name)]
         else:
             names = [name]
 
@@ -109,20 +97,11 @@ def cli_node_restart(
             name=node_name,
             system_folders=system_folders,
             all_nodes=False,
-            force=force,
         )
 
         cmd = ["v6", "node", "start", "--name", node_name]
         if system_folders:
             cmd.append("--system")
-        if image:
-            cmd.extend(["--image", image])
-        if keep:
-            cmd.append("--keep")
-        if mount_src:
-            cmd.extend(["--mount-src", mount_src])
         if attach:
             cmd.append("--attach")
-        if force_db_mount:
-            cmd.append("--force-db-mount")
         subprocess.run(cmd, check=True)

vantage6/cli/node/start.py

@@ -1,334 +1,111 @@
-import os.path
-import time
-from pathlib import Path
-from threading import Thread
-
 import click
-import docker
-from colorama import Fore, Style
 
-from vantage6.common import debug, error, info, warning
-from vantage6.common.dataclass import TaskDB
-from vantage6.common.docker.addons import (
-    check_docker_running,
-    remove_container_if_exists,
-)
-from vantage6.common.globals import (
-    APPNAME,
-    DEFAULT_DOCKER_REGISTRY,
-    DEFAULT_NODE_IMAGE,
-    DEFAULT_NODE_IMAGE_WO_TAG,
-    InstanceType,
-)
+from vantage6.common import info
+from vantage6.common.globals import InstanceType
 
-from vantage6.cli import __version__
 from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.common.start import pull_infra_image
-from vantage6.cli.common.utils import print_log_worker
+from vantage6.cli.common.start import (
+    helm_install,
+    prestart_checks,
+    start_port_forward,
+)
+from vantage6.cli.common.utils import (
+    attach_logs,
+    create_directory_if_not_exists,
+)
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.node.common import create_client
-from vantage6.cli.utils import check_config_name_allowed
+from vantage6.cli.globals import ChartName
+
+from vantage6.node.globals import DEFAULT_PROXY_SERVER_PORT
 
 
 @click.command()
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep node container after finishing. Useful for debugging",
-)
-@click.option(
-    "--force-db-mount",
-    is_flag=True,
-    help="Always mount node databases; skip the check if they are existing files.",
-)
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option(
     "--attach/--detach",
     default=False,
     help="Show node logs on the current console after starting the node",
 )
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source code in this path",
-)
 @click_insert_context(InstanceType.NODE, include_name=True, include_system_folders=True)
 def cli_node_start(
     ctx: NodeContext,
     name: str,
     system_folders: bool,
-    image: str,
-    keep: bool,
-    mount_src: str,
+    context: str,
+    namespace: str,
     attach: bool,
-    force_db_mount: bool,
 ) -> None:
     """
     Start the node.
     """
-    check_docker_running()
     info("Starting node...")
-    info("Finding Docker daemon")
-    docker_client = docker.from_env()
-    NodeContext.LOGGING_ENABLED = False
-
-    # check if config name is allowed docker name, else exit
-    check_config_name_allowed(ctx.name)
-
-    # check that this node is not already running
-    running_nodes = docker_client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.NODE.value}"}
-    )
-
-    suffix = "system" if system_folders else "user"
-    for node in running_nodes:
-        if node.name == f"{APPNAME}-{name}-{suffix}":
-            error(f"Node {Fore.RED}{name}{Style.RESET_ALL} is already running")
-            exit(1)
-
-    # make sure the (host)-task and -log dir exists
-    info("Checking that data and log dirs exist")
-    ctx.data_dir.mkdir(parents=True, exist_ok=True)
-    ctx.log_dir.mkdir(parents=True, exist_ok=True)
-
-    # Determine image-name. First we check if the option --image has been used.
-    # Then we check if the image has been specified in the config file, and
-    # finally we use the default settings from the package.
-    if not image:
-        custom_images: dict = ctx.config.get("images")
-        if custom_images:
-            image = custom_images.get("node")
-        else:
-            # if no custom image is specified, find the server version and use
-            # the latest images from that minor version
-            client = create_client(ctx)
-            major_minor = None
-            try:
-                # try to get server version, skip if can't get a connection
-                version = client.util.get_server_version(attempts_on_timeout=3)[
-                    "version"
-                ]
-                major_minor = ".".join(version.split(".")[:2])
-                image = (
-                    f"{DEFAULT_DOCKER_REGISTRY}/"
-                    f"{DEFAULT_NODE_IMAGE_WO_TAG}"
-                    f":{major_minor}"
-                )
-            except Exception:
-                warning("Could not determine server version. Using default node image")
-
-            if major_minor and not __version__.startswith(major_minor):
-                warning(
-                    "Version mismatch between CLI and server/node. CLI is "
-                    f"running on version {__version__}, while node and server "
-                    f"are on version {major_minor}. This might cause "
-                    f"unexpected issues; changing to {major_minor}.<latest> "
-                    "is recommended."
-                )
-
-        # fail safe, in case no custom image is specified and we can't get the
-        # server version
-        if not image:
-            image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
-
-    info(f"Pulling latest node image '{image}'")
-    pull_infra_image(docker_client, image, InstanceType.NODE)
-
-    data_volume = docker_client.volumes.create(ctx.docker_volume_name)
-    vpn_volume = docker_client.volumes.create(ctx.docker_vpn_volume_name)
-    ssh_volume = docker_client.volumes.create(ctx.docker_ssh_volume_name)
-    squid_volume = docker_client.volumes.create(ctx.docker_squid_volume_name)
-
-    info("Creating file & folder mounts")
-    # FIXME: should obtain mount points from DockerNodeContext
-    mounts = [
-        # (target, source)
-        ("/mnt/log", str(ctx.log_dir)),
-        ("/mnt/data", data_volume.name),
-        ("/mnt/vpn", vpn_volume.name),
-        ("/mnt/ssh", ssh_volume.name),
-        ("/mnt/squid", squid_volume.name),
-        ("/mnt/config", str(ctx.config_dir)),
-        ("/var/run/docker.sock", "/var/run/docker.sock"),
-    ]
-
-    if mount_src:
-        # If mount_src is a relative path, docker will consider it a volume.
-        mount_src = os.path.abspath(mount_src)
-        mounts.append(("/vantage6", mount_src))
-
-    # FIXME: Code duplication: Node.__init__() (vantage6/node/__init__.py)
-    #   uses a lot of the same logic. Suggest moving this to
-    #   ctx.get_private_key()
-    filename = ctx.config.get("encryption", {}).get("private_key")
-    # filename may be set to an empty string
-    if not filename:
-        filename = "private_key.pem"
-
-    # Location may be overridden by the environment
-    filename = os.environ.get("PRIVATE_KEY", filename)
-
-    # If ctx.get_data_file() receives an absolute path, it is returned as-is
-    fullpath = Path(ctx.get_data_file(filename))
-    if fullpath:
-        if Path(fullpath).exists():
-            mounts.append(("/mnt/private_key.pem", str(fullpath)))
-        else:
-            warning(f"Private key file is provided {fullpath}, but does not exist")
-
-    # Mount private keys for ssh tunnels
-    ssh_tunnels = ctx.config.get("ssh-tunnels", [])
-    for ssh_tunnel in ssh_tunnels:
-        hostname = ssh_tunnel.get("hostname")
-        key_path = ssh_tunnel.get("ssh", {}).get("identity", {}).get("key")
-        if not key_path:
-            error(
-                f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
-                "key not provided. Continuing to start without this tunnel."
-            )
-        key_path = Path(key_path)
-        if not key_path.exists():
-            error(
-                f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
-                "key does not exist. Continuing to start without this "
-                "tunnel."
-            )
-
-        info(f"  Mounting private key for {hostname} at {key_path}")
-
-        # we remove the .tmp in the container, this is because the file is
-        # mounted in a volume mount point. Somehow the file is than empty in
-        # the volume but not for the node instance. By removing the .tmp we
-        # make sure that the file is not empty in the volume.
-        mounts.append((f"/mnt/ssh/{hostname}.pem.tmp", str(key_path)))
-
-    env = {
-        "DATA_VOLUME_NAME": data_volume.name,
-        "VPN_VOLUME_NAME": vpn_volume.name,
-        "PRIVATE_KEY": "/mnt/private_key.pem",
-    }
-
-    # only mount the DB if it is a file
-    info("Setting up databases")
-    dbs = [TaskDB.from_dict(db) for db in ctx.databases]
-    for db in dbs:
-        # check that label contains only valid characters
-        if not db.label.isidentifier():
-            error(
-                f"Database label {Fore.RED}{db.label}{Style.RESET_ALL} contains"
-                " invalid characters. Only letters, numbers, and underscores"
-                " are allowed, and it cannot start with a number."
-            )
-            exit(1)
-
-        info(
-            f"  Processing {Fore.GREEN}{db.type}{Style.RESET_ALL} database "
-            f"{Fore.GREEN}{db.label}:{db.uri}{Style.RESET_ALL}"
-        )
-        label_capitals = db.label.upper()
-
-        try:
-            db_file_exists = Path(db.uri).exists()
-        except Exception:
-            # If the database uri cannot be parsed, it is definitely not a
-            # file. In case of http servers or sql servers, checking the path
-            # of the the uri will lead to an OS-dependent error, which is why
-            # we catch all exceptions here.
-            db_file_exists = False
-
-        if db.type.is_file_based() and not db_file_exists:
-            error(
-                f"Database {Fore.RED}{db.uri}{Style.RESET_ALL} not found. Databases of "
-                f"type '{db.type}' must be present on the harddrive. Please "
-                "update your node configuration file."
-            )
-            exit(1)
-
-        if not db_file_exists and not force_db_mount:
-            debug("  - non file-based database added")
-            env[f"{label_capitals}_DATABASE_URI"] = db.uri
-        else:
-            debug("  - file-based database added")
-            suffix = Path(db.uri).suffix
-            env[f"{label_capitals}_DATABASE_URI"] = f"{db.label}{suffix}"
-            mounts.append((f"/mnt/{db.label}{suffix}", str(db.uri)))
-
-    system_folders_option = "--system" if system_folders else "--user"
-    cmd = (
-        f"vnode-local start -c /mnt/config/{name}.yaml -n {name} "
-        f" --dockerized {system_folders_option}"
-    )
-
-    volumes = []
-    for mount in mounts:
-        volumes.append(f"{mount[1]}:{mount[0]}")
-
-    extra_mounts = ctx.config.get("node_extra_mounts", [])
-    for mount in extra_mounts:
-        volumes.append(mount)
-
-    extra_env = ctx.config.get("node_extra_env", {})
-    # all extra env var names should be valid identifiers
-    extra_env_invalid = [key for key in extra_env.keys() if not key.isidentifier()]
-    if extra_env_invalid:
-        error(
-            "Environment variable names should be valid identifiers. "
-            f"The following break this rule: {extra_env_invalid}"
-        )
-        exit(1)
-    # we won't accept overwrites of existing env vars
-    env_overwrites = extra_env.keys() & env.keys()
-    if env_overwrites:
-        error(f"Cannot overwrite existing node environment variables: {env_overwrites}")
-        exit(1)
-    env.update(extra_env)
-
-    # Add extra hosts to the environment
-    extra_hosts = ctx.config.get("node_extra_hosts", {})
-
-    remove_container_if_exists(
-        docker_client=docker_client, name=ctx.docker_container_name
-    )
 
-    info("Running Docker container")
-    container = docker_client.containers.run(
-        image,
-        command=cmd,
-        volumes=volumes,
-        detach=True,
-        labels={
-            f"{APPNAME}-type": InstanceType.NODE.value,
-            "system": str(system_folders),
-            "name": ctx.config_file_name,
-        },
-        environment=env,
-        name=ctx.docker_container_name,
-        auto_remove=not keep,
-        tty=True,
-        extra_hosts=extra_hosts,
+    prestart_checks(ctx, InstanceType.NODE, name, system_folders, context, namespace)
+
+    create_directory_if_not_exists(ctx.log_dir)
+    create_directory_if_not_exists(ctx.data_dir)
+
+    # TODO issue #2256 - run same version node as server
+    # # Determine image-name. First we check if the option --image has been used.
+    # # Then we check if the image has been specified in the config file, and
+    # # finally we use the default settings from the package.
+    # if not image:
+    #     custom_images: dict = ctx.config.get("images")
+    #     if custom_images:
+    #         image = custom_images.get("node")
+    #     else:
+    #         # if no custom image is specified, find the server version and use
+    #         # the latest images from that minor version
+    #         client = create_client(ctx)
+    #         major_minor = None
+    #         try:
+    #             # try to get server version, skip if can't get a connection
+    #             version = client.util.get_server_version(attempts_on_timeout=3)[
+    #                 "version"
+    #             ]
+    #             major_minor = ".".join(version.split(".")[:2])
+    #             image = (
+    #                 f"{DEFAULT_DOCKER_REGISTRY}/"
+    #                 f"{DEFAULT_NODE_IMAGE_WO_TAG}"
+    #                 f":{major_minor}"
+    #             )
+    #         except Exception:
+    #             warning("Could not determine server version. Using default node image")
+
+    #         if major_minor and not __version__.startswith(major_minor):
+    #             warning(
+    #                 "Version mismatch between CLI and server/node. CLI is "
+    #                 f"running on version {__version__}, while node and server "
+    #                 f"are on version {major_minor}. This might cause "
+    #                 f"unexpected issues; changing to {major_minor}.<latest> "
+    #                 "is recommended."
+    #             )
+
+    #     # fail safe, in case no custom image is specified and we can't get the
+    #     # server version
+    #     if not image:
+    #         image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
+
+    # info(f"Pulling latest node image '{image}'")
+    # pull_infra_image(docker_client, image, InstanceType.NODE)
+
+    helm_install(
+        release_name=ctx.helm_release_name,
+        chart_name=ChartName.NODE,
+        values_file=ctx.config_file,
+        context=context,
+        namespace=namespace,
     )
 
-    info("Node container was started!")
-    info(
-        "Please check the node logs to see if the node successfully connects to the "
-        "server."
+    # start port forward for the node proxy server
+    start_port_forward(
+        service_name=f"{ctx.helm_release_name}-node-service",
+        service_port=ctx.config["node"].get("port", DEFAULT_PROXY_SERVER_PORT),
+        port=ctx.config["node"].get("port", DEFAULT_PROXY_SERVER_PORT),
+        context=context,
+        namespace=namespace,
     )
 
     if attach:
-        logs = container.attach(stream=True, logs=True)
-        Thread(target=print_log_worker, args=(logs,), daemon=True).start()
-        while True:
-            try:
-                time.sleep(1)
-            except KeyboardInterrupt:
-                info("Closing log file. Keyboard Interrupt.")
-                info(
-                    "Note that your node is still running! Shut it down with "
-                    f"'{Fore.RED}v6 node stop{Style.RESET_ALL}'"
-                )
-                exit(0)
-    else:
-        info(
-            f"To see the logs, run: {Fore.GREEN}v6 node attach --name "
-            f"{ctx.name}{Style.RESET_ALL}"
-        )
+        attach_logs("app=node")