vantage6 5.0.0a33__py3-none-any.whl → 5.0.0a35__py3-none-any.whl

vantage6/cli/node/set_api_key.py

@@ -1,10 +1,11 @@
 import click
 import questionary as q
 
-from vantage6.common import error, info, ensure_config_dir_writable
+from vantage6.common import ensure_config_dir_writable, error, info
+
+from vantage6.cli.configuration_wizard import NodeConfigurationManager
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli.configuration_wizard import NodeConfigurationManager
 from vantage6.cli.node.common import select_node
 
 
@@ -15,7 +16,7 @@ from vantage6.cli.node.common import select_node
     "--system",
     "system_folders",
     flag_value=True,
-    help="Search for configuration in system folders rather than " "user folders",
+    help="Search for configuration in system folders rather than user folders",
 )
 @click.option(
     "--user",
@@ -50,6 +51,8 @@ def cli_node_set_api_key(name: str, api_key: str, system_folders: bool) -> None:
 
     # set new api key, and save the file
     ctx.config["api_key"] = api_key
+    # TODO v5+ this probably messes up the current config as the template is used...
+    # Fix when reimplementing this in v5
     conf_mgr.put(ctx.config)
     conf_mgr.save(ctx.config_file)
-    info("Your new API key has been uploaded to the config file " f"{ctx.config_file}.")
+    info(f"Your new API key has been uploaded to the config file {ctx.config_file}.")

vantage6/cli/node/start.py

@@ -1,334 +1,111 @@
-import os.path
-import time
-from pathlib import Path
-from threading import Thread
-
 import click
-import docker
-from colorama import Fore, Style
 
-from vantage6.common import debug, error, info, warning
-from vantage6.common.dataclass import TaskDB
-from vantage6.common.docker.addons import (
-    check_docker_running,
-    remove_container_if_exists,
-)
-from vantage6.common.globals import (
-    APPNAME,
-    DEFAULT_DOCKER_REGISTRY,
-    DEFAULT_NODE_IMAGE,
-    DEFAULT_NODE_IMAGE_WO_TAG,
-    InstanceType,
-)
+from vantage6.common import info
+from vantage6.common.globals import InstanceType
 
-from vantage6.cli import __version__
 from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.common.start import pull_infra_image
-from vantage6.cli.common.utils import print_log_worker
+from vantage6.cli.common.start import (
+    helm_install,
+    prestart_checks,
+    start_port_forward,
+)
+from vantage6.cli.common.utils import (
+    attach_logs,
+    create_directory_if_not_exists,
+)
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.node.common import create_client
-from vantage6.cli.utils import check_config_name_allowed
+from vantage6.cli.globals import ChartName
+
+from vantage6.node.globals import DEFAULT_PROXY_SERVER_PORT
 
 
 @click.command()
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep node container after finishing. Useful for debugging",
-)
-@click.option(
-    "--force-db-mount",
-    is_flag=True,
-    help="Always mount node databases; skip the check if they are existing files.",
-)
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option(
     "--attach/--detach",
     default=False,
     help="Show node logs on the current console after starting the node",
 )
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source code in this path",
-)
 @click_insert_context(InstanceType.NODE, include_name=True, include_system_folders=True)
 def cli_node_start(
     ctx: NodeContext,
     name: str,
     system_folders: bool,
-    image: str,
-    keep: bool,
-    mount_src: str,
+    context: str,
+    namespace: str,
     attach: bool,
-    force_db_mount: bool,
 ) -> None:
     """
     Start the node.
     """
-    check_docker_running()
     info("Starting node...")
-    info("Finding Docker daemon")
-    docker_client = docker.from_env()
-    NodeContext.LOGGING_ENABLED = False
-
-    # check if config name is allowed docker name, else exit
-    check_config_name_allowed(ctx.name)
-
-    # check that this node is not already running
-    running_nodes = docker_client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.NODE}"}
-    )
-
-    suffix = "system" if system_folders else "user"
-    for node in running_nodes:
-        if node.name == f"{APPNAME}-{name}-{suffix}":
-            error(f"Node {Fore.RED}{name}{Style.RESET_ALL} is already running")
-            exit(1)
-
-    # make sure the (host)-task and -log dir exists
-    info("Checking that data and log dirs exist")
-    ctx.data_dir.mkdir(parents=True, exist_ok=True)
-    ctx.log_dir.mkdir(parents=True, exist_ok=True)
-
-    # Determine image-name. First we check if the option --image has been used.
-    # Then we check if the image has been specified in the config file, and
-    # finally we use the default settings from the package.
-    if not image:
-        custom_images: dict = ctx.config.get("images")
-        if custom_images:
-            image = custom_images.get("node")
-        else:
-            # if no custom image is specified, find the server version and use
-            # the latest images from that minor version
-            client = create_client(ctx)
-            major_minor = None
-            try:
-                # try to get server version, skip if can't get a connection
-                version = client.util.get_server_version(attempts_on_timeout=3)[
-                    "version"
-                ]
-                major_minor = ".".join(version.split(".")[:2])
-                image = (
-                    f"{DEFAULT_DOCKER_REGISTRY}/"
-                    f"{DEFAULT_NODE_IMAGE_WO_TAG}"
-                    f":{major_minor}"
-                )
-            except Exception:
-                warning("Could not determine server version. Using default node image")
-
-            if major_minor and not __version__.startswith(major_minor):
-                warning(
-                    "Version mismatch between CLI and server/node. CLI is "
-                    f"running on version {__version__}, while node and server "
-                    f"are on version {major_minor}. This might cause "
-                    f"unexpected issues; changing to {major_minor}.<latest> "
-                    "is recommended."
-                )
-
-        # fail safe, in case no custom image is specified and we can't get the
-        # server version
-        if not image:
-            image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
-
-    info(f"Pulling latest node image '{image}'")
-    pull_infra_image(docker_client, image, InstanceType.NODE)
-
-    data_volume = docker_client.volumes.create(ctx.docker_volume_name)
-    vpn_volume = docker_client.volumes.create(ctx.docker_vpn_volume_name)
-    ssh_volume = docker_client.volumes.create(ctx.docker_ssh_volume_name)
-    squid_volume = docker_client.volumes.create(ctx.docker_squid_volume_name)
-
-    info("Creating file & folder mounts")
-    # FIXME: should obtain mount points from DockerNodeContext
-    mounts = [
-        # (target, source)
-        ("/mnt/log", str(ctx.log_dir)),
-        ("/mnt/data", data_volume.name),
-        ("/mnt/vpn", vpn_volume.name),
-        ("/mnt/ssh", ssh_volume.name),
-        ("/mnt/squid", squid_volume.name),
-        ("/mnt/config", str(ctx.config_dir)),
-        ("/var/run/docker.sock", "/var/run/docker.sock"),
-    ]
-
-    if mount_src:
-        # If mount_src is a relative path, docker will consider it a volume.
-        mount_src = os.path.abspath(mount_src)
-        mounts.append(("/vantage6", mount_src))
-
-    # FIXME: Code duplication: Node.__init__() (vantage6/node/__init__.py)
-    #   uses a lot of the same logic. Suggest moving this to
-    #   ctx.get_private_key()
-    filename = ctx.config.get("encryption", {}).get("private_key")
-    # filename may be set to an empty string
-    if not filename:
-        filename = "private_key.pem"
-
-    # Location may be overridden by the environment
-    filename = os.environ.get("PRIVATE_KEY", filename)
-
-    # If ctx.get_data_file() receives an absolute path, it is returned as-is
-    fullpath = Path(ctx.get_data_file(filename))
-    if fullpath:
-        if Path(fullpath).exists():
-            mounts.append(("/mnt/private_key.pem", str(fullpath)))
-        else:
-            warning(f"Private key file is provided {fullpath}, but does not exist")
-
-    # Mount private keys for ssh tunnels
-    ssh_tunnels = ctx.config.get("ssh-tunnels", [])
-    for ssh_tunnel in ssh_tunnels:
-        hostname = ssh_tunnel.get("hostname")
-        key_path = ssh_tunnel.get("ssh", {}).get("identity", {}).get("key")
-        if not key_path:
-            error(
-                f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
-                "key not provided. Continuing to start without this tunnel."
-            )
-        key_path = Path(key_path)
-        if not key_path.exists():
-            error(
-                f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
-                "key does not exist. Continuing to start without this "
-                "tunnel."
-            )
-
-        info(f"  Mounting private key for {hostname} at {key_path}")
-
-        # we remove the .tmp in the container, this is because the file is
-        # mounted in a volume mount point. Somehow the file is than empty in
-        # the volume but not for the node instance. By removing the .tmp we
-        # make sure that the file is not empty in the volume.
-        mounts.append((f"/mnt/ssh/{hostname}.pem.tmp", str(key_path)))
-
-    env = {
-        "DATA_VOLUME_NAME": data_volume.name,
-        "VPN_VOLUME_NAME": vpn_volume.name,
-        "PRIVATE_KEY": "/mnt/private_key.pem",
-    }
-
-    # only mount the DB if it is a file
-    info("Setting up databases")
-    dbs = [TaskDB.from_dict(db) for db in ctx.databases]
-    for db in dbs:
-        # check that label contains only valid characters
-        if not db.label.isidentifier():
-            error(
-                f"Database label {Fore.RED}{db.label}{Style.RESET_ALL} contains"
-                " invalid characters. Only letters, numbers, and underscores"
-                " are allowed, and it cannot start with a number."
-            )
-            exit(1)
-
-        info(
-            f"  Processing {Fore.GREEN}{db.type}{Style.RESET_ALL} database "
-            f"{Fore.GREEN}{db.label}:{db.uri}{Style.RESET_ALL}"
-        )
-        label_capitals = db.label.upper()
-
-        try:
-            db_file_exists = Path(db.uri).exists()
-        except Exception:
-            # If the database uri cannot be parsed, it is definitely not a
-            # file. In case of http servers or sql servers, checking the path
-            # of the the uri will lead to an OS-dependent error, which is why
-            # we catch all exceptions here.
-            db_file_exists = False
-
-        if db.type.is_file_based() and not db_file_exists:
-            error(
-                f"Database {Fore.RED}{db.uri}{Style.RESET_ALL} not found. Databases of "
-                f"type '{db.type}' must be present on the harddrive. Please "
-                "update your node configuration file."
-            )
-            exit(1)
-
-        if not db_file_exists and not force_db_mount:
-            debug("  - non file-based database added")
-            env[f"{label_capitals}_DATABASE_URI"] = db.uri
-        else:
-            debug("  - file-based database added")
-            suffix = Path(db.uri).suffix
-            env[f"{label_capitals}_DATABASE_URI"] = f"{db.label}{suffix}"
-            mounts.append((f"/mnt/{db.label}{suffix}", str(db.uri)))
-
-    system_folders_option = "--system" if system_folders else "--user"
-    cmd = (
-        f"vnode-local start -c /mnt/config/{name}.yaml -n {name} "
-        f" --dockerized {system_folders_option}"
-    )
-
-    volumes = []
-    for mount in mounts:
-        volumes.append(f"{mount[1]}:{mount[0]}")
-
-    extra_mounts = ctx.config.get("node_extra_mounts", [])
-    for mount in extra_mounts:
-        volumes.append(mount)
-
-    extra_env = ctx.config.get("node_extra_env", {})
-    # all extra env var names should be valid identifiers
-    extra_env_invalid = [key for key in extra_env.keys() if not key.isidentifier()]
-    if extra_env_invalid:
-        error(
-            "Environment variable names should be valid identifiers. "
-            f"The following break this rule: {extra_env_invalid}"
-        )
-        exit(1)
-    # we won't accept overwrites of existing env vars
-    env_overwrites = extra_env.keys() & env.keys()
-    if env_overwrites:
-        error(f"Cannot overwrite existing node environment variables: {env_overwrites}")
-        exit(1)
-    env.update(extra_env)
-
-    # Add extra hosts to the environment
-    extra_hosts = ctx.config.get("node_extra_hosts", {})
-
-    remove_container_if_exists(
-        docker_client=docker_client, name=ctx.docker_container_name
-    )
 
-    info("Running Docker container")
-    container = docker_client.containers.run(
-        image,
-        command=cmd,
-        volumes=volumes,
-        detach=True,
-        labels={
-            f"{APPNAME}-type": InstanceType.NODE.value,
-            "system": str(system_folders),
-            "name": ctx.config_file_name,
-        },
-        environment=env,
-        name=ctx.docker_container_name,
-        auto_remove=not keep,
-        tty=True,
-        extra_hosts=extra_hosts,
+    prestart_checks(ctx, InstanceType.NODE, name, system_folders, context, namespace)
+
+    create_directory_if_not_exists(ctx.log_dir)
+    create_directory_if_not_exists(ctx.data_dir)
+
+    # TODO issue #2256 - run same version node as server
+    # # Determine image-name. First we check if the option --image has been used.
+    # # Then we check if the image has been specified in the config file, and
+    # # finally we use the default settings from the package.
+    # if not image:
+    #     custom_images: dict = ctx.config.get("images")
+    #     if custom_images:
+    #         image = custom_images.get("node")
+    #     else:
+    #         # if no custom image is specified, find the server version and use
+    #         # the latest images from that minor version
+    #         client = create_client(ctx)
+    #         major_minor = None
+    #         try:
+    #             # try to get server version, skip if can't get a connection
+    #             version = client.util.get_server_version(attempts_on_timeout=3)[
+    #                 "version"
+    #             ]
+    #             major_minor = ".".join(version.split(".")[:2])
+    #             image = (
+    #                 f"{DEFAULT_DOCKER_REGISTRY}/"
+    #                 f"{DEFAULT_NODE_IMAGE_WO_TAG}"
+    #                 f":{major_minor}"
+    #             )
+    #         except Exception:
+    #             warning("Could not determine server version. Using default node image")
+
+    #         if major_minor and not __version__.startswith(major_minor):
+    #             warning(
+    #                 "Version mismatch between CLI and server/node. CLI is "
+    #                 f"running on version {__version__}, while node and server "
+    #                 f"are on version {major_minor}. This might cause "
+    #                 f"unexpected issues; changing to {major_minor}.<latest> "
+    #                 "is recommended."
+    #             )
+
+    #     # fail safe, in case no custom image is specified and we can't get the
+    #     # server version
+    #     if not image:
+    #         image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
+
+    # info(f"Pulling latest node image '{image}'")
+    # pull_infra_image(docker_client, image, InstanceType.NODE)
+
+    helm_install(
+        release_name=ctx.helm_release_name,
+        chart_name=ChartName.NODE,
+        values_file=ctx.config_file,
+        context=context,
+        namespace=namespace,
     )
 
-    info("Node container was started!")
-    info(
-        "Please check the node logs to see if the node successfully connects to the "
-        "server."
+    # start port forward for the node proxy server
+    start_port_forward(
+        service_name=f"{ctx.helm_release_name}-node-service",
+        service_port=ctx.config["node"].get("port", DEFAULT_PROXY_SERVER_PORT),
+        port=ctx.config["node"].get("port", DEFAULT_PROXY_SERVER_PORT),
+        context=context,
+        namespace=namespace,
     )
 
     if attach:
-        logs = container.attach(stream=True, logs=True)
-        Thread(target=print_log_worker, args=(logs,), daemon=True).start()
-        while True:
-            try:
-                time.sleep(1)
-            except KeyboardInterrupt:
-                info("Closing log file. Keyboard Interrupt.")
-                info(
-                    "Note that your node is still running! Shut it down with "
-                    f"'{Fore.RED}v6 node stop{Style.RESET_ALL}'"
-                )
-                exit(0)
-    else:
-        info(
-            f"To see the logs, run: {Fore.GREEN}v6 node attach --name "
-            f"{ctx.name}{Style.RESET_ALL}"
-        )
+        attach_logs("app=node")

vantage6/cli/node/stop.py

@@ -1,31 +1,24 @@
-import time
 import click
-import questionary as q
-import docker
 
-from colorama import Fore, Style
-from vantage6.cli.context import NodeContext
+from vantage6.common import info
+from vantage6.common.globals import InstanceType
 
-from vantage6.common import warning, error, info
-from vantage6.common.globals import APPNAME
-from vantage6.common.docker.addons import (
-    check_docker_running,
-    delete_volume_if_exists,
-    get_server_config_name,
-    stop_container,
+from vantage6.cli.common.stop import execute_stop, helm_uninstall, stop_port_forward
+from vantage6.cli.globals import (
+    DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL,
+    InfraComponentName,
 )
-from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-
-from vantage6.cli.node.common import find_running_node_names
 
 
 @click.command()
 @click.option("-n", "--name", default=None, help="Configuration name")
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option(
     "--system",
     "system_folders",
     flag_value=True,
-    help="Search for configuration in system folders instead of " "user folders",
+    help="Search for configuration in system folders instead of user folders",
 )
 @click.option(
     "--user",
@@ -36,96 +29,43 @@ from vantage6.cli.node.common import find_running_node_names
     "system folders. This is the default.",
 )
 @click.option("--all", "all_nodes", flag_value=True, help="Stop all running nodes")
-@click.option(
-    "--force",
-    "force",
-    flag_value=True,
-    help="Kill nodes instantly; don't wait for them to shut down",
-)
 def cli_node_stop(
-    name: str, system_folders: bool, all_nodes: bool, force: bool
+    name: str,
+    context: str,
+    namespace: str,
+    system_folders: bool,
+    all_nodes: bool,
 ) -> None:
     """
     Stop one or all running nodes.
     """
-    check_docker_running()
-    client = docker.from_env()
-
-    running_node_names = find_running_node_names(client)
-
-    if not running_node_names:
-        warning("No nodes are currently running.")
-        return
-
-    if force:
-        warning(
-            "Forcing the node to stop will not terminate helper "
-            "containers, neither will it remove routing rules made on the "
-            "host!"
-        )
-
-    if all_nodes:
-        for container_name in running_node_names:
-            _stop_node(client, container_name, force, system_folders)
-    else:
-        if not name:
-            try:
-                container_name = q.select(
-                    "Select the node you wish to stop:", choices=running_node_names
-                ).unsafe_ask()
-            except KeyboardInterrupt:
-                error("Aborted by user!")
-                return
-        else:
-            post_fix = "system" if system_folders else "user"
-            container_name = f"{APPNAME}-{name}-{post_fix}"
-
-        if container_name in running_node_names:
-            _stop_node(client, container_name, force, system_folders)
-            info(f"Stopped the {Fore.GREEN}{container_name}{Style.RESET_ALL} Node.")
-        else:
-            error(f"{Fore.RED}{name}{Style.RESET_ALL} is not running?!")
-
-
-def _stop_node(
-    client: docker.DockerClient, container_name: str, force: bool, system_folders: bool
-) -> None:
+    execute_stop(
+        stop_function=_stop_node,
+        instance_type=InstanceType.NODE,
+        infra_component=InfraComponentName.NODE,
+        stop_all=all_nodes,
+        to_stop=name,
+        namespace=namespace,
+        context=context,
+        system_folders=system_folders,
+    )
+
+
+def _stop_node(node_name: str, namespace: str, context: str) -> None:
     """
     Stop a node
 
     Parameters
     ----------
-    client : docker.DockerClient
-        Docker client
-    name : str
-        Name of the node container to stop
-    force : bool
-        Whether to force the node to stop
-    system_folders : bool
-        Whether to use system folders or not
+    node_name : str
+        Name of the node to stop
+    namespace : str
+        Kubernetes namespace to use
+    context : str
+        Kubernetes context to use
     """
-    container = client.containers.get(container_name)
-    # Stop the container. Using stop() gives the container 10s to exit
-    # itself, if not then it will be killed
-    stop_container(container, force)
-
-    # Sleep for 1 second. Not doing so often causes errors that docker volumes deleted
-    # below are 'still in use' when you try to remove them a few ms after the container
-    # has been removed
-    time.sleep(1)
+    helm_uninstall(release_name=node_name, context=context, namespace=namespace)
 
-    # Delete volumes. This is done here rather than within the node container when
-    # it is stopped, because at that point the volumes are still in use. Here, the node
-    # has already been stopped
-    scope = "system" if system_folders else "user"
-    config_name = get_server_config_name(container_name, scope)
-    ctx = NodeContext(config_name, system_folders, print_log_header=False)
+    stop_port_forward(service_name=f"{node_name}-vantage6-node-service")
 
-    # Do not delete the data volume as this would remove all sessions.
-    for volume in [
-        # ctx.docker_volume_name,
-        ctx.docker_squid_volume_name,
-        ctx.docker_ssh_volume_name,
-        ctx.docker_vpn_volume_name,
-    ]:
-        delete_volume_if_exists(client, volume)
+    info(f"Node {node_name} stopped successfully.")

vantage6/cli/server/import_.py

@@ -17,7 +17,6 @@ from vantage6.common.globals import (
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.common.utils import print_log_worker
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.globals import ServerGlobals
 from vantage6.cli.utils import check_config_name_allowed
 
 
@@ -111,7 +110,7 @@ def cli_server_import(
         mounts.append(docker.types.Mount("/mnt/database/", dirname, type="bind"))
 
         environment_vars = {
-            ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}"
+            # ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}"
         }
 
     else:

vantage6/cli/server/list.py

@@ -1,6 +1,5 @@
 import click
 
-from vantage6.common.docker.addons import check_docker_running
 from vantage6.common.globals import InstanceType
 
 from vantage6.cli.common.utils import get_server_configuration_list
@@ -11,6 +10,4 @@ def cli_server_configuration_list() -> None:
     """
     Print the available server configurations.
     """
-    check_docker_running()
-
     get_server_configuration_list(InstanceType.SERVER)