vantage6 4.0.2__py3-none-any.whl → 4.1.0b0__py3-none-any.whl

vantage6/cli/node/start.py

@@ -0,0 +1,311 @@
+import sys
+from pathlib import Path
+from threading import Thread
+import time
+import os.path
+
+import click
+import questionary as q
+import docker
+
+from colorama import Fore, Style
+
+from vantage6.common import (
+    warning, error, info, debug,
+    get_database_config
+)
+from vantage6.common.globals import (
+    APPNAME,
+    DEFAULT_DOCKER_REGISTRY,
+    DEFAULT_NODE_IMAGE,
+    DEFAULT_NODE_IMAGE_WO_TAG,
+)
+from vantage6.common.docker.addons import (
+  pull_if_newer,
+  remove_container_if_exists,
+  check_docker_running
+)
+
+from vantage6.cli.context import NodeContext
+from vantage6.cli.globals import (
+    DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+)
+from vantage6.cli.configuration_wizard import (
+    configuration_wizard,
+    select_configuration_questionaire,
+)
+from vantage6.cli.utils import check_config_name_allowed
+from vantage6.cli import __version__
+from vantage6.cli.node.common import print_log_worker, create_client
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="Configuration name")
+@click.option("-c", "--config", default=None,
+              help='Absolute path to configuration-file; overrides NAME')
+@click.option('--system', 'system_folders', flag_value=True,
+              help="Search for the configuration in the system folders")
+@click.option('--user', 'system_folders', flag_value=False, default=N_FOL,
+              help="Search for the configuration in the user folders. This is "
+                   "the default")
+@click.option('-i', '--image', default=None, help="Node Docker image to use")
+@click.option('--keep/--auto-remove', default=False,
+              help="Keep node container after finishing. Useful for debugging")
+@click.option('--force-db-mount', is_flag=True,
+              help="Always mount node databases; skip the check if they are "
+                   "existing files.")
+@click.option('--attach/--detach', default=False,
+              help="Show node logs on the current console after starting the "
+                   "node")
+@click.option('--mount-src', default='',
+              help="Override vantage6 source code in container with the source"
+                   " code in this path")
+def cli_node_start(name: str, config: str, system_folders: bool, image: str,
+                   keep: bool, mount_src: str, attach: bool,
+                   force_db_mount: bool) -> None:
+    """
+    Start the node.
+    """
+    check_docker_running()
+    info("Starting node...")
+    info("Finding Docker daemon")
+    docker_client = docker.from_env()
+    NodeContext.LOGGING_ENABLED = False
+    if config:
+        name = Path(config).stem
+        ctx = NodeContext(name, system_folders, config)
+
+    else:
+        # in case no name is supplied, ask the user to select one
+        if not name:
+            name = select_configuration_questionaire("node", system_folders)
+
+        # check that config exists, if not a questionaire will be invoked
+        if not NodeContext.config_exists(name, system_folders):
+            warning(f"Configuration {Fore.RED}{name}{Style.RESET_ALL} does not"
+                    " exist.")
+
+            if q.confirm("Create this configuration now?").ask():
+                configuration_wizard("node", name, system_folders)
+
+            else:
+                error("Config file couldn't be loaded")
+                sys.exit(0)
+
+        ctx = NodeContext(name, system_folders)
+
+    # check if config name is allowed docker name, else exit
+    check_config_name_allowed(ctx.name)
+
+    # check that this node is not already running
+    running_nodes = docker_client.containers.list(
+        filters={"label": f"{APPNAME}-type=node"}
+    )
+
+    suffix = "system" if system_folders else "user"
+    for node in running_nodes:
+        if node.name == f"{APPNAME}-{name}-{suffix}":
+            error(f"Node {Fore.RED}{name}{Style.RESET_ALL} is already running")
+            exit(1)
+
+    # make sure the (host)-task and -log dir exists
+    info("Checking that data and log dirs exist")
+    ctx.data_dir.mkdir(parents=True, exist_ok=True)
+    ctx.log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Determine image-name. First we check if the option --image has been used.
+    # Then we check if the image has been specified in the config file, and
+    # finally we use the default settings from the package.
+    if not image:
+        custom_images: dict = ctx.config.get('images')
+        if custom_images:
+            image = custom_images.get("node")
+        else:
+            # if no custom image is specified, find the server version and use
+            # the latest images from that minor version
+            client = create_client(ctx)
+            major_minor = None
+            try:
+                # try to get server version, skip if can't get a connection
+                version = client.util.get_server_version(
+                    attempts_on_timeout=3
+                )['version']
+                major_minor = '.'.join(version.split('.')[:2])
+                image = (f"{DEFAULT_DOCKER_REGISTRY}/"
+                         f"{DEFAULT_NODE_IMAGE_WO_TAG}"
+                         f":{major_minor}")
+            except Exception:
+                warning("Could not determine server version. Using default "
+                        "node image")
+
+            if major_minor and not __version__.startswith(major_minor):
+                warning(
+                    "Version mismatch between CLI and server/node. CLI is "
+                    f"running on version {__version__}, while node and server "
+                    f"are on version {major_minor}. This might cause "
+                    f"unexpected issues; changing to {major_minor}.<latest> "
+                    "is recommended."
+                )
+
+        # fail safe, in case no custom image is specified and we can't get the
+        # server version
+        if not image:
+            image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
+
+    info(f"Pulling latest node image '{image}'")
+    try:
+        # docker_client.images.pull(image)
+        pull_if_newer(docker.from_env(), image)
+
+    except Exception as e:
+        warning(' ... Getting latest node image failed:')
+        warning(f"     {e}")
+    else:
+        info(" ... success!")
+
+    info("Creating Docker data volume")
+
+    data_volume = docker_client.volumes.create(ctx.docker_volume_name)
+    vpn_volume = docker_client.volumes.create(ctx.docker_vpn_volume_name)
+    ssh_volume = docker_client.volumes.create(ctx.docker_ssh_volume_name)
+    squid_volume = docker_client.volumes.create(ctx.docker_squid_volume_name)
+
+    info("Creating file & folder mounts")
+    # FIXME: should obtain mount points from DockerNodeContext
+    mounts = [
+        # (target, source)
+        ("/mnt/log", str(ctx.log_dir)),
+        ("/mnt/data", data_volume.name),
+        ("/mnt/vpn", vpn_volume.name),
+        ("/mnt/ssh", ssh_volume.name),
+        ("/mnt/squid", squid_volume.name),
+        ("/mnt/config", str(ctx.config_dir)),
+        ("/var/run/docker.sock", "/var/run/docker.sock"),
+    ]
+
+    if mount_src:
+        # If mount_src is a relative path, docker will consider it a volume.
+        mount_src = os.path.abspath(mount_src)
+        mounts.append(('/vantage6', mount_src))
+
+    # FIXME: Code duplication: Node.__init__() (vantage6/node/__init__.py)
+    #   uses a lot of the same logic. Suggest moving this to
+    #   ctx.get_private_key()
+    filename = ctx.config.get("encryption", {}).get("private_key")
+    # filename may be set to an empty string
+    if not filename:
+        filename = 'private_key.pem'
+
+    # Location may be overridden by the environment
+    filename = os.environ.get('PRIVATE_KEY', filename)
+
+    # If ctx.get_data_file() receives an absolute path, it is returned as-is
+    fullpath = Path(ctx.get_data_file(filename))
+    if fullpath:
+        if Path(fullpath).exists():
+            mounts.append(("/mnt/private_key.pem", str(fullpath)))
+        else:
+            warning(f"private key file provided {fullpath}, "
+                    "but does not exists")
+
+    # Mount private keys for ssh tunnels
+    ssh_tunnels = ctx.config.get("ssh-tunnels", [])
+    for ssh_tunnel in ssh_tunnels:
+        hostname = ssh_tunnel.get("hostname")
+        key_path = ssh_tunnel.get("ssh", {}).get("identity", {}).get("key")
+        if not key_path:
+            error(f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
+                  "key not provided. Continuing to start without this tunnel.")
+        key_path = Path(key_path)
+        if not key_path.exists():
+            error(f"SSH tunnel identity {Fore.RED}{hostname}{Style.RESET_ALL} "
+                  "key does not exist. Continuing to start without this "
+                  "tunnel.")
+
+        info(f"  Mounting private key for {hostname} at {key_path}")
+
+        # we remove the .tmp in the container, this is because the file is
+        # mounted in a volume mount point. Somehow the file is than empty in
+        # the volume but not for the node instance. By removing the .tmp we
+        # make sure that the file is not empty in the volume.
+        mounts.append((f"/mnt/ssh/{hostname}.pem.tmp", str(key_path)))
+
+    env = {
+        "DATA_VOLUME_NAME": data_volume.name,
+        "VPN_VOLUME_NAME": vpn_volume.name,
+        "PRIVATE_KEY": "/mnt/private_key.pem"
+    }
+
+    # only mount the DB if it is a file
+    info("Setting up databases")
+    db_labels = [db['label'] for db in ctx.databases]
+    for label in db_labels:
+
+        db_config = get_database_config(ctx.databases, label)
+        uri = db_config['uri']
+        db_type = db_config['type']
+
+        info(f"  Processing {Fore.GREEN}{db_type}{Style.RESET_ALL} database "
+             f"{Fore.GREEN}{label}:{uri}{Style.RESET_ALL}")
+        label_capitals = label.upper()
+
+        try:
+            file_based = Path(uri).exists()
+        except Exception:
+            # If the database uri cannot be parsed, it is definitely not a
+            # file. In case of http servers or sql servers, checking the path
+            # of the the uri will lead to an OS-dependent error, which is why
+            # we catch all exceptions here.
+            file_based = False
+
+        if not file_based and not force_db_mount:
+            debug('  - non file-based database added')
+            env[f'{label_capitals}_DATABASE_URI'] = uri
+        else:
+            debug('  - file-based database added')
+            suffix = Path(uri).suffix
+            env[f'{label_capitals}_DATABASE_URI'] = f'{label}{suffix}'
+            mounts.append((f'/mnt/{label}{suffix}', str(uri)))
+
+    system_folders_option = "--system" if system_folders else "--user"
+    cmd = f'vnode-local start -c /mnt/config/{name}.yaml -n {name} '\
+          f' --dockerized {system_folders_option}'
+
+    info("Running Docker container")
+    volumes = []
+    for mount in mounts:
+        volumes.append(f'{mount[1]}:{mount[0]}')
+
+    remove_container_if_exists(
+        docker_client=docker_client, name=ctx.docker_container_name
+    )
+
+    container = docker_client.containers.run(
+        image,
+        command=cmd,
+        volumes=volumes,
+        detach=True,
+        labels={
+            f"{APPNAME}-type": "node",
+            "system": str(system_folders),
+            "name": ctx.config_file_name
+        },
+        environment=env,
+        name=ctx.docker_container_name,
+        auto_remove=not keep,
+        tty=True
+    )
+
+    info(f"Success! container id = {container}")
+
+    if attach:
+        logs = container.attach(stream=True, logs=True)
+        Thread(target=print_log_worker, args=(logs,), daemon=True).start()
+        while True:
+            try:
+                time.sleep(1)
+            except KeyboardInterrupt:
+                info("Closing log file. Keyboard Interrupt.")
+                info("Note that your node is still running! Shut it down with "
+                     f"'{Fore.RED}v6 node stop{Style.RESET_ALL}'")
+                exit(0)

vantage6/cli/node/stop.py

@@ -0,0 +1,73 @@
+import click
+import questionary as q
+import docker
+
+from colorama import Fore, Style
+
+from vantage6.common import warning, error, info
+from vantage6.common.globals import APPNAME
+from vantage6.common.docker.addons import check_docker_running
+from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+
+from vantage6.cli.node.common import find_running_node_names
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="Configuration name")
+@click.option('--system', 'system_folders', flag_value=True,
+              help="Search for configuration in system folders instead of "
+                   "user folders")
+@click.option('--user', 'system_folders', flag_value=False, default=N_FOL,
+              help="Search for configuration in the user folders instead of "
+                   "system folders. This is the default.")
+@click.option('--all', 'all_nodes', flag_value=True,
+              help="Stop all running nodes")
+@click.option('--force', 'force', flag_value=True,
+              help="Kill nodes instantly; don't wait for them to shut down")
+def cli_node_stop(name: str, system_folders: bool, all_nodes: bool,
+                  force: bool) -> None:
+    """
+    Stop one or all running nodes.
+    """
+    check_docker_running()
+    client = docker.from_env()
+
+    running_node_names = find_running_node_names(client)
+
+    if not running_node_names:
+        warning("No nodes are currently running.")
+        return
+
+    if force:
+        warning('Forcing the node to stop will not terminate helper '
+                'containers, neither will it remove routing rules made on the '
+                'host!')
+
+    if all_nodes:
+        for name in running_node_names:
+            container = client.containers.get(name)
+            if force:
+                container.kill()
+            else:
+                container.stop()
+            info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
+    else:
+        if not name:
+            name = q.select("Select the node you wish to stop:",
+                            choices=running_node_names).ask()
+        else:
+
+            post_fix = "system" if system_folders else "user"
+            name = f"{APPNAME}-{name}-{post_fix}"
+
+        if name in running_node_names:
+            container = client.containers.get(name)
+            # Stop the container. Using stop() gives the container 10s to exit
+            # itself, if not then it will be killed
+            if force:
+                container.kill()
+            else:
+                container.stop()
+            info(f"Stopped the {Fore.GREEN}{name}{Style.RESET_ALL} Node.")
+        else:
+            error(f"{Fore.RED}{name}{Style.RESET_ALL} is not running?")

vantage6/cli/node/version.py

@@ -0,0 +1,47 @@
+import click
+import questionary as q
+import docker
+
+from vantage6.common import error
+from vantage6.common.globals import APPNAME
+from vantage6.common.docker.addons import check_docker_running
+from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
+from vantage6.cli import __version__
+from vantage6.cli.node.common import find_running_node_names
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="Configuration name")
+@click.option('--system', 'system_folders', flag_value=True,
+              help="Search for configuration in system folders rather than "
+                   "user folders")
+@click.option('--user', 'system_folders', flag_value=False, default=N_FOL,
+              help="Search for configuration in user folders rather than "
+                   "system folders. This is the default")
+def cli_node_version(name: str, system_folders: bool) -> None:
+    """
+    Returns current version of a vantage6 node.
+    """
+    check_docker_running()
+    client = docker.from_env()
+
+    running_node_names = find_running_node_names(client)
+
+    if not name:
+        if not running_node_names:
+            error("No nodes are running! You can only check the version for "
+                  "nodes that are running")
+            exit(1)
+        name = q.select("Select the node you wish to inspect:",
+                        choices=running_node_names).ask()
+    else:
+        post_fix = "system" if system_folders else "user"
+        name = f"{APPNAME}-{name}-{post_fix}"
+
+    if name in running_node_names:
+        container = client.containers.get(name)
+        version = container.exec_run(cmd='vnode-local version', stdout=True)
+        click.echo(
+            {"node": version.output.decode('utf-8'), "cli": __version__})
+    else:
+        error(f"Node {name} is not running! Cannot provide version...")

vantage6/cli/server/attach.py

@@ -0,0 +1,54 @@
+import time
+from threading import Thread
+
+import click
+import questionary as q
+import docker
+
+from colorama import (Fore, Style)
+
+from vantage6.common import info, error
+from vantage6.common.docker.addons import check_docker_running
+from vantage6.common.globals import APPNAME
+
+from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
+from vantage6.cli.server.common import print_log_worker
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="configuration name")
+@click.option('--system', 'system_folders', flag_value=True)
+@click.option('--user', 'system_folders', flag_value=False,
+              default=DEFAULT_SERVER_SYSTEM_FOLDERS)
+def cli_server_attach(name: str, system_folders: bool) -> None:
+    """
+    Show the server logs in the current console.
+    """
+    check_docker_running()
+    client = docker.from_env()
+
+    running_servers = client.containers.list(
+        filters={"label": f"{APPNAME}-type=server"})
+    running_server_names = [node.name for node in running_servers]
+
+    if not name:
+        name = q.select("Select the server you wish to inspect:",
+                        choices=running_server_names).ask()
+    else:
+        post_fix = "system" if system_folders else "user"
+        name = f"{APPNAME}-{name}-{post_fix}-server"
+
+    if name in running_server_names:
+        container = client.containers.get(name)
+        logs = container.attach(stream=True, logs=True, stdout=True)
+        Thread(target=print_log_worker, args=(logs,), daemon=True).start()
+        while True:
+            try:
+                time.sleep(1)
+            except KeyboardInterrupt:
+                info("Closing log file. Keyboard Interrupt.")
+                info("Note that your server is still running! Shut it down "
+                     f"with {Fore.RED}v6 server stop{Style.RESET_ALL}")
+                exit(0)
+    else:
+        error(f"{Fore.RED}{name}{Style.RESET_ALL} was not running!?")

vantage6/cli/server/common/__init__.py

@@ -0,0 +1,146 @@
+from typing import Iterable
+from functools import wraps
+
+from docker.client import DockerClient
+import click
+from colorama import Fore, Style
+
+from vantage6.common import error, info
+from vantage6.common.globals import STRING_ENCODING, APPNAME
+from vantage6.common.docker.addons import remove_container, get_container
+
+from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
+from vantage6.cli.context import ServerContext
+from vantage6.cli.configuration_wizard import select_configuration_questionaire
+
+
+def click_insert_context(func: callable) -> callable:
+    """
+    Supply the Click function with additional context parameters. The context
+    is then passed to the function as the first argument.
+
+    Parameters
+    ----------
+    func : Callable
+        function you want the context to be passed to
+
+    Returns
+    -------
+    Callable
+        Click function with context
+    """
+    @click.option('-n', '--name', default=None,
+                  help="Name of the configuration you want to use.")
+    @click.option('-c', '--config', default=None,
+                  help='Absolute path to configuration-file; overrides NAME')
+    @click.option('--system', 'system_folders', flag_value=True,
+                  help='Use system folders instead of user folders. This is '
+                  'the default')
+    @click.option('--user', 'system_folders', flag_value=False,
+                  default=DEFAULT_SERVER_SYSTEM_FOLDERS,
+                  help='Use user folders instead of system folders')
+    @wraps(func)
+    def func_with_context(name: str, config: str, system_folders: bool, *args,
+                          **kwargs) -> callable:
+        """
+        Decorator function that adds the context to the function.
+
+        Returns
+        -------
+        Callable
+            Decorated function
+        """
+        # path to configuration file always overrides name
+        if config:
+            ctx = ServerContext.from_external_config_file(
+                config,
+                system_folders
+            )
+        elif 'ctx' in kwargs:
+            # if ctx is already in kwargs (typically when one click command
+            # calls another internally), use that existing ctx
+            ctx = kwargs.pop('ctx')
+        else:
+            # in case no name, ctx or config file is supplied, ask the user to
+            # select an existing config by name
+            if not name:
+                try:
+                    # select configuration if none supplied
+                    name = select_configuration_questionaire(
+                        "server", system_folders
+                    )
+                except Exception:
+                    error("No configurations could be found!")
+                    exit(1)
+
+            ctx = get_server_context(name, system_folders)
+        return func(ctx, *args, **kwargs)
+
+    return func_with_context
+
+
+def get_server_context(name: str, system_folders: bool) \
+        -> ServerContext:
+    """
+    Load the server context from the configuration file.
+
+    Parameters
+    ----------
+    name : str
+        Name of the server to inspect
+    system_folders : bool
+        Wether to use system folders or if False, the user folders
+
+    Returns
+    -------
+    ServerContext
+        Server context object
+    """
+    if not ServerContext.config_exists(name, system_folders):
+        scope = "system" if system_folders else "user"
+        error(
+            f"Configuration {Fore.RED}{name}{Style.RESET_ALL} does not "
+            f"exist in the {Fore.RED}{scope}{Style.RESET_ALL} folders!"
+        )
+        exit(1)
+
+    # We do not want to log this here, we do this in the container and not on
+    # the host. We only want CLI logging here.
+    ServerContext.LOGGING_ENABLED = False
+
+    # create server context, and initialize db
+    ctx = ServerContext(name, system_folders=system_folders)
+
+    return ctx
+
+
+def print_log_worker(logs_stream: Iterable[bytes]) -> None:
+    """
+    Print the logs from the docker container to the terminal.
+
+    Parameters
+    ----------
+    logs_stream : Iterable[bytes]
+        Output of the `container.attach(.)` method
+    """
+    for log in logs_stream:
+        print(log.decode(STRING_ENCODING), end="")
+
+
+def stop_ui(client: DockerClient, ctx: ServerContext) -> None:
+    """
+    Check if the UI container is running, and if so, stop and remove it.
+
+    Parameters
+    ----------
+    client : DockerClient
+        Docker client
+    ctx : ServerContext
+        Server context object
+    """
+    ui_container_name = f"{APPNAME}-{ctx.name}-{ctx.scope}-ui"
+    ui_container = get_container(client, name=ui_container_name)
+    if ui_container:
+        remove_container(ui_container, kill=True)
+        info(f"Stopped the {Fore.GREEN}{ui_container_name}"
+             f"{Style.RESET_ALL} User Interface container.")

vantage6/cli/server/files.py

@@ -0,0 +1,16 @@
+import click
+
+from vantage6.common import info
+from vantage6.cli.context import ServerContext
+from vantage6.cli.server.common import click_insert_context
+
+
+@click.command()
+@click_insert_context
+def cli_server_files(ctx: ServerContext) -> None:
+    """
+    List files that belong to a particular server instance.
+    """
+    info(f"Configuration file = {ctx.config_file}")
+    info(f"Log file           = {ctx.log_file}")
+    info(f"Database           = {ctx.get_database_uri()}")