vantage6 4.2.1__py3-none-any.whl → 4.3.0b3__py3-none-any.whl

vantage6/cli/algorithm/update.py

@@ -9,8 +9,14 @@ from vantage6.cli.utils import info
 
 
 @click.command()
-@click.option('-d', '--dir', 'directory', default=None, type=str,
-              help="Directory to put the algorithm into")
+@click.option(
+    "-d",
+    "--dir",
+    "directory",
+    default=None,
+    type=str,
+    help="Directory to put the algorithm into",
+)
 def cli_algorithm_update(directory: str) -> dict:
     """Update an algorithm template
 
@@ -20,10 +26,7 @@ def cli_algorithm_update(directory: str) -> dict:
     """
     if not directory:
         default_dir = str(Path(os.getcwd()))
-        directory = q.text(
-            "Algorithm directory:",
-            default=default_dir
-        ).ask()
+        directory = q.text("Algorithm directory:", default=default_dir).ask()
 
     run_update(directory, overwrite=True)
 

vantage6/cli/algostore/attach.py

@@ -0,0 +1,32 @@
+import click
+import docker
+
+from colorama import Fore, Style
+
+from vantage6.common import error
+from vantage6.common.docker.addons import check_docker_running
+from vantage6.common.globals import APPNAME, InstanceType
+from vantage6.cli.common.decorator import insert_context
+from vantage6.cli.common.start import attach_logs
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+
+
+@click.command()
+@insert_context(InstanceType.ALGORITHM_STORE)
+def cli_algo_store_attach(ctx: AlgorithmStoreContext) -> None:
+    """
+    Show the server logs in the current console.
+    """
+    check_docker_running()
+    client = docker.from_env()
+
+    running_servers = client.containers.list(
+        filters={"label": f"{APPNAME}-type={InstanceType.ALGORITHM_STORE}"}
+    )
+    running_server_names = [container.name for container in running_servers]
+
+    if ctx.docker_container_name in running_server_names:
+        container = client.containers.get(ctx.docker_container_name)
+        attach_logs(container, InstanceType.ALGORITHM_STORE)
+    else:
+        error(f"{Fore.RED}{ctx.name}{Style.RESET_ALL} is not running!")

vantage6/cli/algostore/new.py

@@ -0,0 +1,55 @@
+import click
+from colorama import Fore, Style
+
+from vantage6.common import info, error, check_config_writeable
+from vantage6.common.globals import InstanceType
+from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+from vantage6.cli.configuration_wizard import configuration_wizard
+from vantage6.cli.utils import check_config_name_allowed, prompt_config_name
+
+
+@click.command()
+@click.option(
+    "-n", "--name", default=None, help="name of the configuration you want to use."
+)
+@click.option("--system", "system_folders", flag_value=True)
+@click.option(
+    "--user", "system_folders", flag_value=False, default=DEFAULT_SERVER_SYSTEM_FOLDERS
+)
+def cli_algo_store_new(name: str, system_folders: bool) -> None:
+    """
+    Create a new server configuration.
+    """
+    name = prompt_config_name(name)
+
+    # check if name is allowed for docker volume, else exit
+    check_config_name_allowed(name)
+
+    # check that this config does not exist
+    try:
+        if AlgorithmStoreContext.config_exists(name, system_folders):
+            error(f"Configuration {Fore.RED}{name}{Style.RESET_ALL} already " "exists!")
+            exit(1)
+    except Exception as e:
+        error(e)
+        exit(1)
+
+    # Check that we can write in this folder
+    if not check_config_writeable(system_folders):
+        error("Your user does not have write access to all folders. Exiting")
+        info(
+            f"Create a new server using '{Fore.GREEN}v6 algorithm-store new "
+            f"--user{Style.RESET_ALL}' instead!"
+        )
+        exit(1)
+
+    # create config in ctx location
+    cfg_file = configuration_wizard(InstanceType.ALGORITHM_STORE, name, system_folders)
+    info(f"New configuration created: {Fore.GREEN}{cfg_file}{Style.RESET_ALL}")
+
+    flag = "" if system_folders else "--user"
+    info(
+        f"You can start the algorithm store by running {Fore.GREEN}v6 "
+        f"algorithm-store start {flag}{Style.RESET_ALL}"
+    )

vantage6/cli/algostore/start.py

@@ -0,0 +1,102 @@
+import click
+
+from vantage6.common import info
+from vantage6.common.globals import APPNAME, DEFAULT_ALGO_STORE_IMAGE, InstanceType
+from vantage6.cli.common.start import (
+    attach_logs,
+    check_for_start,
+    get_image,
+    mount_config_file,
+    mount_database,
+    mount_source,
+    pull_image,
+)
+from vantage6.cli.globals import AlgoStoreGlobals
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+from vantage6.cli.common.decorator import insert_context
+
+
+@click.command()
+@click.option("--ip", default=None, help="IP address to listen on")
+@click.option("-p", "--port", default=None, type=int, help="Port to listen on")
+@click.option("-i", "--image", default=None, help="Algorithm store Docker image to use")
+@click.option(
+    "--keep/--auto-remove",
+    default=False,
+    help="Keep image after algorithm store has been stopped. Useful " "for debugging",
+)
+@click.option(
+    "--mount-src",
+    default="",
+    help="Override vantage6 source code in container with the source"
+    " code in this path",
+)
+@click.option(
+    "--attach/--detach",
+    default=False,
+    help="Print server logs to the console after start",
+)
+@insert_context(InstanceType.ALGORITHM_STORE)
+def cli_algo_store_start(
+    ctx: AlgorithmStoreContext,
+    ip: str,
+    port: int,
+    image: str,
+    keep: bool,
+    mount_src: str,
+    attach: bool,
+) -> None:
+    """
+    Start the algorithm store server.
+    """
+    info("Starting algorithm store...")
+    docker_client = check_for_start(ctx, InstanceType.ALGORITHM_STORE)
+
+    image = get_image(image, ctx, "algorithm-store", DEFAULT_ALGO_STORE_IMAGE)
+
+    pull_image(docker_client, image)
+
+    config_file = "/mnt/config.yaml"
+    mounts = mount_config_file(ctx, config_file)
+
+    src_mount = mount_source(mount_src)
+    if src_mount:
+        mounts.append(src_mount)
+
+    mount, environment_vars = mount_database(ctx, InstanceType.ALGORITHM_STORE)
+    if mount:
+        mounts.append(mount)
+
+    # The `ip` and `port` refer here to the ip and port within the container.
+    # So we do not really care that is it listening on all interfaces.
+    internal_port = 5000
+    cmd = (
+        f"uwsgi --http :{internal_port} --gevent 1000 --http-websockets "
+        "--master --callable app --disable-logging "
+        "--wsgi-file /vantage6/vantage6-algorithm-store/vantage6/algorithm"
+        f"/store/wsgi.py --pyargv {config_file}"
+    )
+    info(cmd)
+
+    info("Run Docker container")
+    port_ = str(port or ctx.config["port"] or AlgoStoreGlobals.PORT)
+    container = docker_client.containers.run(
+        image,
+        command=cmd,
+        mounts=mounts,
+        detach=True,
+        labels={
+            f"{APPNAME}-type": InstanceType.ALGORITHM_STORE,
+            "name": ctx.config_file_name,
+        },
+        environment=environment_vars,
+        ports={f"{internal_port}/tcp": (ip, port_)},
+        name=ctx.docker_container_name,
+        auto_remove=not keep,
+        tty=True,
+    )
+
+    info(f"Success! container id = {container.id}")
+
+    if attach:
+        attach_logs(container, InstanceType.ALGORITHM_STORE)

vantage6/cli/algostore/stop.py

@@ -0,0 +1,60 @@
+import click
+import docker
+from colorama import Fore, Style
+
+from vantage6.common import info, warning, error
+from vantage6.common.docker.addons import (
+    check_docker_running,
+    remove_container_if_exists,
+)
+from vantage6.common.globals import APPNAME, InstanceType
+from vantage6.cli.common.decorator import insert_context
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+
+
+@click.command()
+@insert_context(InstanceType.ALGORITHM_STORE)
+@click.option("--all", "all_servers", flag_value=True, help="Stop all servers")
+def cli_algo_store_stop(ctx: AlgorithmStoreContext, all_servers: bool):
+    """
+    Stop one or all running server(s).
+    """
+    check_docker_running()
+    client = docker.from_env()
+
+    running_servers = client.containers.list(
+        filters={"label": f"{APPNAME}-type={InstanceType.ALGORITHM_STORE}"}
+    )
+
+    if not running_servers:
+        warning("No servers are currently running.")
+        return
+
+    running_server_names = [server.name for server in running_servers]
+
+    if all_servers:
+        for container_name in running_server_names:
+            _stop_algorithm_store(client, container_name)
+        return
+
+    container_name = ctx.docker_container_name
+    if container_name not in running_server_names:
+        error(f"{Fore.RED}{ctx.name}{Style.RESET_ALL} is not running!")
+        return
+
+    _stop_algorithm_store(client, container_name)
+
+
+def _stop_algorithm_store(client, container_name) -> None:
+    """
+    Stop the algorithm store server.
+
+    Parameters
+    ----------
+    client : DockerClient
+        The docker client
+    container_name : str
+        The name of the container to stop
+    """
+    remove_container_if_exists(client, name=container_name)
+    info(f"Stopped the {Fore.GREEN}{container_name}{Style.RESET_ALL} server.")

vantage6/cli/cli.py

@@ -29,10 +29,14 @@ from vantage6.cli.algorithm.create import cli_algorithm_create
 from vantage6.cli.algorithm.update import cli_algorithm_update
 from vantage6.cli.test.feature_tester import cli_test_features
 from vantage6.cli.test.integration_test import cli_test_integration
+from vantage6.cli.algostore.attach import cli_algo_store_attach
+from vantage6.cli.algostore.new import cli_algo_store_new
+from vantage6.cli.algostore.start import cli_algo_store_start
+from vantage6.cli.algostore.stop import cli_algo_store_stop
 
 
 # Define the server group
-@click.group(name='server')
+@click.group(name="server")
 def cli_server() -> None:
     """
     Manage your vantage6 server instances.
@@ -40,16 +44,16 @@ def cli_server() -> None:
 
 
 # Define the commands for the server group
-cli_server.add_command(cli_server_attach, name='attach')
-cli_server.add_command(cli_server_files, name='files')
-cli_server.add_command(cli_server_import, name='import')
-cli_server.add_command(cli_server_configuration_list, name='list')
-cli_server.add_command(cli_server_new, name='new')
-cli_server.add_command(cli_server_remove, name='remove')
-cli_server.add_command(cli_server_shell, name='shell')
-cli_server.add_command(cli_server_start, name='start')
-cli_server.add_command(cli_server_stop, name='stop')
-cli_server.add_command(cli_server_version, name='version')
+cli_server.add_command(cli_server_attach, name="attach")
+cli_server.add_command(cli_server_files, name="files")
+cli_server.add_command(cli_server_import, name="import")
+cli_server.add_command(cli_server_configuration_list, name="list")
+cli_server.add_command(cli_server_new, name="new")
+cli_server.add_command(cli_server_remove, name="remove")
+cli_server.add_command(cli_server_shell, name="shell")
+cli_server.add_command(cli_server_start, name="start")
+cli_server.add_command(cli_server_stop, name="stop")
+cli_server.add_command(cli_server_version, name="version")
 
 
 # Define the node group
@@ -117,8 +121,23 @@ cli_test.add_command(cli_test_features, name="feature-test")
 cli_test.add_command(cli_test_integration, name="integration-test")
 
 
+# Define the algorithm-store group
+@click.group(name="algorithm-store")
+def cli_algo_store() -> None:
+    """
+    Manage your vantage6 algorithm store server instances.
+    """
+
+
+# Define the commands for the test group
+cli_algo_store.add_command(cli_algo_store_attach, name="attach")
+cli_algo_store.add_command(cli_algo_store_new, name="new")
+cli_algo_store.add_command(cli_algo_store_start, name="start")
+cli_algo_store.add_command(cli_algo_store_stop, name="stop")
+
+
 # Define the overall group
-@click.group(name='cli')
+@click.group(name="cli")
 def cli_complete() -> None:
     """
     The `v6` command line interface allows you to manage your vantage6
@@ -134,3 +153,4 @@ cli_complete.add_command(cli_server)
 cli_complete.add_command(cli_dev)
 cli_complete.add_command(cli_algorithm)
 cli_complete.add_command(cli_test)
+cli_complete.add_command(cli_algo_store)

vantage6/cli/common/decorator.py

@@ -0,0 +1,92 @@
+from functools import wraps
+import click
+
+from vantage6.common import error
+from vantage6.common.globals import InstanceType
+from vantage6.cli.configuration_wizard import select_configuration_questionaire
+from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
+from vantage6.cli.context import select_context_class, get_context
+
+
+# TODO to make this decorator usable by nodes as well, we should make the
+# default for --user/--system configurable
+def insert_context(type_: InstanceType) -> callable:
+    """
+    Supply the Click function with an additional context parameter. The context
+    is passed to the function as the first argument.
+
+    Parameters
+    ----------
+    type_ : InstanceType
+        The type of instance for which the context should be inserted
+
+    Returns
+    -------
+    Callable
+        Click function with context
+
+    Examples
+    --------
+    >>> @insert_context(InstanceType.SERVER)
+    >>> def cli_server_start(ctx: ServerContext, *args, **kwargs) -> None:
+    >>>     pass
+    """
+
+    def protection_decorator(func: callable) -> callable:
+        @click.option("-n", "--name", default=None, help="Name of the configuration.")
+        @click.option(
+            "-c",
+            "--config",
+            default=None,
+            help="Absolute path to " "configuration-file; overrides --name",
+        )
+        @click.option(
+            "--system",
+            "system_folders",
+            flag_value=True,
+            help="Use system folders instead of user folders. This " "is the default",
+        )
+        @click.option(
+            "--user",
+            "system_folders",
+            flag_value=False,
+            default=DEFAULT_SERVER_SYSTEM_FOLDERS,
+            help="Use user folders instead of system folders",
+        )
+        @wraps(func)
+        def decorator(
+            name: str, config: str, system_folders: bool, *args, **kwargs
+        ) -> callable:
+            """
+            Decorator function that adds the context to the function.
+
+            Returns
+            -------
+            Callable
+                Decorated function
+            """
+            ctx_class = select_context_class(type_)
+            # path to configuration file always overrides name
+            if config:
+                ctx = ctx_class.from_external_config_file(config, system_folders)
+            elif "ctx" in kwargs:
+                # if ctx is already in kwargs (typically when one click command
+                # calls another internally), use that existing ctx
+                ctx = kwargs.pop("ctx")
+            else:
+                # in case no name, ctx or config file is supplied, ask the user
+                # to select an existing config by name
+                if not name:
+                    try:
+                        # select configuration if none supplied
+                        name = select_configuration_questionaire(type_, system_folders)
+                    except Exception:
+                        error("No configurations could be found!")
+                        exit(1)
+
+                ctx = get_context(type_, name, system_folders)
+            return func(ctx, *args, **kwargs)
+
+        return decorator
+
+    return protection_decorator

vantage6/cli/common/start.py

@@ -0,0 +1,232 @@
+import os
+from threading import Thread
+import time
+import docker
+import enum
+from docker.client import DockerClient
+from docker.models.containers import Container
+from colorama import Fore, Style
+from sqlalchemy.engine.url import make_url
+
+from vantage6.common import error, info, warning
+from vantage6.common.context import AppContext
+from vantage6.common.globals import InstanceType, APPNAME, DEFAULT_DOCKER_REGISTRY
+from vantage6.common.docker.addons import check_docker_running, pull_if_newer
+from vantage6.cli.context import AlgorithmStoreContext, ServerContext
+from vantage6.cli.server.common import print_log_worker
+from vantage6.cli.utils import check_config_name_allowed
+from vantage6.cli.globals import ServerGlobals, AlgoStoreGlobals
+
+
+def check_for_start(ctx: AppContext, type_: InstanceType) -> DockerClient:
+    """
+    Check if all requirements are met to start the instance.
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    type_ : InstanceType
+        The type of instance to check for
+    """
+    # will print an error if not
+    check_docker_running()
+
+    info("Finding Docker daemon.")
+    docker_client = docker.from_env()
+
+    # check if name is allowed for docker volume, else exit
+    check_config_name_allowed(ctx.name)
+
+    # check that this server is not already running
+    running_servers = docker_client.containers.list(
+        filters={"label": f"{APPNAME}-type={type_}"}
+    )
+    for server in running_servers:
+        if server.name == f"{APPNAME}-{ctx.name}-{ctx.scope}-{type_}":
+            error(f"Server {Fore.RED}{ctx.name}{Style.RESET_ALL} " "is already running")
+            exit(1)
+    return docker_client
+
+
+def get_image(
+    image: str, ctx: AppContext, custom_image_key: str, default_image: str
+) -> str:
+    """
+    Get the image name for the given instance type.
+
+    Parameters
+    ----------
+    image : str | None
+        The image name to use if specified
+    ctx : AppContext
+        The context object
+    custom_image_key : str
+        The key to look for in the config file
+    default_image : str
+        The default image name
+
+    Returns
+    -------
+    str
+        The image name to use
+    """
+    # Determine image-name. First we check if the option --image has been used.
+    # Then we check if the image has been specified in the config file, and
+    # finally we use the default settings from the package.
+    if image is None:
+        custom_images: dict = ctx.config.get("images")
+        if custom_images:
+            image = custom_images.get(custom_image_key)
+        if not image:
+            image = f"{DEFAULT_DOCKER_REGISTRY}/{default_image}"
+    return image
+
+
+def pull_image(docker_client: DockerClient, image: str) -> None:
+    """
+    Pull the image if it is not already available.
+
+    Parameters
+    ----------
+    docker : DockerClient
+        The docker client
+    image : str
+        The image name
+    """
+    info(f"Pulling latest image '{image}'.")
+    try:
+        pull_if_newer(docker_client.from_env(), image)
+    except Exception as e:
+        warning(" ... Getting latest server image failed:")
+        warning(f"     {e}")
+    else:
+        info(" ... success!")
+
+
+def mount_config_file(ctx: AppContext, config_file: str) -> list[docker.types.Mount]:
+    """
+    Mount the config file in the container.
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    config_file : str
+        The path to the config file
+
+    Returns
+    -------
+    list[docker.types.Mount]
+        The mounts to use
+    """
+    info("Creating mounts")
+    return [docker.types.Mount(config_file, str(ctx.config_file), type="bind")]
+
+
+def mount_source(mount_src: str) -> docker.types.Mount:
+    """
+    Mount the vantage6 source code in the container.
+
+    Parameters
+    ----------
+    mount_src : str
+        The path to the source code
+
+    Returns
+    -------
+    docker.types.Mount | None
+        The mount to use
+    """
+    if mount_src:
+        mount_src = os.path.abspath(mount_src)
+        return docker.types.Mount("/vantage6", mount_src, type="bind")
+
+
+def mount_database(
+    ctx: ServerContext | AlgorithmStoreContext, type_: InstanceType
+) -> tuple[docker.types.Mount, dict]:
+    """
+    Mount database in the container if it is file-based (e.g. a SQLite DB).
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    type_ : InstanceType
+        The type of instance to mount the database for
+
+    Returns
+    -------
+    docker.types.Mount | None
+        The mount to use
+    dict | None
+        The environment variables to use
+    """
+    # FIXME: code duplication with cli_server_import()
+    # try to mount database
+    uri = ctx.config["uri"]
+    url = make_url(uri)
+    environment_vars = None
+    mount = None
+
+    # If host is None, we're dealing with a file-based DB, like SQLite
+    if url.host is None:
+        db_path = url.database
+
+        if not os.path.isabs(db_path):
+            # We're dealing with a relative path here -> make it absolute
+            db_path = ctx.data_dir / url.database
+
+        basename = os.path.basename(db_path)
+        dirname = os.path.dirname(db_path)
+        os.makedirs(dirname, exist_ok=True)
+
+        # we're mounting the entire folder that contains the database
+        mount = docker.types.Mount("/mnt/database/", dirname, type="bind")
+
+        if type_ == InstanceType.SERVER:
+            environment_vars = {
+                ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}",
+                ServerGlobals.CONFIG_NAME_ENV_VAR.value: ctx.config_file_name,
+            }
+        elif type_ == InstanceType.ALGORITHM_STORE:
+            environment_vars = {
+                AlgoStoreGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}",
+                AlgoStoreGlobals.CONFIG_NAME_ENV_VAR.value: ctx.config_file_name,
+            }
+    else:
+        warning(
+            f"Database could not be transferred, make sure {url.host} "
+            "is reachable from the Docker container"
+        )
+        info("Consider using the docker-compose method to start a server")
+
+    return mount, environment_vars
+
+
+def attach_logs(container: Container, type_: InstanceType) -> None:
+    """
+    Attach container logs to the console if specified.
+
+    Parameters
+    ----------
+    container : Container
+        The container to attach the logs from
+    type_ : InstanceType
+        The type of instance to attach the logs for
+    """
+    if isinstance(type_, enum.Enum):
+        type_ = type_.value
+    logs = container.attach(stream=True, logs=True, stdout=True)
+    Thread(target=print_log_worker, args=(logs,), daemon=True).start()
+    while True:
+        try:
+            time.sleep(1)
+        except KeyboardInterrupt:
+            info("Closing log file. Keyboard Interrupt.")
+            info(
+                "Note that your server is still running! Shut it down "
+                f"with {Fore.RED}v6 {type_} stop{Style.RESET_ALL}"
+            )
+            exit(0)