vantage6 4.2.2__py3-none-any.whl → 4.3.0b3__py3-none-any.whl

vantage6/cli/common/start.py

@@ -0,0 +1,232 @@
+import os
+from threading import Thread
+import time
+import docker
+import enum
+from docker.client import DockerClient
+from docker.models.containers import Container
+from colorama import Fore, Style
+from sqlalchemy.engine.url import make_url
+
+from vantage6.common import error, info, warning
+from vantage6.common.context import AppContext
+from vantage6.common.globals import InstanceType, APPNAME, DEFAULT_DOCKER_REGISTRY
+from vantage6.common.docker.addons import check_docker_running, pull_if_newer
+from vantage6.cli.context import AlgorithmStoreContext, ServerContext
+from vantage6.cli.server.common import print_log_worker
+from vantage6.cli.utils import check_config_name_allowed
+from vantage6.cli.globals import ServerGlobals, AlgoStoreGlobals
+
+
+def check_for_start(ctx: AppContext, type_: InstanceType) -> DockerClient:
+    """
+    Check if all requirements are met to start the instance.
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    type_ : InstanceType
+        The type of instance to check for
+    """
+    # will print an error if not
+    check_docker_running()
+
+    info("Finding Docker daemon.")
+    docker_client = docker.from_env()
+
+    # check if name is allowed for docker volume, else exit
+    check_config_name_allowed(ctx.name)
+
+    # check that this server is not already running
+    running_servers = docker_client.containers.list(
+        filters={"label": f"{APPNAME}-type={type_}"}
+    )
+    for server in running_servers:
+        if server.name == f"{APPNAME}-{ctx.name}-{ctx.scope}-{type_}":
+            error(f"Server {Fore.RED}{ctx.name}{Style.RESET_ALL} " "is already running")
+            exit(1)
+    return docker_client
+
+
+def get_image(
+    image: str, ctx: AppContext, custom_image_key: str, default_image: str
+) -> str:
+    """
+    Get the image name for the given instance type.
+
+    Parameters
+    ----------
+    image : str | None
+        The image name to use if specified
+    ctx : AppContext
+        The context object
+    custom_image_key : str
+        The key to look for in the config file
+    default_image : str
+        The default image name
+
+    Returns
+    -------
+    str
+        The image name to use
+    """
+    # Determine image-name. First we check if the option --image has been used.
+    # Then we check if the image has been specified in the config file, and
+    # finally we use the default settings from the package.
+    if image is None:
+        custom_images: dict = ctx.config.get("images")
+        if custom_images:
+            image = custom_images.get(custom_image_key)
+        if not image:
+            image = f"{DEFAULT_DOCKER_REGISTRY}/{default_image}"
+    return image
+
+
+def pull_image(docker_client: DockerClient, image: str) -> None:
+    """
+    Pull the image if it is not already available.
+
+    Parameters
+    ----------
+    docker : DockerClient
+        The docker client
+    image : str
+        The image name
+    """
+    info(f"Pulling latest image '{image}'.")
+    try:
+        pull_if_newer(docker_client.from_env(), image)
+    except Exception as e:
+        warning(" ... Getting latest server image failed:")
+        warning(f"     {e}")
+    else:
+        info(" ... success!")
+
+
+def mount_config_file(ctx: AppContext, config_file: str) -> list[docker.types.Mount]:
+    """
+    Mount the config file in the container.
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    config_file : str
+        The path to the config file
+
+    Returns
+    -------
+    list[docker.types.Mount]
+        The mounts to use
+    """
+    info("Creating mounts")
+    return [docker.types.Mount(config_file, str(ctx.config_file), type="bind")]
+
+
+def mount_source(mount_src: str) -> docker.types.Mount:
+    """
+    Mount the vantage6 source code in the container.
+
+    Parameters
+    ----------
+    mount_src : str
+        The path to the source code
+
+    Returns
+    -------
+    docker.types.Mount | None
+        The mount to use
+    """
+    if mount_src:
+        mount_src = os.path.abspath(mount_src)
+        return docker.types.Mount("/vantage6", mount_src, type="bind")
+
+
+def mount_database(
+    ctx: ServerContext | AlgorithmStoreContext, type_: InstanceType
+) -> tuple[docker.types.Mount, dict]:
+    """
+    Mount database in the container if it is file-based (e.g. a SQLite DB).
+
+    Parameters
+    ----------
+    ctx : AppContext
+        The context object
+    type_ : InstanceType
+        The type of instance to mount the database for
+
+    Returns
+    -------
+    docker.types.Mount | None
+        The mount to use
+    dict | None
+        The environment variables to use
+    """
+    # FIXME: code duplication with cli_server_import()
+    # try to mount database
+    uri = ctx.config["uri"]
+    url = make_url(uri)
+    environment_vars = None
+    mount = None
+
+    # If host is None, we're dealing with a file-based DB, like SQLite
+    if url.host is None:
+        db_path = url.database
+
+        if not os.path.isabs(db_path):
+            # We're dealing with a relative path here -> make it absolute
+            db_path = ctx.data_dir / url.database
+
+        basename = os.path.basename(db_path)
+        dirname = os.path.dirname(db_path)
+        os.makedirs(dirname, exist_ok=True)
+
+        # we're mounting the entire folder that contains the database
+        mount = docker.types.Mount("/mnt/database/", dirname, type="bind")
+
+        if type_ == InstanceType.SERVER:
+            environment_vars = {
+                ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}",
+                ServerGlobals.CONFIG_NAME_ENV_VAR.value: ctx.config_file_name,
+            }
+        elif type_ == InstanceType.ALGORITHM_STORE:
+            environment_vars = {
+                AlgoStoreGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}",
+                AlgoStoreGlobals.CONFIG_NAME_ENV_VAR.value: ctx.config_file_name,
+            }
+    else:
+        warning(
+            f"Database could not be transferred, make sure {url.host} "
+            "is reachable from the Docker container"
+        )
+        info("Consider using the docker-compose method to start a server")
+
+    return mount, environment_vars
+
+
+def attach_logs(container: Container, type_: InstanceType) -> None:
+    """
+    Attach container logs to the console if specified.
+
+    Parameters
+    ----------
+    container : Container
+        The container to attach the logs from
+    type_ : InstanceType
+        The type of instance to attach the logs for
+    """
+    if isinstance(type_, enum.Enum):
+        type_ = type_.value
+    logs = container.attach(stream=True, logs=True, stdout=True)
+    Thread(target=print_log_worker, args=(logs,), daemon=True).start()
+    while True:
+        try:
+            time.sleep(1)
+        except KeyboardInterrupt:
+            info("Closing log file. Keyboard Interrupt.")
+            info(
+                "Note that your server is still running! Shut it down "
+                f"with {Fore.RED}v6 {type_} stop{Style.RESET_ALL}"
+            )
+            exit(0)

vantage6/cli/configuration_manager.py

@@ -2,6 +2,17 @@ from schema import And, Or, Use, Optional
 
 from vantage6.common.configuration_manager import Configuration, ConfigurationManager
 
+LOGGING_VALIDATORS = {
+    "level": And(
+        Use(str), lambda lvl: lvl in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")
+    ),
+    "use_console": Use(bool),
+    "backup_count": And(Use(int), lambda n: n > 0),
+    "max_size": And(Use(int), lambda b: b > 16),
+    "format": Use(str),
+    "datefmt": Use(str),
+}
+
 
 class ServerConfiguration(Configuration):
     """
@@ -16,18 +27,7 @@ class ServerConfiguration(Configuration):
         "api_path": Use(str),
         "uri": Use(str),
         "allow_drop_all": Use(bool),
-        "logging": {
-            "level": And(
-                Use(str),
-                lambda lvl: lvl in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"),
-            ),
-            "file": Use(str),
-            "use_console": Use(bool),
-            "backup_count": And(Use(int), lambda n: n > 0),
-            "max_size": And(Use(int), lambda b: b > 16),
-            "format": Use(str),
-            "datefmt": Use(str),
-        },
+        "logging": {**LOGGING_VALIDATORS, "file": Use(str)},
     }
 
 
@@ -45,17 +45,7 @@ class NodeConfiguration(Configuration):
         # TODO: remove `dict` validation from databases
         "databases": Or([Use(dict)], dict, None),
         "api_path": Use(str),
-        "logging": {
-            "level": And(
-                Use(str),
-                lambda lvl: lvl in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"),
-            ),
-            "use_console": Use(bool),
-            "backup_count": And(Use(int), lambda n: n > 0),
-            "max_size": And(Use(int), lambda b: b > 16),
-            "format": Use(str),
-            "datefmt": Use(str),
-        },
+        "logging": LOGGING_VALIDATORS,
         "encryption": {"enabled": bool, Optional("private_key"): Use(str)},
         Optional("node_extra_env"): dict,
         Optional("node_extra_mounts"): [str],

vantage6/cli/configuration_wizard.py

@@ -3,14 +3,16 @@ import questionary as q
 from pathlib import Path
 
 from vantage6.common import generate_apikey
-from vantage6.common.globals import DATABASE_TYPES
+from vantage6.common.globals import DATABASE_TYPES, InstanceType
 from vantage6.common.client.node_client import NodeClient
+from vantage6.common.context import AppContext
 from vantage6.common import error, warning
-from vantage6.cli.context import NodeContext, ServerContext
+from vantage6.cli.context import select_context_class
 from vantage6.cli.configuration_manager import (
     NodeConfigurationManager,
     ServerConfigurationManager,
 )
+from vantage6.cli.globals import AlgoStoreGlobals, ServerGlobals
 
 
 def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
@@ -139,29 +141,32 @@ def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
 
     private_key = "" if not encryption else q.text("Path to private key file:").ask()
 
-    config["encryption"] = {
-        "enabled": encryption is True or encryption == "true",
-        "private_key": private_key,
-    }
+    config["encryption"] = {"enabled": encryption == "true", "private_key": private_key}
 
     return config
 
 
-def server_configuration_questionaire(instance_name: str) -> dict:
+def _get_common_server_config(
+    instance_type: InstanceType, instance_name: str, include_api_path: bool = True
+) -> dict:
     """
-    Questionary to generate a config file for the node instance.
+    Part of the questionaire that is common to all server types (vantage6
+    server and algorithm store server).
 
     Parameters
     ----------
+    instance_type : InstanceType
+        Type of server instance.
     instance_name : str
-        Name of the node instance.
+        Name of the server instance.
+    include_api_path : bool
+        Whether to include the api path in the questionaire.
 
     Returns
     -------
     dict
-        Dictionary with the new server configuration
+        Dictionary with new (partial) server configuration
     """
-
     config = q.prompt(
         [
             {
@@ -174,38 +179,99 @@ def server_configuration_questionaire(instance_name: str) -> dict:
                 "type": "text",
                 "name": "port",
                 "message": "Enter port to which the server listens:",
-                "default": "5000",
-            },
-            {
-                "type": "text",
-                "name": "api_path",
-                "message": "Path of the api:",
-                "default": "/api",
-            },
-            {
-                "type": "text",
-                "name": "uri",
-                "message": "Database URI:",
-                "default": "sqlite:///default.sqlite",
-            },
-            {
-                "type": "select",
-                "name": "allow_drop_all",
-                "message": "Allowed to drop all tables: ",
-                "choices": ["True", "False"],
+                "default": (
+                    # Note that .value is required in YAML to get proper str value
+                    ServerGlobals.PORT.value
+                    if instance_type == InstanceType.SERVER
+                    else AlgoStoreGlobals.PORT.value
+                ),
             },
         ]
     )
 
-    constant_jwt_secret = q.confirm("Do you want a constant JWT secret?").ask()
-    if constant_jwt_secret:
-        config["jwt_secret_key"] = generate_apikey()
+    # TODO v5+ remove api_path. It complicates configuration
+    if include_api_path:
+        config.update(
+            q.prompt(
+                [
+                    {
+                        "type": "text",
+                        "name": "api_path",
+                        "message": "Path of the api:",
+                        "default": "/api",
+                    }
+                ]
+            )
+        )
+
+    config.update(
+        q.prompt(
+            [
+                {
+                    "type": "text",
+                    "name": "uri",
+                    "message": "Database URI:",
+                    "default": "sqlite:///default.sqlite",
+                },
+                {
+                    "type": "select",
+                    "name": "allow_drop_all",
+                    "message": "Allowed to drop all tables: ",
+                    "choices": ["True", "False"],
+                },
+            ]
+        )
+    )
 
     res = q.select(
         "Which level of logging would you like?",
         choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "NOTSET"],
     ).ask()
 
+    config["logging"] = {
+        "level": res,
+        "file": f"{instance_name}.log",
+        "use_console": True,
+        "backup_count": 5,
+        "max_size": 1024,
+        "format": "%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s",
+        "datefmt": "%Y-%m-%d %H:%M:%S",
+        "loggers": [
+            {"name": "urllib3", "level": "warning"},
+            {"name": "socketIO-client", "level": "warning"},
+            {"name": "socketio.server", "level": "warning"},
+            {"name": "engineio.server", "level": "warning"},
+            {"name": "sqlalchemy.engine", "level": "warning"},
+            {"name": "requests_oauthlib.oauth2_session", "level": "warning"},
+        ],
+    }
+
+    return config
+
+
+def server_configuration_questionaire(instance_name: str) -> dict:
+    """
+    Questionary to generate a config file for the server instance.
+
+    Parameters
+    ----------
+    instance_name : str
+        Name of the server instance.
+
+    Returns
+    -------
+    dict
+        Dictionary with the new server configuration
+    """
+
+    config = _get_common_server_config(
+        InstanceType.SERVER, instance_name, include_api_path=True
+    )
+
+    constant_jwt_secret = q.confirm("Do you want a constant JWT secret?").ask()
+    if constant_jwt_secret:
+        config["jwt_secret_key"] = generate_apikey()
+
     is_mfa = q.confirm("Do you want to enforce two-factor authentication?").ask()
     if is_mfa:
         config["two_factor_auth"] = is_mfa
@@ -260,35 +326,40 @@ def server_configuration_questionaire(instance_name: str) -> dict:
             "start_with_server": run_rabbit_locally,
         }
 
-    config["logging"] = {
-        "level": res,
-        "file": f"{instance_name}.log",
-        "use_console": True,
-        "backup_count": 5,
-        "max_size": 1024,
-        "format": "%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s",
-        "datefmt": "%Y-%m-%d %H:%M:%S",
-        "loggers": [
-            {"name": "urllib3", "level": "warning"},
-            {"name": "socketIO-client", "level": "warning"},
-            {"name": "socketio.server", "level": "warning"},
-            {"name": "engineio.server", "level": "warning"},
-            {"name": "sqlalchemy.engine", "level": "warning"},
-            {"name": "requests_oauthlib.oauth2_session", "level": "warning"},
-        ],
-    }
+    return config
+
+
+def algo_store_configuration_questionaire(instance_name: str) -> dict:
+    """
+    Questionary to generate a config file for the algorithm store server
+    instance.
 
+    Parameters
+    ----------
+    instance_name : str
+        Name of the server instance.
+
+    Returns
+    -------
+    dict
+        Dictionary with the new server configuration
+    """
+    config = _get_common_server_config(
+        InstanceType.ALGORITHM_STORE, instance_name, include_api_path=False
+    )
     return config
 
 
-def configuration_wizard(type_: str, instance_name: str, system_folders: bool) -> Path:
+def configuration_wizard(
+    type_: InstanceType, instance_name: str, system_folders: bool
+) -> Path:
     """
     Create a configuration file for a node or server instance.
 
     Parameters
     ----------
-    type_ : str
-        Type of the instance. Either "node" or "server"
+    type_ : InstanceType
+        Type of the instance to create a configuration for
     instance_name : str
         Name of the instance
     system_folders : bool
@@ -300,15 +371,18 @@ def configuration_wizard(type_: str, instance_name: str, system_folders: bool) -
         Path to the configuration file
     """
     # for defaults and where to save the config
-    dirs = NodeContext.instance_folders(type_, instance_name, system_folders)
+    dirs = AppContext.instance_folders(type_, instance_name, system_folders)
 
     # invoke questionaire to create configuration file
-    if type_ == "node":
+    if type_ == InstanceType.NODE:
         conf_manager = NodeConfigurationManager
         config = node_configuration_questionaire(dirs, instance_name)
-    else:
+    elif type_ == InstanceType.SERVER:
         conf_manager = ServerConfigurationManager
         config = server_configuration_questionaire(instance_name)
+    else:
+        conf_manager = ServerConfigurationManager
+        config = algo_store_configuration_questionaire(instance_name)
 
     # in the case of an environment we need to add it to the current
     # configuration. In the case of application we can simply overwrite this
@@ -326,15 +400,15 @@ def configuration_wizard(type_: str, instance_name: str, system_folders: bool) -
     return config_file
 
 
-def select_configuration_questionaire(type_: str, system_folders: bool) -> str:
+def select_configuration_questionaire(type_: InstanceType, system_folders: bool) -> str:
     """
     Ask which configuration the user wants to use. It shows only configurations
     that are in the default folder.
 
     Parameters
     ----------
-    type_ : str
-        Type of the instance. Either "node" or "server"
+    type_ : InstanceType
+        Type of the instance to create a configuration for
     system_folders : bool
         Whether to use the system folders or not
 
@@ -343,7 +417,7 @@ def select_configuration_questionaire(type_: str, system_folders: bool) -> str:
     str
         Name of the configuration
     """
-    context = NodeContext if type_ == "node" else ServerContext
+    context = select_context_class(type_)
     configs, _ = context.available_configurations(system_folders)
 
     # each collection (file) can contain multiple configs. (e.g. test,

vantage6/cli/context/__init__.py

@@ -0,0 +1,86 @@
+"""
+The context module in the CLI package contains the Context classes of instances
+started from the CLI, such as nodes and servers. These contexts are related to
+the host system and therefore part of the CLI package.
+
+All classes are derived from the abstract AppContext class and provide the
+vantage6 applications with naming conventions, standard file locations, and
+more.
+"""
+from colorama import Fore, Style
+
+from vantage6.common.globals import InstanceType
+from vantage6.common import error
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+from vantage6.cli.context.node import NodeContext
+from vantage6.cli.context.server import ServerContext
+
+
+def select_context_class(
+    type_: InstanceType,
+) -> ServerContext | NodeContext | AlgorithmStoreContext:
+    """
+    Select the context class based on the type of instance.
+
+    Parameters
+    ----------
+    type_ : InstanceType
+        The type of instance for which the context should be inserted
+
+    Returns
+    -------
+    ServerContext | NodeContext | AlgorithmStoreContext
+        Specialized subclass of AppContext for the given instance type
+
+    Raises
+    ------
+    NotImplementedError
+        If the type_ is not implemented
+    """
+    if type_ == InstanceType.SERVER:
+        return ServerContext
+    elif type_ == InstanceType.ALGORITHM_STORE:
+        return AlgorithmStoreContext
+    elif type_ == InstanceType.NODE:
+        return NodeContext
+    else:
+        raise NotImplementedError
+
+
+def get_context(
+    type_: InstanceType, name: str, system_folders: bool
+) -> ServerContext | NodeContext | AlgorithmStoreContext:
+    """
+    Load the server context from the configuration file.
+
+    Parameters
+    ----------
+    type_ : InstanceType
+        The type of instance to get the context for
+    name : str
+        Name of the instance
+    system_folders : bool
+        Wether to use system folders or if False, the user folders
+
+    Returns
+    -------
+    AppContext
+        Specialized subclass context of AppContext for the given instance type
+    """
+    ctx_class = select_context_class(type_)
+    if not ctx_class.config_exists(name, system_folders):
+        scope = "system" if system_folders else "user"
+        error(
+            f"Configuration {Fore.RED}{name}{Style.RESET_ALL} does not "
+            f"exist in the {Fore.RED}{scope}{Style.RESET_ALL} folders!"
+        )
+        exit(1)
+
+    # We do not want to log this here, we do this in the container and not on
+    # the host. We only want CLI logging here.
+    ctx_class.LOGGING_ENABLED = False
+
+    # create server context, and initialize db
+    ctx = ctx_class(name, system_folders=system_folders)
+
+    return ctx