vantage6 5.0.0a21__py3-none-any.whl → 5.0.0a26__py3-none-any.whl

vantage6/cli/server/start.py

@@ -1,65 +1,27 @@
 import click
-import docker
-from docker.client import DockerClient
 
-from vantage6.common import info, warning, error
-from vantage6.common.docker.network_manager import NetworkManager
-from vantage6.common.globals import (
-    APPNAME,
-    DEFAULT_SERVER_IMAGE,
-    DEFAULT_UI_IMAGE,
-    InstanceType,
-)
+from vantage6.common import info
 
-from vantage6.common.globals import Ports
-from vantage6.cli.context.server import ServerContext
-from vantage6.cli.rabbitmq.queue_manager import RabbitMQManager
-from vantage6.cli.server.common import stop_ui
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.common.start import (
-    attach_logs,
-    check_for_start,
-    get_image,
-    mount_database,
-    mount_source,
-    pull_infra_image,
+    helm_install,
+    start_port_forward,
 )
+from vantage6.cli.common.utils import attach_logs
+from vantage6.cli.config import CliConfig
+from vantage6.cli.context.server import ServerContext
 
 
 @click.command()
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
 @click.option("--ip", default=None, help="IP address to listen on")
-@click.option("-p", "--port", default=None, type=int, help="Port to listen on")
-@click.option("-i", "--image", default=None, help="Server Docker image to use")
 @click.option(
-    "--with-ui",
-    "start_ui",
-    flag_value=True,
-    default=False,
-    help="Start the graphical User Interface as well",
+    "-p", "--port", default=None, type=int, help="Port to listen on for the server"
 )
 @click.option(
     "--ui-port", default=None, type=int, help="Port to listen on for the User Interface"
 )
-@click.option(
-    "--with-rabbitmq",
-    "start_rabbitmq",
-    flag_value=True,
-    default=False,
-    help="Start RabbitMQ message broker as local "
-    "container - use in development only",
-)
-@click.option("--rabbitmq-image", default=None, help="RabbitMQ docker image to use")
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep image after server has stopped. Useful for debugging",
-)
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source"
-    " code in this path",
-)
 @click.option(
     "--attach/--detach",
     default=False,
@@ -68,189 +30,57 @@ from vantage6.cli.common.start import (
 @click_insert_context(type_="server")
 def cli_server_start(
     ctx: ServerContext,
+    context: str,
+    namespace: str,
     ip: str,
     port: int,
-    image: str,
-    start_ui: bool,
     ui_port: int,
-    start_rabbitmq: bool,
-    rabbitmq_image: str,
-    keep: bool,
-    mount_src: str,
     attach: bool,
 ) -> None:
     """
     Start the server.
     """
     info("Starting server...")
-    docker_client = check_for_start(ctx, InstanceType.SERVER.value)
+    cli_config = CliConfig()
 
-    image = get_image(image, ctx, "server", DEFAULT_SERVER_IMAGE)
+    context, namespace = cli_config.compare_changes_config(
+        context=context,
+        namespace=namespace,
+    )
 
     # check that log directory exists - or create it
     ctx.log_dir.mkdir(parents=True, exist_ok=True)
 
-    info("Pulling server image...")
-    pull_infra_image(docker_client, image, InstanceType.SERVER)
-
-    info("Creating mounts")
-    config_file = "/mnt/config.yaml"
-    mounts = [
-        docker.types.Mount(config_file, str(ctx.config_file), type="bind"),
-        docker.types.Mount("/mnt/log/", str(ctx.log_dir), type="bind"),
-    ]
-
-    src_mount = mount_source(mount_src)
-    if src_mount:
-        mounts.append(src_mount)
-
-    mount, environment_vars = mount_database(ctx, InstanceType.SERVER)
-    if mount:
-        mounts.append(mount)
-
-    # Create a docker network for the server and other services like RabbitMQ
-    # to reside in
-    server_network_mgr = NetworkManager(
-        network_name=f"{APPNAME}-{ctx.name}-{ctx.scope}-network"
+    release_name = f"{ctx.name}-{ctx.instance_type}"
+    helm_install(
+        release_name=release_name,
+        chart_name="server",
+        values_file=ctx.config_file,
+        context=context,
+        namespace=namespace,
     )
-    server_network_mgr.create_network(is_internal=False)
-
-    if (
-        start_rabbitmq
-        or ctx.config.get("rabbitmq")
-        and ctx.config["rabbitmq"].get("start_with_server", False)
-    ):
-        # Note that ctx.data_dir has been created at this point, which is
-        # required for putting some RabbitMQ configuration files inside
-        info("Starting RabbitMQ container")
-        _start_rabbitmq(ctx, rabbitmq_image, server_network_mgr)
-    elif ctx.config.get("rabbitmq"):
-        info(
-            "RabbitMQ is provided in the config file as external service. "
-            "Assuming this service is up and running."
-        )
-    else:
-        warning(
-            "Message queue is not set up! This means that the vantage6 server "
-            "cannot be scaled horizontally!"
-        )
-
-    # start the UI if requested
-    if start_ui or ctx.config.get("ui") and ctx.config["ui"].get("enabled"):
-        _start_ui(docker_client, ctx, ui_port)
 
-    # The `ip` and `port` refer here to the ip and port within the container.
-    # So we do not really care that is it listening on all interfaces.
-    internal_port = 5000
-    cmd = (
-        f"uwsgi --http :{internal_port} --gevent 1000 --http-websockets "
-        "--master --callable app --disable-logging "
-        "--wsgi-file /vantage6/vantage6-server/vantage6/server/wsgi.py "
-        f"--pyargv {config_file}"
+    # port forward for server
+    info("Port forwarding for server")
+    start_port_forward(
+        service_name=f"{release_name}-vantage6-server-service",
+        service_port=ctx.config["server"].get("port", 7601),
+        port=port or ctx.config["server"].get("port", 7601),
+        ip=ip,
+        context=context,
+        namespace=namespace,
     )
-    info(cmd)
 
-    info("Run Docker container")
-    port_ = str(port or ctx.config["port"] or Ports.DEV_SERVER.value)
-    container = docker_client.containers.run(
-        image,
-        command=cmd,
-        mounts=mounts,
-        detach=True,
-        labels={
-            f"{APPNAME}-type": InstanceType.SERVER.value,
-            "name": ctx.config_file_name,
-        },
-        environment=environment_vars,
-        ports={f"{internal_port}/tcp": (ip, port_)},
-        name=ctx.docker_container_name,
-        auto_remove=not keep,
-        tty=True,
-        network=server_network_mgr.network_name,
+    # port forward for UI
+    info("Port forwarding for UI")
+    start_port_forward(
+        service_name=f"{release_name}-vantage6-frontend-service",
+        service_port=ctx.config["ui"].get("port", 7600),
+        port=ui_port or ctx.config["ui"].get("port", 7600),
+        ip=ip,
+        context=context,
+        namespace=namespace,
     )
 
-    info(f"Success! container id = {container.id}")
-
     if attach:
-        attach_logs(container, InstanceType.SERVER)
-
-
-def _start_rabbitmq(
-    ctx: ServerContext, rabbitmq_image: str, network_mgr: NetworkManager
-) -> None:
-    """
-    Start the RabbitMQ container if it is not already running.
-
-    Parameters
-    ----------
-    ctx : ServerContext
-        Server context object
-    rabbitmq_image : str
-        RabbitMQ image to use
-    network_mgr : NetworkManager
-        Network manager object
-    """
-    rabbit_uri = ctx.config["rabbitmq"].get("uri")
-    if not rabbit_uri:
-        error(
-            "No RabbitMQ URI found in the configuration file! Please add"
-            "a 'uri' key to the 'rabbitmq' section of the configuration."
-        )
-        exit(1)
-    # kick off RabbitMQ container
-    rabbit_mgr = RabbitMQManager(ctx=ctx, network_mgr=network_mgr, image=rabbitmq_image)
-    rabbit_mgr.start()
-
-
-def _start_ui(client: DockerClient, ctx: ServerContext, ui_port: int) -> None:
-    """
-    Start the UI container.
-
-    Parameters
-    ----------
-    client : DockerClient
-        Docker client
-    ctx : ServerContext
-        Server context object
-    ui_port : int
-        Port to expose the UI on
-    """
-    # if no port is specified, check if config contains a port
-    ui_config = ctx.config.get("ui")
-    if ui_config and not ui_port:
-        ui_port = ui_config.get("port")
-
-    # check if the port is valid
-    # TODO make function to check if port is valid, and use in more places
-    if not isinstance(ui_port, int) or not 0 < ui_port < 65536:
-        warning(
-            f"UI port '{ui_port}' is not valid! Using default port "
-            f"{Ports.DEV_UI.value}"
-        )
-        ui_port = str(Ports.DEV_UI.value)
-
-    # find image to use
-    image = get_image(None, ctx, "ui", DEFAULT_UI_IMAGE)
-
-    pull_infra_image(client, image, InstanceType.UI)
-
-    # set environment variables
-    env_vars = {
-        "SERVER_URL": f"http://localhost:{ctx.config.get('port')}",
-        "API_PATH": ctx.config.get("api_path"),
-    }
-
-    # stop the UI container if it is already running
-    stop_ui(client, ctx)
-
-    info(f"Starting User Interface at port {ui_port}")
-    ui_container_name = f"{APPNAME}-{ctx.name}-{ctx.scope}-ui"
-    client.containers.run(
-        image,
-        detach=True,
-        labels={f"{APPNAME}-type": "ui", "name": ctx.config_file_name},
-        ports={"80/tcp": (ctx.config.get("ip"), ui_port)},
-        name=ui_container_name,
-        environment=env_vars,
-        tty=True,
-    )
+        attach_logs("app=vantage6-server", "component=vantage6-server")

vantage6/cli/server/stop.py

@@ -1,119 +1,49 @@
 import click
-import questionary as q
-import docker
-from colorama import Fore, Style
-from docker.client import DockerClient
 
-from vantage6.common import info, warning, error
-from vantage6.common.docker.addons import (
-    check_docker_running,
-    remove_container,
-    get_server_config_name,
-    get_container,
-    get_num_nonempty_networks,
-    get_network,
-    delete_network,
-    remove_container_if_exists,
-)
-from vantage6.common.globals import APPNAME, InstanceType
-from vantage6.common import split_rabbitmq_uri
+from vantage6.common import info
+
+from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.config import CliConfig
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
-from vantage6.cli.server.common import get_server_context, stop_ui
+from vantage6.cli.common.stop import helm_uninstall, stop_port_forward
 
 
 @click.command()
-@click.option("-n", "--name", default=None, help="Configuration name")
-@click.option("--system", "system_folders", flag_value=True)
-@click.option(
-    "--user", "system_folders", flag_value=False, default=DEFAULT_SERVER_SYSTEM_FOLDERS
-)
-@click.option("--all", "all_servers", flag_value=True, help="Stop all servers")
-def cli_server_stop(name: str, system_folders: bool, all_servers: bool):
+# add context and namespace options
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click_insert_context(type_="server")
+def cli_server_stop(
+    ctx: ServerContext,
+    context: str,
+    namespace: str,
+):
     """
-    Stop one or all running server(s).
+    Stop an running server.
     """
-    check_docker_running()
-    client = docker.from_env()
+    cli_config = CliConfig()
 
-    running_servers = client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.SERVER.value}"}
+    context, namespace = cli_config.compare_changes_config(
+        context=context,
+        namespace=namespace,
     )
 
-    if not running_servers:
-        warning("No servers are currently running.")
-        return
-
-    running_server_names = [server.name for server in running_servers]
-
-    if all_servers:
-        for container_name in running_server_names:
-            _stop_server_containers(client, container_name, system_folders)
-        return
-
-    # make sure we have a configuration name to work with
-    if not name:
-        try:
-            container_name = q.select(
-                "Select the server you wish to stop:", choices=running_server_names
-            ).unsafe_ask()
-        except KeyboardInterrupt:
-            error("Aborted by user!")
-            return
-    else:
-        post_fix = "system" if system_folders else "user"
-        container_name = f"{APPNAME}-{name}-{post_fix}-{InstanceType.SERVER.value}"
-
-    if container_name not in running_server_names:
-        error(f"{Fore.RED}{name}{Style.RESET_ALL} is not running!")
-        return
-
-    _stop_server_containers(client, container_name, system_folders)
-
-
-def _stop_server_containers(
-    client: DockerClient, container_name: str, system_folders: bool
-) -> None:
-    """
-    Given a server's name, kill its docker container and related (RabbitMQ)
-    containers.
-
-    Parameters
-    ----------
-    client : DockerClient
-        Docker client
-    container_name : str
-        Name of the server to stop
-    system_folders : bool
-        Wether to use system folders or not
-    """
-    # kill the server
-    remove_container_if_exists(client, name=container_name)
-    info(f"Stopped the {Fore.GREEN}{container_name}{Style.RESET_ALL} server.")
-
-    # find the configuration name from the docker container name
-    scope = "system" if system_folders else "user"
-    config_name = get_server_config_name(container_name, scope)
-
-    ctx = get_server_context(config_name, system_folders, ServerContext)
+    # uninstall the helm release
+    info("Stopping server...")
+    release_name = f"{ctx.name}-{ctx.instance_type}"
+    helm_uninstall(
+        release_name=release_name,
+        context=context,
+        namespace=namespace,
+    )
 
-    # kill the UI container (if it exists)
-    stop_ui(client, ctx)
+    # stop the port forwarding for server and UI
+    stop_port_forward(
+        service_name=f"{release_name}-vantage6-server-service",
+    )
 
-    # delete the server network
-    network_name = f"{APPNAME}-{ctx.name}-{ctx.scope}-network"
-    network = get_network(client, name=network_name)
-    delete_network(network, kill_containers=False)
+    stop_port_forward(
+        service_name=f"{release_name}-vantage6-frontend-service",
+    )
 
-    # kill RabbitMQ if it exists and no other servers are using to it (i.e. it
-    # is not in other docker networks with other containers)
-    rabbit_uri = ctx.config.get("rabbitmq", {}).get("uri")
-    if rabbit_uri:
-        rabbit_container_name = split_rabbitmq_uri(rabbit_uri=rabbit_uri)["host"]
-        rabbit_container = get_container(client, name=rabbit_container_name)
-        if rabbit_container and get_num_nonempty_networks(rabbit_container) == 0:
-            remove_container(rabbit_container, kill=True)
-            info(
-                f"Stopped the {Fore.GREEN}{rabbit_container_name}"
-                f"{Style.RESET_ALL} container."
-            )
+    info("Server stopped successfully.")

vantage6/cli/template/algo_store_config.j2

@@ -19,5 +19,4 @@ dev:
   disable_review: true
 policies:
   algorithm_view: public
-  allow_localhost: true
 {{ user_provided_config }}

vantage6/cli/template/node_config.j2

@@ -28,4 +28,4 @@ port: {{ port }}
 server_url: {{ server_url }}
 task_dir: {{ task_dir}}
 task_namespace: {{ task_namespace }}
-{{ user_provided_config }}
+{{- user_provided_config -}}

vantage6/cli/template/server_import_config.j2

@@ -9,8 +9,6 @@ organizations:
   {% if org['make_admin'] %}
   users:
     - username: dev_admin
-      firstname: admin
-      lastname: robot
       password: password
   {% endif %}
   public_key: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQ0FnRUF2eU4wWVZhWWVZcHVWRVlpaDJjeQphTjdxQndCUnB5bVVibnRQNmw2Vk9OOGE1eGwxMmJPTlQyQ1hwSEVGUFhZQTFFZThQRFZwYnNQcVVKbUlseWpRCkgyN0NhZTlIL2lJbUNVNnViUXlnTzFsbG1KRTJQWDlTNXVxendVV3BXMmRxRGZFSHJLZTErUUlDRGtGSldmSEIKWkJkczRXMTBsMWlxK252dkZ4OWY3dk8xRWlLcVcvTGhQUS83Mm52YlZLMG9nRFNaUy9Jc1NnUlk5ZnJVU1FZUApFbGVZWUgwYmI5VUdlNUlYSHRMQjBkdVBjZUV4dXkzRFF5bXh2WTg3bTlkelJsN1NqaFBqWEszdUplSDAwSndjCk80TzJ0WDVod0lLL1hEQ3h4eCt4b3cxSDdqUWdXQ0FybHpodmdzUkdYUC9wQzEvL1hXaVZSbTJWZ3ZqaXNNaisKS2VTNWNaWWpkUkMvWkRNRW1QU29rS2Y4UnBZUk1lZk0xMWtETTVmaWZIQTlPcmY2UXEyTS9SMy90Mk92VDRlRgorUzVJeTd1QWk1N0ROUkFhejVWRHNZbFFxTU5QcUpKYlRtcGlYRWFpUHVLQitZVEdDSC90TXlrRG1JK1dpejNRCjh6SVo1bk1IUnhySFNqSWdWSFdwYnZlTnVaL1Q1aE95aE1uZHU0c3NpRkJyUXN5ZGc1RlVxR3lkdE1JMFJEVHcKSDVBc1ovaFlLeHdiUm1xTXhNcjFMaDFBaDB5SUlsZDZKREY5MkF1UlNTeDl0djNaVWRndEp5VVlYN29VZS9GKwpoUHVwVU4rdWVTUndGQjBiVTYwRXZQWTdVU2RIR1diVVIrRDRzTVQ4Wjk0UVl2S2ZCanU3ZXVKWSs0Mmd2Wm9jCitEWU9ZS05qNXFER2V5azErOE9aTXZNQ0F3RUFBUT09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=

vantage6/cli/test/algo_test_scripts/algo_test_arguments.py

@@ -0,0 +1,29 @@
+average = {
+    "collaboration": 1,
+    "organizations": [1],
+    "name": "test_average_task",
+    "image": "harbor2.vantage6.ai/demo/average",
+    "description": "",
+    "method": "central_average",
+    "arguments": {
+        "column_name": "Age",
+    },
+    "databases": [{"label": "olympic_athletes"}],
+}
+
+kaplan_meier = {
+    "collaboration": 1,
+    "organizations": [1],
+    "name": "test_average_task",
+    "image": "harbor2.vantage6.ai/algorithms/kaplan-meier",
+    "description": "",
+    "method": "kaplan_meier_central",
+    "arguments": {
+        "time_column_name": "days",
+        "censor_column_name": "censor",
+        "organizations_to_include": [1, 2, 3],
+    },
+    "databases": [{"label": "kaplan_meier_test"}],
+}
+
+args = {"average": average, "kaplan_meier": kaplan_meier}

vantage6/cli/test/algo_test_scripts/algo_test_script.py

@@ -0,0 +1,91 @@
+import logging
+import json
+import sys
+
+from vantage6.common.enum import TaskStatus
+
+from vantage6.client import Client
+from vantage6.common import error
+from vantage6.common.globals import Ports
+import vantage6.cli.test.algo_test_scripts.algo_test_arguments as arguments
+
+
+def create_and_run_task(client: Client, task_args: dict, algo_name: str = "algorithm"):
+    """
+    Create and run a task using the provided client and task arguments.
+
+    Parameters
+    ----------
+        client: Client
+            The client instance to use for creating and running the task.
+        task_args: dict
+            The arguments to pass to the task creation method.
+        algo_name: str, optional
+            The name of the algorithm for logging purposes. Default is "algorithm".
+
+    Raises
+    ------
+        AssertionError: If the task fails.
+    """
+    task = client.task.create(**task_args)
+    task_id = task["id"]
+    client.wait_for_results(task_id)
+
+    try:
+        # check if the task has failed
+        assert client.task.get(task_id)["status"] != TaskStatus.FAILED.value
+
+        logging.info("Task for %s completed successfully.", algo_name)
+
+    except AssertionError:
+        error(
+            f"Task for {algo_name} failed. Check the log file of the task "
+            f"{task_id} for more information."
+        )
+        exit(1)
+
+
+def run_test(custom_args: dict | None = None):
+    """
+    Run a test by creating and running tasks using the provided arguments.
+
+    Parameters
+    ----------
+    custom_args: dict, optional
+        The arguments to pass to the task creation method. If not provided,
+        the arguments from the `arguments` module will be used.
+    """
+    # Create a client and authenticate
+    client = Client(
+        server_url=f"http://localhost:{Ports.DEV_SERVER.value}/api",
+        auth_url="http://localhost:8080",
+    )
+    try:
+        client.authenticate()
+    except ConnectionError:
+        error(
+            "Could not connect to the server. Please check if a dev network is running."
+        )
+        exit(1)
+
+    # if custom arguments are provided, use them for running the task
+    if custom_args:
+        create_and_run_task(client, custom_args)
+
+    else:
+        # Run the task for each algorithm in the arguments file
+        for algo, task_args in arguments.args.items():
+            logging.info("Running task for %s", algo)
+            logging.info("Task arguments: %s", task_args)
+            create_and_run_task(client, task_args, algo)
+
+
+if __name__ == "__main__":
+    # check if arguments are provided
+    if len(sys.argv) > 1:
+        input_string = sys.argv[1].replace("'", '"')
+        json_input = json.loads(input_string)
+    else:
+        json_input = None
+
+    run_test(json_input)