vantage6 5.0.0a33__py3-none-any.whl → 5.0.0a35__py3-none-any.whl

vantage6/cli/server/new.py

@@ -1,23 +1,34 @@
+from typing import Any
+
 import click
-from colorama import Fore, Style
+import questionary as q
 
-from vantage6.common import ensure_config_dir_writable, error, info
-from vantage6.common.globals import InstanceType
+from vantage6.common.globals import (
+    InstanceType,
+)
 
+from vantage6.cli.common.new import new
 from vantage6.cli.config import CliConfig
-from vantage6.cli.configuration_wizard import configuration_wizard
-from vantage6.cli.context.server import ServerContext
+from vantage6.cli.configuration_wizard import add_common_server_config
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
-from vantage6.cli.utils import check_config_name_allowed, prompt_config_name
 
 
 @click.command()
 @click.option(
     "-n", "--name", default=None, help="name of the configuration you want to use."
 )
-@click.option("--system", "system_folders", flag_value=True)
 @click.option(
-    "--user", "system_folders", flag_value=False, default=DEFAULT_SERVER_SYSTEM_FOLDERS
+    "--system",
+    "system_folders",
+    flag_value=True,
+    help="Use system folders instead of user folders. This is the default",
+)
+@click.option(
+    "--user",
+    "system_folders",
+    flag_value=False,
+    default=DEFAULT_SERVER_SYSTEM_FOLDERS,
+    help="Use user folders instead of system folders",
 )
 @click.option("--context", default=None, help="Kubernetes context to use")
 @click.option(
@@ -34,46 +45,65 @@ def cli_server_new(
     """
     Create a new server configuration.
     """
-    cli_config = CliConfig()
-    context, namespace = cli_config.compare_changes_config(
-        context=context,
+    new(
+        questionnaire_function=server_configuration_questionaire,
+        name=name,
+        system_folders=system_folders,
         namespace=namespace,
+        context=context,
+        type_=InstanceType.SERVER,
     )
 
-    name = prompt_config_name(name)
 
-    # check if name is valid
-    check_config_name_allowed(name)
+def server_configuration_questionaire(instance_name: str) -> dict[str, Any]:
+    """
+    Kubernetes-specific questionnaire to generate Helm values for server.
 
-    # check that this config does not exist
-    try:
-        if ServerContext.config_exists(name, system_folders):
-            error(f"Configuration {Fore.RED}{name}{Style.RESET_ALL} already " "exists!")
-            exit(1)
-    except Exception as e:
-        error(e)
-        exit(1)
+    Parameters
+    ----------
+    instance_name : str
+        Name of the server instance.
 
-    # Check that we can write in this folder
-    if not ensure_config_dir_writable(system_folders):
-        error("Your user does not have write access to all folders. Exiting")
-        info(
-            f"Create a new server using '{Fore.GREEN}v6 server new "
-            f"--user{Style.RESET_ALL}' instead!"
-        )
-        exit(1)
+    Returns
+    -------
+    dict[str, Any]
+        dictionary with Helm values for the server configuration
+    """
+    # Get active kube namespace
+    cli_config = CliConfig()
+    kube_namespace = cli_config.get_last_namespace()
 
-    # create config in ctx location
-    try:
-        cfg_file = configuration_wizard(InstanceType.SERVER, name, system_folders)
-    except KeyboardInterrupt:
-        error("Configuration creation aborted.")
-        exit(1)
-    info(f"New configuration created: {Fore.GREEN}{cfg_file}{Style.RESET_ALL}")
+    # Initialize config with basic structure
+    config = {"server": {}, "database": {}, "ui": {}, "rabbitmq": {}}
 
-    # info(f"root user created.")
-    flag = "" if system_folders else "--user"
-    info(
-        f"You can start the server by running {Fore.GREEN}v6 server start "
-        f"{flag}{Style.RESET_ALL}"
+    config, is_production = add_common_server_config(
+        config, InstanceType.SERVER, instance_name
     )
+    if not is_production:
+        config["server"]["jwt"] = {
+            "secret": "constant_development_secret`",
+        }
+        config["server"]["dev"] = {
+            "host_uri": "host.docker.internal",
+            "store_in_local_cluster": True,
+        }
+
+    # TODO v5+ these should be removed, latest should usually be used so question is
+    # not needed. However, for now we want to specify alpha/beta images.
+    # === Server settings ===
+    config["server"]["image"] = q.text(
+        "Server Docker image:",
+        default="harbor2.vantage6.ai/infrastructure/server:latest",
+    ).unsafe_ask()
+
+    # === UI settings ===
+    config["ui"]["image"] = q.text(
+        "UI Docker image:",
+        default="harbor2.vantage6.ai/infrastructure/ui:latest",
+    ).unsafe_ask()
+
+    # === Keycloak settings ===
+    keycloak_url = f"http://vantage6-auth-keycloak.{kube_namespace}.svc.cluster.local"
+    config["server"]["keycloakUrl"] = keycloak_url
+
+    return config

vantage6/cli/server/remove.py

@@ -1,22 +1,21 @@
-import itertools
-from pathlib import Path
-from shutil import rmtree
-
 import click
-import questionary as q
 
-from vantage6.common import info
-from vantage6.common.docker.addons import check_docker_running
 from vantage6.common.globals import InstanceType
+
 from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.common.remove import execute_remove
 from vantage6.cli.context import ServerContext
-from vantage6.cli.utils import remove_file
+from vantage6.cli.globals import InfraComponentName
 
 
 @click.command()
-@click_insert_context(type_=InstanceType.SERVER)
+@click_insert_context(
+    type_=InstanceType.SERVER, include_name=True, include_system_folders=True
+)
 @click.option("-f", "--force", "force", flag_value=True)
-def cli_server_remove(ctx: ServerContext, force: bool) -> None:
+def cli_server_remove(
+    ctx: ServerContext, name: str, system_folders: bool, force: bool
+) -> None:
     """
     Function to remove a server.
 
@@ -27,26 +26,6 @@ def cli_server_remove(ctx: ServerContext, force: bool) -> None:
     force : bool
         Whether to ask for confirmation before removing or not
     """
-    check_docker_running()
-
-    if not force:
-        if not q.confirm(
-            "This server will be deleted permanently including its "
-            "configuration. Are you sure?",
-            default=False,
-        ).ask():
-            info("Server will not be deleted")
-            exit(0)
-
-    # now remove the folders...
-    remove_file(ctx.config_file, "configuration")
-
-    # ensure log files are closed before removing
-    log_dir = Path(ctx.log_file.parent)
-    if log_dir.exists():
-        info(f"Removing log directory: {log_dir}")
-        for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-            handler.close()
-        # remove the whole folder with all the log files. This may also still contain other
-        # files like RabbitMQ configuration etc
-        rmtree(log_dir)
+    execute_remove(
+        ctx, InstanceType.SERVER, InfraComponentName.SERVER, name, system_folders, force
+    )

vantage6/cli/server/shell.py

@@ -29,7 +29,7 @@ def cli_server_shell(ctx: ServerContext) -> None:
     docker_client = docker.from_env()
 
     running_servers = docker_client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.SERVER}"}
+        filters={"label": f"{APPNAME}-type={InstanceType.SERVER.value}"}
     )
 
     if ctx.docker_container_name not in [s.name for s in running_servers]:

vantage6/cli/server/start.py

@@ -1,16 +1,20 @@
 import click
 
 from vantage6.common import info
-from vantage6.common.globals import InstanceType
+from vantage6.common.globals import InstanceType, Ports
 
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.common.start import (
     helm_install,
+    prestart_checks,
     start_port_forward,
 )
-from vantage6.cli.common.utils import attach_logs
-from vantage6.cli.config import CliConfig
+from vantage6.cli.common.utils import (
+    attach_logs,
+    create_directory_if_not_exists,
+)
 from vantage6.cli.context.server import ServerContext
+from vantage6.cli.globals import ChartName
 
 
 @click.command()
@@ -28,9 +32,13 @@ from vantage6.cli.context.server import ServerContext
     default=False,
     help="Print server logs to the console after start",
 )
-@click_insert_context(type_=InstanceType.SERVER)
+@click_insert_context(
+    type_=InstanceType.SERVER, include_name=True, include_system_folders=True
+)
 def cli_server_start(
     ctx: ServerContext,
+    name: str,
+    system_folders: bool,
     context: str,
     namespace: str,
     ip: str,
@@ -42,20 +50,14 @@ def cli_server_start(
     Start the server.
     """
     info("Starting server...")
-    cli_config = CliConfig()
 
-    context, namespace = cli_config.compare_changes_config(
-        context=context,
-        namespace=namespace,
-    )
+    prestart_checks(ctx, InstanceType.SERVER, name, system_folders, context, namespace)
 
-    # check that log directory exists - or create it
-    ctx.log_dir.mkdir(parents=True, exist_ok=True)
+    create_directory_if_not_exists(ctx.log_dir)
 
-    release_name = f"{ctx.name}-{ctx.instance_type}"
     helm_install(
-        release_name=release_name,
-        chart_name="server",
+        release_name=ctx.helm_release_name,
+        chart_name=ChartName.SERVER,
         values_file=ctx.config_file,
         context=context,
         namespace=namespace,
@@ -64,9 +66,9 @@ def cli_server_start(
     # port forward for server
     info("Port forwarding for server")
     start_port_forward(
-        service_name=f"{release_name}-vantage6-server-service",
-        service_port=ctx.config["server"].get("port", 7601),
-        port=port or ctx.config["server"].get("port", 7601),
+        service_name=f"{ctx.helm_release_name}-vantage6-server-service",
+        service_port=ctx.config["server"].get("port", Ports.DEV_SERVER.value),
+        port=port or ctx.config["server"].get("port", Ports.DEV_SERVER.value),
         ip=ip,
         context=context,
         namespace=namespace,
@@ -75,9 +77,9 @@ def cli_server_start(
     # port forward for UI
     info("Port forwarding for UI")
     start_port_forward(
-        service_name=f"{release_name}-vantage6-frontend-service",
-        service_port=ctx.config["ui"].get("port", 7600),
-        port=ui_port or ctx.config["ui"].get("port", 7600),
+        service_name=f"{ctx.helm_release_name}-vantage6-frontend-service",
+        service_port=ctx.config["ui"].get("port", Ports.DEV_UI.value),
+        port=ui_port or ctx.config["ui"].get("port", Ports.DEV_UI.value),
         ip=ip,
         context=context,
         namespace=namespace,

vantage6/cli/server/stop.py

@@ -3,48 +3,68 @@ import click
 from vantage6.common import info
 from vantage6.common.globals import InstanceType
 
-from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.common.stop import helm_uninstall, stop_port_forward
-from vantage6.cli.config import CliConfig
-from vantage6.cli.context.server import ServerContext
+from vantage6.cli.common.stop import execute_stop, helm_uninstall, stop_port_forward
+from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS, InfraComponentName
 
 
 @click.command()
-# add context and namespace options
+@click.option("-n", "--name", default=None, help="Configuration name")
 @click.option("--context", default=None, help="Kubernetes context to use")
 @click.option("--namespace", default=None, help="Kubernetes namespace to use")
-@click_insert_context(type_=InstanceType.SERVER)
+@click.option(
+    "--system",
+    "system_folders",
+    flag_value=True,
+    default=DEFAULT_SERVER_SYSTEM_FOLDERS,
+    help="Search for configuration in system folders instead of user folders. "
+    "This is the default.",
+)
+@click.option(
+    "--user",
+    "system_folders",
+    flag_value=False,
+    help="Search for configuration in the user folders instead of system folders.",
+)
+@click.option("--all", "all_servers", flag_value=True, help="Stop all running servers")
 def cli_server_stop(
-    ctx: ServerContext,
+    name: str,
     context: str,
     namespace: str,
+    system_folders: bool,
+    all_servers: bool,
 ):
     """
     Stop an running server.
     """
-    cli_config = CliConfig()
-
-    context, namespace = cli_config.compare_changes_config(
-        context=context,
+    execute_stop(
+        stop_function=_stop_server,
+        instance_type=InstanceType.SERVER,
+        infra_component=InfraComponentName.SERVER,
+        stop_all=all_servers,
+        to_stop=name,
         namespace=namespace,
+        context=context,
+        system_folders=system_folders,
     )
 
+
+def _stop_server(server_name: str, namespace: str, context: str) -> None:
+    info(f"Stopping server {server_name}...")
+
     # uninstall the helm release
-    info("Stopping server...")
-    release_name = f"{ctx.name}-{ctx.instance_type}"
     helm_uninstall(
-        release_name=release_name,
+        release_name=server_name,
         context=context,
         namespace=namespace,
     )
 
     # stop the port forwarding for server and UI
     stop_port_forward(
-        service_name=f"{release_name}-vantage6-server-service",
+        service_name=f"{server_name}-vantage6-server-service",
     )
 
     stop_port_forward(
-        service_name=f"{release_name}-vantage6-frontend-service",
+        service_name=f"{server_name}-vantage6-frontend-service",
     )
 
-    info("Server stopped successfully.")
+    info(f"Server {server_name} stopped successfully.")

vantage6/cli/template/algo_store_config.j2

@@ -1,22 +1,195 @@
-allow_drop_all: true
-description: 'default_algorithm_store'
-ip: 0.0.0.0
-logging:
-  backup_count: 5
-  datefmt: '%Y-%m-%d %H:%M:%S'
-  file: test-server.log
-  format: '%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s'
-  level: DEBUG
-  max_size: 1024
-  use_console: true
-port: {{ port }}
-uri: sqlite:///default.sqlite
-root_user:
-  v6_server_uri: http://localhost:{{ server_port }}/api
-  username: dev_admin
-dev:
-  host_uri: {{ host_uri }}
-  disable_review: true
-policies:
-  algorithm_view: public
-{{ user_provided_config }}
+# Override the chart name
+nameOverride: {{ nameOverride | default('vantage6-store') }}
+# Optional, by default the Release.Namespace is used
+# namespace: vantage6-store
+store:
+
+  # The description of the store
+  description: {{ store.description | default('Vantage algorithm store version 5') }}
+
+  api_path: {{ store.api_path | default('/store') }}
+
+  # Keycloak configuration
+  keycloakUrl: {{ store.keycloakUrl }}
+  keycloakRealm: {{ store.keycloakRealm | default('vantage6') }}
+  keycloakAdminUsername: {{ store.keycloakAdminUsername | default('admin') }}
+  keycloakAdminPassword: {{ store.keycloakAdminPassword | default('admin') }}
+  keycloakAdminClient: {{ store.keycloakAdminClient | default('backend-admin-client') }}
+  keycloakAdminClientSecret: {{ store.keycloakAdminClientSecret | default('myadminclientsecret') }}
+
+  vantage6ServerUri: {{ store.vantage6ServerUri | default('http://localhost:7601/server') }}
+
+  image: {{ store.image | default('harbor2.vantage6.ai/infrastructure/algorithm-store:latest') }}
+
+  replications: {{ store.replications | default(1) }}
+
+  policies:
+    # Set who is allowed to view the algorithms in the store. Possible values are:
+    # - "public": everyone can view the algorithms
+    # - "authenticated": users with a token valid for this store can view the algorithms
+    # - "private": only users with explicit permission in the algorithm store can view the
+    #   algorithms
+    {% if store.policies is defined %}
+    algorithmView: {{ store.policies.algorithmView | default('public') }}
+    {% else %}
+    algorithmView: public
+    {% endif %}
+
+    # Set the minimum number of reviewers that need to approve an algorithm before it
+    # is available in the store. In case this number is lower than min_reviewing_organizations,
+    # the min_reviewing_organizations still has to be met to proceed with the review process.
+    {% if store.policies is defined %}
+    minReviewers: {{ store.policies.minReviewers | default(2) }}
+    {% else %}
+    minReviewers: 2
+    {% endif %}
+
+    # Define whether or not developers are able to assign reviewers to their own algorithms.
+    {% if store.policies is defined %}
+    assignReviewOwnAlgorithm: {{ store.policies.assignReviewOwnAlgorithm | default(false) }}
+    {% else %}
+    assignReviewOwnAlgorithm: false
+    {% endif %}
+
+    # Define the minimum amount of organizations that must be involved in the review process.
+    {% if store.policies is defined %}
+    minReviewingOrganizations: {{ store.policies.minReviewingOrganizations | default(2) }}
+    {% else %}
+    minReviewingOrganizations: 2
+    {% endif %}
+
+    # Specify the users that are allowed to review algorithms. This is a list of usernames
+    # that identify unique users. This policy works in combination with the permission
+    # system. If this policy is not set, all users with the right permissions are allowed
+    # to review algorithms.
+    {% if store.policies is defined and store.policies.allowedReviewers is defined %}
+    allowedReviewers:
+      {% for reviewer in store.policies.allowedReviewers %}
+      - {{ reviewer }}
+      {% endfor %}
+    {% else %}
+    # allowedReviewers:
+    #   - some_username
+    {% endif %}
+
+    # Specify the users that are allowed to assign reviews. This is a list of usernames
+    # that identify unique users. This policy works in combination with the permission
+    # system. If this policy is not set, all users with the right permissions are allowed
+    # to assign reviews.
+    {% if store.policies is defined and store.policies.allowedReviewAssigners is defined %}
+    allowedReviewAssigners:
+      {% for assigner in store.policies.allowedReviewAssigners %}
+      - {{ assigner }}
+      {% endfor %}
+    {% else %}
+    # allowedReviewAssigners:
+    #   - some_username
+    {% endif %}
+
+  {% if store.dev is defined %}
+  dev:
+    host_uri: {{ store.dev.host_uri | default('host.docker.internal') }}
+    disable_review: {{ store.dev.disable_review | default(false) }}
+    review_own_algorithm: {{ store.dev.review_own_algorithm | default(false) }}
+  {% endif %}
+
+  {% if store.internal is defined %}
+  internal:
+    port: {{ store.internal.port | default(7602) }}
+  {% endif %}
+  {% if store.internal is not defined %}
+  internal:
+    port: 7602
+  {% endif %}
+
+  logging:
+    # Controls the logging output level. Could be one of the following
+    # levels: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET
+    level: {{ store.logging.level | default('INFO') }}
+
+    # Location to the log file
+    file: {{ store.logging.file | default('store.log') }}
+
+    # Size in kb of a single log file
+    max_size: {{ store.logging.max_size | default(1024) }}
+    use_console: {{ store.logging.use_console | default(true) }}
+
+    # Date format for the log file
+    datefmt: "{{ store.logging.datefmt | default('%Y-%m-%d %H:%M:%S') }}"
+
+    # Format for the log file
+    format: "{{ store.logging.format | default('%(asctime)s - %(name)-14s - %(levelname)-8s - %(message)s') }}"
+
+    # Maximum number of log files to keep. Log files are rotated when the size of the
+    # current log file exceeds `max_size`.
+    backup_count: {{ store.logging.backup_count | default(5) }}
+
+    # Loggers to include in the log file
+    loggers:
+    {% if store.logging.loggers is defined %}
+    {% for logger in store.logging.loggers %}
+    - level: {{ logger.level }}
+      name: {{ logger.name }}
+    {% endfor %}
+    {% endif %}
+    {% if store.logging.loggers is not defined %}
+    - level: warning
+      name: urllib3
+    - level: warning
+      name: socketIO-client
+    - level: warning
+      name: socketio.server
+    - level: warning
+      name: engineio.server
+    - level: warning
+      name: sqlalchemy.engine
+    {% endif %}
+
+  # set up with which origins the store should allow CORS requests. The default
+  # is to allow all origins. If you want to restrict this, you can specify a list
+  # of origins here. Below are examples to allow requests from the Cotopaxi UI, and
+  # port 7600 on localhost. Usually, only the UI and the server need to access the
+  # store.
+  {% if store.cors_allowed_origins is defined %}
+  cors_allowed_origins:
+    {% for origin in store.cors_allowed_origins %}
+    - {{ origin }}
+    {% endfor %}
+  {% else %}
+  # cors_allowed_origins:
+  #   - https://portal.cotopaxi.vantage6.ai
+  #   - http://localhost:7600
+  {% endif %}
+
+database:
+  # whether or not to use an external database
+  external: {{ database.external | default(false) }}
+  {% if database.external is defined and database.external %}
+  # the URI of the external database
+  uri: {{ database.uri | default('postgresql://vantage6:vantage6@localhost:5432/vantage6') }}
+  {% else %}
+  # The image of the database
+  image:
+    {% if database.image is defined %}
+    repository: {{ database.image.repository | default('postgres') }}
+    tag: {{ database.image.tag | default('13') }}
+    {% else %}
+    repository: postgres
+    tag: 13
+    {% endif %}
+
+  # The username of the database
+  username: {{ database.username | default('vantage6') }}
+
+  # The password of the database
+  password: {{ database.password | default('vantage6') }}
+
+  # The name of the database
+  name: {{ database.name | default('vantage6') }}
+
+  #hostpath of the database mount
+  volumePath: {{ database.volumePath }}
+
+  # The name of the k8s node where the database is running
+  k8sNodeName: {{ database.k8sNodeName }}
+  {% endif %}