vantage6 5.0.0a36__py3-none-any.whl → 5.0.0a38__py3-none-any.whl

vantage6/cli/server/import_.py

@@ -1,26 +1,19 @@
-import os
-from threading import Thread
-
 import click
-import docker
-from sqlalchemy.engine.url import make_url
+import requests
+import yaml
 
-from vantage6.common import info, warning
-from vantage6.common.docker.addons import check_docker_running, pull_image
+from vantage6.common import error, info
 from vantage6.common.globals import (
-    APPNAME,
-    DEFAULT_DOCKER_REGISTRY,
-    DEFAULT_SERVER_IMAGE,
     InstanceType,
 )
 
+from vantage6.client import UserClient
+
+from vantage6.cli import __version__
 from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.common.utils import print_log_worker
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.utils import check_config_name_allowed
 
 
-# TODO this method has a lot of duplicated code from `start`
 @click.command()
 @click.argument("file", type=click.Path(exists=True))
 @click.option(
@@ -29,121 +22,146 @@ from vantage6.cli.utils import check_config_name_allowed
     default=False,
     help="Drop all existing data before importing",
 )
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source code in this path",
-)
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep image after finishing. Useful for debugging",
-)
-@click.option("--wait", default=False, help="Wait for the import to finish")
 @click_insert_context(type_=InstanceType.SERVER)
-def cli_server_import(
-    ctx: ServerContext,
-    file: str,
-    drop_all: bool,
-    image: str | None,
-    mount_src: str,
-    keep: bool,
-    wait: bool,
-) -> None:
+def cli_server_import(ctx: ServerContext, file: str, drop_all: bool) -> None:
     """
-    Import vantage6 resources into a server instance.
+    Import vantage6 resources into a server instance from a yaml FILE.
 
-    This allows you to create organizations, collaborations, users, tasks, etc
-    from a yaml file.
+    This allows you to create organizations, collaborations, users, tasks, etc. from a
+    yaml FILE. This method expects the server configuration file to be located on the
+    same machine as this method is invoked from.
 
-    The FILE_ argument should be a path to a yaml file containing the vantage6
-    formatted data to import.
+    This import assigns the root role to all users, which contains all permissions. So
+    use this with caution.
+
+    The FILE argument should be a path to a yaml file containing the vantage6 formatted
+    data to import.
     """
-    # will print an error if not
-    check_docker_running()
+    info("Validating server version: ")
+    info(f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}/version")
+    response = requests.get(
+        f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}/version"
+    )
+    if response.status_code != 200:
+        error("Unable to get server version")
+        return
+
+    body = response.json()
+    if "version" not in body:
+        error("Server gave a valid response but did not include the version")
+        return
+
+    # Compare it to this package version
+    server_version = body["version"]
+    if server_version != __version__:
+        error(
+            f"You are using CLI version {__version__} but the server is running "
+            f"version {server_version}. Please use the same version of the CLI and "
+            "server."
+        )
+        return
+
+    info("Loading and validating import file")
+    with open(file, "r") as f:
+        import_data = yaml.safe_load(f)
+    # TODO: validate import file
+
+    client = UserClient(
+        server_url=f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}",
+        auth_url=ctx.config["ui"]["keycloakPublicUrl"],
+        auth_realm=ctx.config["ui"]["keycloakRealm"],
+        auth_client=ctx.config["ui"]["keycloakClient"],
+        log_level="info",
+    )
+
+    info("Authenticate using admin credentials (opens browser for login)")
+    client.authenticate()
 
-    info("Starting server...")
-    info("Finding Docker daemon.")
-    docker_client = docker.from_env()
+    # TODO: validate that the user has the correct permissions to import data
 
-    # check if name is allowed for docker volume, else exit
-    check_config_name_allowed(ctx.name)
+    if drop_all:
+        info("Dropping all existing data")
+        _drop_all(client)
 
-    # pull latest Docker image
-    if image is None:
-        image = ctx.config.get(
-            "image", f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_SERVER_IMAGE}"
+    info("Collecting root role and rule")
+    root_role_id = client.role.list(name="Root", include_root=True)["data"][0]["id"]
+
+    info("Importing organizations")
+    organizations = []
+    for organization in import_data["organizations"]:
+        org = client.organization.create(
+            name=organization["name"],
+            address1=organization["address1"] or "",
+            address2=organization["address2"] or "",
+            zipcode=organization["zipcode"] or "",
+            country=organization["country"] or "",
+            domain=organization["domain"] or "",
         )
-    info(f"Pulling latest server image '{image}'.")
-    pull_image(docker_client, image)
-
-    info("Creating mounts")
-    mounts = [
-        docker.types.Mount("/mnt/config.yaml", str(ctx.config_file), type="bind"),
-        docker.types.Mount("/mnt/import.yaml", str(file), type="bind"),
-    ]
-
-    # FIXME: code duplication with cli_server_start()
-    # try to mount database
-    uri = ctx.config["uri"]
-    url = make_url(uri)
-    environment_vars = None
-
-    if mount_src:
-        mount_src = os.path.abspath(mount_src)
-        mounts.append(docker.types.Mount("/vantage6", mount_src, type="bind"))
-
-    # If host is None, we're dealing with a file-based DB, like SQLite
-    if url.host is None:
-        db_path = url.database
-
-        if not os.path.isabs(db_path):
-            # We're dealing with a relative path here -> make it absolute
-            db_path = ctx.data_dir / url.database
-
-        # basename = os.path.basename(db_path)
-        dirname = os.path.dirname(db_path)
-        os.makedirs(dirname, exist_ok=True)
-
-        # we're mounting the entire folder that contains the database
-        mounts.append(docker.types.Mount("/mnt/database/", dirname, type="bind"))
-
-        environment_vars = {
-            # ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}"
-        }
-
-    else:
-        warning(
-            f"Database could not be transferred, make sure {url.host} "
-            "is reachable from the Docker container"
+        organizations.append(org)
+
+        info(f"Importing users for organization {org['name']}")
+        for user in organization.get("users", []):
+            client.user.create(
+                username=user["username"],
+                password=user["password"],
+                organization=org["id"],
+                roles=[root_role_id],
+            )
+
+    info("Importing collaborations")
+    all_nodes = []
+    for collaboration in import_data["collaborations"]:
+        # Collecting organization ids
+        organization_ids = []
+        for participant in collaboration["participants"]:
+            for org in organizations:
+                if org["name"] == participant["name"]:
+                    organization_ids.append(org["id"])
+
+        col = client.collaboration.create(
+            name=collaboration["name"],
+            organizations=organization_ids,
+            encrypted=collaboration.get("encrypted", False),
         )
-        info("Consider using the docker-compose method to start a server")
-
-    drop_all_ = "--drop-all" if drop_all else ""
-    cmd = f"vserver-local import -c /mnt/config.yaml {drop_all_} /mnt/import.yaml"
-
-    info(cmd)
-
-    info("Run Docker container")
-    container = docker_client.containers.run(
-        image,
-        command=cmd,
-        mounts=mounts,
-        detach=True,
-        labels={
-            f"{APPNAME}-type": InstanceType.SERVER.value,
-            "name": ctx.config_file_name,
-        },
-        environment=environment_vars,
-        auto_remove=not keep,
-        tty=True,
-    )
-    logs = container.logs(stream=True, stdout=True)
-    Thread(target=print_log_worker, args=(logs,), daemon=False).start()
 
-    info(f"Success! container id = {container.id}")
+        info("Registering nodes for collaboration")
+        for participant in collaboration["participants"]:
+            for org in organizations:
+                if org["name"] == participant["name"]:
+                    node = client.node.create(
+                        name=f"{collaboration['name']}-{org['name'].replace(' ', '-')}-node",
+                        organization=org["id"],
+                        collaboration=col["id"],
+                    )
+                    all_nodes.append(node)
 
-    if wait:
-        container.wait()
-        info("Container finished!")
+    return all_nodes
+
+
+def _drop_all(client: UserClient) -> None:
+    """
+    Drop all existing data from the server.
+    """
+    while nodes := client.node.list()["data"]:
+        for node in nodes:
+            info(f"Deleting node {node['name']}")
+            client.node.delete(node["id"])
+
+    while collaborations := client.collaboration.list(scope="global")["data"]:
+        for collaboration in collaborations:
+            info(f"Deleting collaboration {collaboration['name']}")
+            client.collaboration.delete(collaboration["id"], delete_dependents=True)
+
+    # TODO: For some reason, the `delete_dependents` parameter is not working for users,
+    # so we delete them here first.
+    while users := [u for u in client.user.list()["data"] if u["username"] != "admin"]:
+        for user in users:
+            info(f"Deleting user {user['username']}")
+            client.user.delete(user["id"])
+
+    while orgs := [
+        o for o in client.organization.list()["data"] if o["name"] != "root"
+    ]:
+        for org in orgs:
+            info(f"Deleting organization {org['name']}")
+            client.organization.delete(org["id"], delete_dependents=True)

vantage6/cli/server/list.py

@@ -2,7 +2,7 @@ import click
 
 from vantage6.common.globals import InstanceType
 
-from vantage6.cli.common.utils import get_server_configuration_list
+from vantage6.cli.common.list import get_configuration_list
 
 
 @click.command()
@@ -10,4 +10,4 @@ def cli_server_configuration_list() -> None:
     """
     Print the available server configurations.
     """
-    get_server_configuration_list(InstanceType.SERVER)
+    get_configuration_list(InstanceType.SERVER)

vantage6/cli/server/new.py

@@ -9,7 +9,7 @@ from vantage6.common.globals import (
 
 from vantage6.cli.common.new import new
 from vantage6.cli.config import CliConfig
-from vantage6.cli.configuration_wizard import add_common_server_config
+from vantage6.cli.configuration_create import add_common_server_config
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 
 
@@ -57,7 +57,8 @@ def cli_server_new(
     #     print("awefawef")
 
     new(
-        questionnaire_function=server_configuration_questionaire,
+        config_producing_func=server_configuration_questionaire,
+        config_producing_func_args=(name,),
         name=name,
         system_folders=system_folders,
         namespace=namespace,
@@ -92,7 +93,8 @@ def server_configuration_questionaire(instance_name: str) -> dict[str, Any]:
         }
         config["server"]["dev"] = {
             "host_uri": "host.docker.internal",
-            "store_in_local_cluster": True,
+            # TODO v5+ adapt to name space and service name etc, check from sandbox
+            "store_address": "http://vantage6-store-store.service.default.svc.cluster.local",
         }
 
     # TODO v5+ these should be removed, latest should usually be used so question is

vantage6/cli/server/start.py

@@ -3,6 +3,7 @@ import click
 from vantage6.common import info
 from vantage6.common.globals import InstanceType, Ports
 
+from vantage6.cli.common.attach import attach_logs
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.common.start import (
     helm_install,
@@ -10,11 +11,11 @@ from vantage6.cli.common.start import (
     start_port_forward,
 )
 from vantage6.cli.common.utils import (
-    attach_logs,
     create_directory_if_not_exists,
+    select_context_and_namespace,
 )
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.globals import ChartName
+from vantage6.cli.globals import ChartName, InfraComponentName
 
 
 @click.command()
@@ -32,6 +33,7 @@ from vantage6.cli.globals import ChartName
     default=False,
     help="Print server logs to the console after start",
 )
+@click.option("--local-chart-dir", default=None, help="Local chart directory to use")
 @click_insert_context(
     type_=InstanceType.SERVER, include_name=True, include_system_folders=True
 )
@@ -45,13 +47,18 @@ def cli_server_start(
     port: int,
     ui_port: int,
     attach: bool,
+    local_chart_dir: str | None,
 ) -> None:
     """
     Start the server.
     """
     info("Starting server...")
+    prestart_checks(ctx, InstanceType.SERVER, name, system_folders)
 
-    prestart_checks(ctx, InstanceType.SERVER, name, system_folders, context, namespace)
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
 
     create_directory_if_not_exists(ctx.log_dir)
 
@@ -61,11 +68,13 @@ def cli_server_start(
         values_file=ctx.config_file,
         context=context,
         namespace=namespace,
+        local_chart_dir=local_chart_dir,
     )
 
     # port forward for server
     info("Port forwarding for server")
     start_port_forward(
+        # TODO: make this dynamic
         service_name=f"{ctx.helm_release_name}-vantage6-server-service",
         service_port=ctx.config["server"].get("port", Ports.DEV_SERVER.value),
         port=port or ctx.config["server"].get("port", Ports.DEV_SERVER.value),
@@ -86,4 +95,12 @@ def cli_server_start(
     )
 
     if attach:
-        attach_logs("app=vantage6-server", "component=vantage6-server")
+        attach_logs(
+            name,
+            instance_type=InstanceType.SERVER,
+            infra_component=InfraComponentName.SERVER,
+            system_folders=system_folders,
+            context=context,
+            namespace=namespace,
+            is_sandbox=ctx.is_sandbox,
+        )

vantage6/cli/server/stop.py

@@ -32,6 +32,7 @@ def cli_server_stop(
     namespace: str,
     system_folders: bool,
     all_servers: bool,
+    is_sandbox: bool = False,
 ):
     """
     Stop an running server.
@@ -45,6 +46,7 @@ def cli_server_stop(
         namespace=namespace,
         context=context,
         system_folders=system_folders,
+        is_sandbox=is_sandbox,
     )
 
 

vantage6/cli/server/version.py

@@ -1,12 +1,11 @@
 import click
-import docker
+import requests
 
-from vantage6.common import error
-from vantage6.common.docker.addons import check_docker_running
+from vantage6.common import error, info
 from vantage6.common.globals import InstanceType
 
 from vantage6.cli import __version__
-from vantage6.cli.common.utils import get_running_servers, get_server_name
+from vantage6.cli.common.version import get_and_select_ctx
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 
 
@@ -16,22 +15,36 @@ from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 @click.option(
     "--user", "system_folders", flag_value=False, default=DEFAULT_SERVER_SYSTEM_FOLDERS
 )
-def cli_server_version(name: str, system_folders: bool) -> None:
+@click.option("--context", default=None, help="Kubernetes context to use")
+@click.option("--namespace", default=None, help="Kubernetes namespace to use")
+@click.option(
+    "--sandbox", "is_sandbox", flag_value=True, help="Is this a sandbox environment?"
+)
+def cli_server_version(
+    name: str, system_folders: bool, context: str, namespace: str, is_sandbox: bool
+) -> None:
     """
     Print the version of the vantage6 server.
     """
-    check_docker_running()
-    client = docker.from_env()
-
-    running_server_names = get_running_servers(client, InstanceType.SERVER)
-
-    name = get_server_name(
-        name, system_folders, running_server_names, InstanceType.SERVER
+    ctx = get_and_select_ctx(
+        InstanceType.SERVER, name, system_folders, context, namespace, is_sandbox
     )
+    server_config = ctx.config.get("server", {})
+    base_url = server_config.get("baseUrl", "")
+    api_path = server_config.get("apiPath", "")
+    if not base_url:
+        error("No base URL found in server configuration.")
+        return
+    if not api_path:
+        error("No API path found in server configuration.")
+        return
+
+    response = requests.get(f"{base_url}{api_path}/version")
+    if response.status_code != 200:
+        error("Failed to get server version.")
+        return
+    server_version = response.json().get("version", "")
 
-    if name in running_server_names:
-        container = client.containers.get(name)
-        version = container.exec_run(cmd="vserver-local version", stdout=True)
-        click.echo({"server": version.output.decode("utf-8"), "cli": __version__})
-    else:
-        error(f"Server {name} is not running! Cannot provide version...")
+    info("")
+    info(f"Server version: {server_version}")
+    info(f"CLI version: {__version__}")

vantage6/cli/template/algo_store_config.j2

@@ -102,6 +102,9 @@ store:
     port: 7602
   {% endif %}
 
+  # The port to expose the store on in the cluster
+  port: {{ store.port | default(7602) }}
+
   logging:
     # Controls the logging output level. Could be one of the following
     # levels: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET

vantage6/cli/template/auth_config.j2

@@ -111,8 +111,13 @@ keycloak:
         # clientSessionMaxLifespan to higher values to effectively lengthen the session.
         ssoSessionIdleTimeout: {{ keycloak.ssoSessionIdleTimeout | default(1800) }}
 
-        # password policy configuration. If you prefer not to have a
+        # password policy configuration. If you prefer not to have a password policy,
+        # you can set this to an empty string.
+        {% if keycloak.production %}
         passwordPolicy: "length(8) and upperCase(1) and lowerCase(1) and digits(1) and specialChars(1)"
+        {% else %}
+        passwordPolicy: ""
+        {% endif %}
 
         # do not allow users to edit their username - this would lead to problems with
         # syncing the user between keycloak and vantage6 server/store. This setting
@@ -152,16 +157,34 @@ keycloak:
           # assigned admin permissions on initial startup.
           - username: {{ keycloak.adminUser | default('admin') }}
             enabled: true
+            {% if not keycloak.production %}
+            email: admin@vantage6.org
+            firstName: Admin
+            lastName: Admin
+            {% endif %}
             credentials:
               - type: password
+                {% if keycloak.production %}
                 value: {{ keycloak.adminPassword | default('Admin123!') }}
+                {% else %}
+                value: {{ keycloak.adminPassword | default('admin') }}
+                {% endif %}
             requiredActions:
               {% if keycloak.production %}
+              # this requires all users to update their password on first login and
+              # configure two-factor authentication
               - CONFIGURE_TOTP
               - UPDATE_PASSWORD
+              {% elif keycloak.no_password_update_required%}
+              # enable configure OTP only if you want to use two-factor authentication
+              # - CONFIGURE_TOTP
+              # enable password update if you want every user to have to change their
+              # password on first login
+              # - UPDATE_PASSWORD
               {% else %}
               # enable configure OTP only if you want to use two-factor authentication
               # - CONFIGURE_TOTP
+              # this requires all users to update their password on first login
               - UPDATE_PASSWORD
               {% endif %}
           # create a service account user for the backend admin client. The

vantage6/cli/template/node_config.j2

@@ -79,6 +79,8 @@ node:
       name: engineio.server
     - level: warning
       name: sqlalchemy.engine
+    - level: warning
+      name: kubernetes.client.rest
     {% endif %}
 
   {% if node.encryption is defined %}

vantage6/cli/template/server_config.j2

@@ -96,9 +96,9 @@ server:
   {% if server.dev is defined %}
   dev:
     host_uri: {{ server.dev.host_uri | default('host.docker.internal') }}
-    # This should be set to true if you are running the algorithm store in the same
-    # cluster and namespace as the server
-    store_in_local_cluster: {{ server.dev.store_in_local_cluster | default(false) }}
+    # store address of the algorithm store. This can be provided to the server for
+    # a custom address of the local algorithm store within the k8s cluster.
+    store_address: {{ server.dev.store_address | default('http://vantage6-store-store.service.default.svc.cluster.local') }}
   {% endif %}
 
   # If you want to couple algorithm stores to the server on startup, you can add them here.
@@ -109,11 +109,13 @@ server:
     {% for store in server.algorithm_stores %}
     - name: {{ store.name }}
       url: {{ store.url }}
+      api_path: {{ store.api_path }}
     {% endfor %}
   {% else %}
   # algorithm_stores:
-  # - name: local store
+  # - name: Local store
   #   url: http://localhost:7602/api
+  #   api_path: /api
   {% endif %}
 
   # Cleanup settings: This is used to cleanup the database from old results.
@@ -180,6 +182,7 @@ server:
   #   - http://localhost:7600
   {% endif %}
 
+
 rabbitmq:
   # The image of the RabbitMQ server
   image:
@@ -226,7 +229,7 @@ database:
   # The name of the database
   name: {{ database.name | default('vantage6') }}
 
-  #hostpath of the database mount
+  # hostpath of the database mount
   volumePath: {{ database.volumePath }}
 
   # The name of the k8s node where the database is running
@@ -238,7 +241,7 @@ ui:
   # The image of the UI
   image: {{ ui.image | default('harbor2.vantage6.ai/infrastructure/ui:latest') }}
   # Service port for the UI
-  port: 7600
+  port: {{ ui.port | default(7600) }}
   # keycloak realm
   keycloakRealm: {{ ui.keycloakRealm | default('vantage6') }}
   # keycloak client
@@ -253,4 +256,4 @@ ui:
   # Community algorithm store URL.
   communityStoreUrl: {{ ui.communityStoreUrl | default('https://store.cotopaxi.vantage6.ai') }}
   # Community algorithm store API path
-  communityStoreApiPath: {{ ui.communityStoreApiPath | default('/api') }}
+  communityStoreApiPath: {{ ui.communityStoreApiPath | default('/api') }}

vantage6/cli/use/context.py

@@ -1,11 +1,12 @@
 import click
 import questionary
-from kubernetes import config
+from kubernetes import client, config
 
 from vantage6.common import error
 
 from vantage6.cli.config import CliConfig
 from vantage6.cli.utils import switch_context_and_namespace
+from vantage6.cli.utils_kubernetes import configure_kubernetes_client_for_microk8s
 
 
 @click.command()
@@ -14,6 +15,12 @@ def cli_use_context(context: str):
     """
     Set which Kubernetes context to use.
     """
+    # Configure for MicroK8s if needed
+    config.load_kube_config()
+    cfg = client.Configuration.get_default_copy()
+    configure_kubernetes_client_for_microk8s(cfg)
+    client.Configuration.set_default(cfg)
+
     # Get available contexts
     contexts, active_context = config.list_kube_config_contexts()
     context_names = [ctx["name"] for ctx in contexts]