vantage6 5.0.0a35__py3-none-any.whl → 5.0.0a37__py3-none-any.whl

vantage6/cli/server/import_.py

@@ -1,26 +1,19 @@
-import os
-from threading import Thread
-
 import click
-import docker
-from sqlalchemy.engine.url import make_url
+import requests
+import yaml
 
-from vantage6.common import info, warning
-from vantage6.common.docker.addons import check_docker_running, pull_image
+from vantage6.common import error, info
 from vantage6.common.globals import (
-    APPNAME,
-    DEFAULT_DOCKER_REGISTRY,
-    DEFAULT_SERVER_IMAGE,
     InstanceType,
 )
 
+from vantage6.client import UserClient
+
+from vantage6.cli import __version__
 from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.common.utils import print_log_worker
 from vantage6.cli.context.server import ServerContext
-from vantage6.cli.utils import check_config_name_allowed
 
 
-# TODO this method has a lot of duplicated code from `start`
 @click.command()
 @click.argument("file", type=click.Path(exists=True))
 @click.option(
@@ -29,121 +22,146 @@ from vantage6.cli.utils import check_config_name_allowed
     default=False,
     help="Drop all existing data before importing",
 )
-@click.option("-i", "--image", default=None, help="Node Docker image to use")
-@click.option(
-    "--mount-src",
-    default="",
-    help="Override vantage6 source code in container with the source code in this path",
-)
-@click.option(
-    "--keep/--auto-remove",
-    default=False,
-    help="Keep image after finishing. Useful for debugging",
-)
-@click.option("--wait", default=False, help="Wait for the import to finish")
 @click_insert_context(type_=InstanceType.SERVER)
-def cli_server_import(
-    ctx: ServerContext,
-    file: str,
-    drop_all: bool,
-    image: str | None,
-    mount_src: str,
-    keep: bool,
-    wait: bool,
-) -> None:
+def cli_server_import(ctx: ServerContext, file: str, drop_all: bool) -> None:
     """
-    Import vantage6 resources into a server instance.
+    Import vantage6 resources into a server instance from a yaml FILE.
 
-    This allows you to create organizations, collaborations, users, tasks, etc
-    from a yaml file.
+    This allows you to create organizations, collaborations, users, tasks, etc. from a
+    yaml FILE. This method expects the server configuration file to be located on the
+    same machine as this method is invoked from.
 
-    The FILE_ argument should be a path to a yaml file containing the vantage6
-    formatted data to import.
+    This import assigns the root role to all users, which contains all permissions. So
+    use this with caution.
+
+    The FILE argument should be a path to a yaml file containing the vantage6 formatted
+    data to import.
     """
-    # will print an error if not
-    check_docker_running()
+    info("Validating server version: ")
+    info(f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}/version")
+    response = requests.get(
+        f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}/version"
+    )
+    if response.status_code != 200:
+        error("Unable to get server version")
+        return
+
+    body = response.json()
+    if "version" not in body:
+        error("Server gave a valid response but did not include the version")
+        return
+
+    # Compare it to this package version
+    server_version = body["version"]
+    if server_version != __version__:
+        error(
+            f"You are using CLI version {__version__} but the server is running "
+            f"version {server_version}. Please use the same version of the CLI and "
+            "server."
+        )
+        return
+
+    info("Loading and validating import file")
+    with open(file, "r") as f:
+        import_data = yaml.safe_load(f)
+    # TODO: validate import file
+
+    client = UserClient(
+        server_url=f"{ctx.config['server']['baseUrl']}{ctx.config['server']['apiPath']}",
+        auth_url=ctx.config["ui"]["keycloakPublicUrl"],
+        auth_realm=ctx.config["ui"]["keycloakRealm"],
+        auth_client=ctx.config["ui"]["keycloakClient"],
+        log_level="info",
+    )
+
+    info("Authenticate using admin credentials (opens browser for login)")
+    client.authenticate()
 
-    info("Starting server...")
-    info("Finding Docker daemon.")
-    docker_client = docker.from_env()
+    # TODO: validate that the user has the correct permissions to import data
 
-    # check if name is allowed for docker volume, else exit
-    check_config_name_allowed(ctx.name)
+    if drop_all:
+        info("Dropping all existing data")
+        _drop_all(client)
 
-    # pull latest Docker image
-    if image is None:
-        image = ctx.config.get(
-            "image", f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_SERVER_IMAGE}"
+    info("Collecting root role and rule")
+    root_role_id = client.role.list(name="Root", include_root=True)["data"][0]["id"]
+
+    info("Importing organizations")
+    organizations = []
+    for organization in import_data["organizations"]:
+        org = client.organization.create(
+            name=organization["name"],
+            address1=organization["address1"] or "",
+            address2=organization["address2"] or "",
+            zipcode=organization["zipcode"] or "",
+            country=organization["country"] or "",
+            domain=organization["domain"] or "",
         )
-    info(f"Pulling latest server image '{image}'.")
-    pull_image(docker_client, image)
-
-    info("Creating mounts")
-    mounts = [
-        docker.types.Mount("/mnt/config.yaml", str(ctx.config_file), type="bind"),
-        docker.types.Mount("/mnt/import.yaml", str(file), type="bind"),
-    ]
-
-    # FIXME: code duplication with cli_server_start()
-    # try to mount database
-    uri = ctx.config["uri"]
-    url = make_url(uri)
-    environment_vars = None
-
-    if mount_src:
-        mount_src = os.path.abspath(mount_src)
-        mounts.append(docker.types.Mount("/vantage6", mount_src, type="bind"))
-
-    # If host is None, we're dealing with a file-based DB, like SQLite
-    if url.host is None:
-        db_path = url.database
-
-        if not os.path.isabs(db_path):
-            # We're dealing with a relative path here -> make it absolute
-            db_path = ctx.data_dir / url.database
-
-        basename = os.path.basename(db_path)
-        dirname = os.path.dirname(db_path)
-        os.makedirs(dirname, exist_ok=True)
-
-        # we're mounting the entire folder that contains the database
-        mounts.append(docker.types.Mount("/mnt/database/", dirname, type="bind"))
-
-        environment_vars = {
-            # ServerGlobals.DB_URI_ENV_VAR.value: f"sqlite:////mnt/database/{basename}"
-        }
-
-    else:
-        warning(
-            f"Database could not be transferred, make sure {url.host} "
-            "is reachable from the Docker container"
+        organizations.append(org)
+
+        info(f"Importing users for organization {org['name']}")
+        for user in organization.get("users", []):
+            client.user.create(
+                username=user["username"],
+                password=user["password"],
+                organization=org["id"],
+                roles=[root_role_id],
+            )
+
+    info("Importing collaborations")
+    all_nodes = []
+    for collaboration in import_data["collaborations"]:
+        # Collecting organization ids
+        organization_ids = []
+        for participant in collaboration["participants"]:
+            for org in organizations:
+                if org["name"] == participant["name"]:
+                    organization_ids.append(org["id"])
+
+        col = client.collaboration.create(
+            name=collaboration["name"],
+            organizations=organization_ids,
+            encrypted=collaboration.get("encrypted", False),
         )
-        info("Consider using the docker-compose method to start a server")
-
-    drop_all_ = "--drop-all" if drop_all else ""
-    cmd = f"vserver-local import -c /mnt/config.yaml {drop_all_} /mnt/import.yaml"
-
-    info(cmd)
-
-    info("Run Docker container")
-    container = docker_client.containers.run(
-        image,
-        command=cmd,
-        mounts=mounts,
-        detach=True,
-        labels={
-            f"{APPNAME}-type": InstanceType.SERVER.value,
-            "name": ctx.config_file_name,
-        },
-        environment=environment_vars,
-        auto_remove=not keep,
-        tty=True,
-    )
-    logs = container.logs(stream=True, stdout=True)
-    Thread(target=print_log_worker, args=(logs,), daemon=False).start()
 
-    info(f"Success! container id = {container.id}")
+        info("Registering nodes for collaboration")
+        for participant in collaboration["participants"]:
+            for org in organizations:
+                if org["name"] == participant["name"]:
+                    node = client.node.create(
+                        name=f"{collaboration['name']}-{org['name'].replace(' ', '-')}-node",
+                        organization=org["id"],
+                        collaboration=col["id"],
+                    )
+                    all_nodes.append(node)
 
-    if wait:
-        container.wait()
-        info("Container finished!")
+    return all_nodes
+
+
+def _drop_all(client: UserClient) -> None:
+    """
+    Drop all existing data from the server.
+    """
+    while nodes := client.node.list()["data"]:
+        for node in nodes:
+            info(f"Deleting node {node['name']}")
+            client.node.delete(node["id"])
+
+    while collaborations := client.collaboration.list(scope="global")["data"]:
+        for collaboration in collaborations:
+            info(f"Deleting collaboration {collaboration['name']}")
+            client.collaboration.delete(collaboration["id"], delete_dependents=True)
+
+    # TODO: For some reason, the `delete_dependents` parameter is not working for users,
+    # so we delete them here first.
+    while users := [u for u in client.user.list()["data"] if u["username"] != "admin"]:
+        for user in users:
+            info(f"Deleting user {user['username']}")
+            client.user.delete(user["id"])
+
+    while orgs := [
+        o for o in client.organization.list()["data"] if o["name"] != "root"
+    ]:
+        for org in orgs:
+            info(f"Deleting organization {org['name']}")
+            client.organization.delete(org["id"], delete_dependents=True)

vantage6/cli/server/new.py

@@ -9,7 +9,7 @@ from vantage6.common.globals import (
 
 from vantage6.cli.common.new import new
 from vantage6.cli.config import CliConfig
-from vantage6.cli.configuration_wizard import add_common_server_config
+from vantage6.cli.configuration_create import add_common_server_config
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 
 
@@ -45,8 +45,20 @@ def cli_server_new(
     """
     Create a new server configuration.
     """
+    # info(
+    #     "Vantage6 uses keycloak for authentication. Please configure a keycloak "
+    #     "server here unless you have one running externally."
+    # )
+    # configure_keycloak = q.confirm(
+    #     "Do you want to configure a keycloak server?",
+    #     default=True,
+    # ).unsafe_ask()
+    # if configure_keycloak:
+    #     print("awefawef")
+
     new(
-        questionnaire_function=server_configuration_questionaire,
+        config_producing_func=server_configuration_questionaire,
+        config_producing_func_args=(name,),
         name=name,
         system_folders=system_folders,
         namespace=namespace,
@@ -69,10 +81,6 @@ def server_configuration_questionaire(instance_name: str) -> dict[str, Any]:
     dict[str, Any]
         dictionary with Helm values for the server configuration
     """
-    # Get active kube namespace
-    cli_config = CliConfig()
-    kube_namespace = cli_config.get_last_namespace()
-
     # Initialize config with basic structure
     config = {"server": {}, "database": {}, "ui": {}, "rabbitmq": {}}
 
@@ -85,7 +93,8 @@ def server_configuration_questionaire(instance_name: str) -> dict[str, Any]:
         }
         config["server"]["dev"] = {
             "host_uri": "host.docker.internal",
-            "store_in_local_cluster": True,
+            # TODO v5+ adapt to name space and service name etc, check from sandbox
+            "store_address": "http://vantage6-store-store.service.default.svc.cluster.local",
         }
 
     # TODO v5+ these should be removed, latest should usually be used so question is
@@ -102,7 +111,13 @@ def server_configuration_questionaire(instance_name: str) -> dict[str, Any]:
         default="harbor2.vantage6.ai/infrastructure/ui:latest",
     ).unsafe_ask()
 
+    # TODO v5+ we need to add a question to ask which algorithm stores are allowed, to
+    # set the CSP headers in the UI. This is not done now because it becomes easier when
+    # store and keycloak service can also be setup in the `v6 server new` command.
+
     # === Keycloak settings ===
+    cli_config = CliConfig()
+    kube_namespace = cli_config.get_last_namespace()
     keycloak_url = f"http://vantage6-auth-keycloak.{kube_namespace}.svc.cluster.local"
     config["server"]["keycloakUrl"] = keycloak_url
 

vantage6/cli/server/start.py

@@ -12,6 +12,7 @@ from vantage6.cli.common.start import (
 from vantage6.cli.common.utils import (
     attach_logs,
     create_directory_if_not_exists,
+    select_context_and_namespace,
 )
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.globals import ChartName
@@ -32,6 +33,7 @@ from vantage6.cli.globals import ChartName
     default=False,
     help="Print server logs to the console after start",
 )
+@click.option("--local-chart-dir", default=None, help="Local chart directory to use")
 @click_insert_context(
     type_=InstanceType.SERVER, include_name=True, include_system_folders=True
 )
@@ -45,13 +47,18 @@ def cli_server_start(
     port: int,
     ui_port: int,
     attach: bool,
+    local_chart_dir: str | None,
 ) -> None:
     """
     Start the server.
     """
     info("Starting server...")
+    prestart_checks(ctx, InstanceType.SERVER, name, system_folders)
 
-    prestart_checks(ctx, InstanceType.SERVER, name, system_folders, context, namespace)
+    context, namespace = select_context_and_namespace(
+        context=context,
+        namespace=namespace,
+    )
 
     create_directory_if_not_exists(ctx.log_dir)
 
@@ -61,11 +68,13 @@ def cli_server_start(
         values_file=ctx.config_file,
         context=context,
         namespace=namespace,
+        local_chart_dir=local_chart_dir,
     )
 
     # port forward for server
     info("Port forwarding for server")
     start_port_forward(
+        # TODO: make this dynamic
         service_name=f"{ctx.helm_release_name}-vantage6-server-service",
         service_port=ctx.config["server"].get("port", Ports.DEV_SERVER.value),
         port=port or ctx.config["server"].get("port", Ports.DEV_SERVER.value),

vantage6/cli/server/stop.py

@@ -32,6 +32,7 @@ def cli_server_stop(
     namespace: str,
     system_folders: bool,
     all_servers: bool,
+    is_sandbox: bool = False,
 ):
     """
     Stop an running server.
@@ -45,6 +46,7 @@ def cli_server_stop(
         namespace=namespace,
         context=context,
         system_folders=system_folders,
+        is_sandbox=is_sandbox,
     )
 
 

vantage6/cli/template/auth_config.j2

@@ -0,0 +1,253 @@
+# for more options on the deployment of this chart, see:
+# https://artifacthub.io/packages/helm/bitnami/keycloak
+keycloak:
+
+  # for development, use a local PostgreSQL instance.
+  {% if keycloak.production %}
+  postgresql:
+    enabled: false
+
+  # TODO v5+ set these variables from the CLI
+  externalDatabase:
+    host: "sql.example.vantage6.ai"
+    port: 5432
+    username: my-username
+    password: my-secret-password
+    database: my-keycloak-database
+    schema: public
+    existingSecretHostKey: my-secret-contains-the-host
+    existingSecretPortKey: my-secret-contains-the-port
+    existingSecretUsernameKey: my-secret-contains-the-username
+    existingSecretPasswordKey: my-secret-contains-the-password
+    existingSecretDatabaseKey: my-secret-contains-the-database
+
+  # TODO v5+ set secrets in CLI
+  auth:
+    # Set the username and password for the Keycloak admin. This user is created when
+    # the service is initialized.
+    adminUser: {{ keycloak.adminUser | default('admin') }}
+    existingSecret: secret-containing-admin-password
+    passwordSecretKey: key-to-admin-password-in-secret
+    # For a development environment, you can set the admin user and password directly.
+    # adminPassword: admin
+
+  # for production, TLS should be enabled for internal Keycloak communication
+  # TODO v5+ we should test if this works when we have CLI commands (#1923)
+  production: true
+  tls:
+    enabled: true
+    autoGenerated: true
+  {% else %}
+  postgresql:
+    enabled: true
+    auth:
+      postgresPassword: postgres
+      password: keycloak
+      database: keycloak
+
+  # for production, use an external PostgreSQL instance. This requires setting up config
+  # as follows. Be sure to set postgres.enabled to false just above here.
+  # externalDatabase:
+  #   host: "sql.example.vantage6.ai"
+  #   port: 5432
+  #   username: my-username
+  #   password: my-secret-password
+  #   database: my-keycloak-database
+  #   schema: public
+  #   # or alternatively, use secrets for all of the above formatted as:
+  #   existingSecretHostKey: my-secret-contains-the-host
+  #   existingSecretPortKey: my-secret-contains-the-port
+  #   existingSecretUsernameKey: my-secret-contains-the-username
+  #   existingSecretPasswordKey: my-secret-contains-the-password
+  #   existingSecretDatabaseKey: my-secret-contains-the-database
+
+  auth:
+    # for development environment, set a dummy password for the admin user.
+    adminUser: admin
+    adminPassword: admin
+    # for production, the password should be stored in a secret. Below you should then
+    # give the name of the secret and the key to where the password is within the
+    # secret.
+    # existingSecret: secret-containing-admin-password
+    # passwordSecretKey: key-to-admin-password-in-secret
+
+  # if you want to switch to production, you should set the following settings to true.
+  production: false
+  tls:
+    enabled: false
+    autoGenerated: false
+  {% endif %}
+
+  # ensure that the auth pod has enough resources to run. The default values are enough
+  # in most cases, but for a larger environment, you might need to increase the limits.
+  resources:
+    limits:
+      memory: 2Gi
+      cpu: 1000m
+    requests:
+      memory: 1Gi
+      cpu: 500m
+
+  # The following configuration is run via the CLI when the Keycloak service is
+  # initialized. It creates a number of users, roles and secrets that are required for
+  # vantage6 to work properly.
+  keycloakConfigCli:
+    enabled: true
+    configuration:
+      # Keycloak realm configuration. For all options, see
+      # https://www.keycloak.org/docs-api/latest/rest-api/index.html#RealmRepresentation
+      realm:
+        # Keycloak realm name
+        realm: {{ keycloak.realm | default('vantage6') }}
+        enabled: true
+
+        # access token lifespan in seconds
+        accessTokenLifespan: {{ keycloak.accessTokenLifespan | default(300) }}
+
+        # sso session idle timeout in seconds. This is the time before the refresh token
+        # expires. With default settings, this value controls the time before the
+        # refresh token expires. Note that if setting this to >3600, you also need to
+        # set ssoSessionMaxLifespan and/or clientSessionIdleTimeout and/or
+        # clientSessionMaxLifespan to higher values to effectively lengthen the session.
+        ssoSessionIdleTimeout: {{ keycloak.ssoSessionIdleTimeout | default(1800) }}
+
+        # password policy configuration. If you prefer not to have a password policy,
+        # you can set this to an empty string.
+        {% if keycloak.production %}
+        passwordPolicy: "length(8) and upperCase(1) and lowerCase(1) and digits(1) and specialChars(1)"
+        {% else %}
+        passwordPolicy: ""
+        {% endif %}
+
+        # do not allow users to edit their username - this would lead to problems with
+        # syncing the user between keycloak and vantage6 server/store. This setting
+        # should always be set to false.
+        editUsernameAllowed: false
+
+        {% if keycloak.production %}
+        # required actions for users. By setting defaultAction to true for configuring
+        # OTP, the user will be prompted to configure OTP (for two-factor
+        # authentication) on first login.
+        requiredActions:
+          - alias: CONFIGURE_TOTP
+            name: Configure OTP
+            providerId: CONFIGURE_TOTP
+            enabled: true
+            defaultAction: true
+            priority: 10
+        {% else %}
+        # If you want to require users to use two-factor authentication on first login,
+        # enable the settings below.
+        # requiredActions:
+        #  - alias: CONFIGURE_TOTP
+        #    name: Configure OTP
+        #    providerId: CONFIGURE_TOTP
+        #    enabled: true
+        #    defaultAction: true
+        #    priority: 10
+        {% endif %}
+
+        # users to be created in the realm. This initializes the realm with a default
+        # admin user. It also initializes the service account for the backend admin
+        # client to give it the necessary permissions to manage the realm.
+        # TODO v5+ configure secrets where necessary
+        users:
+          # create the vantage6 admin user. The name of this user should also be present
+          # in the vantage6 server and store configuration - it is the user that will be
+          # assigned admin permissions on initial startup.
+          - username: {{ keycloak.adminUser | default('admin') }}
+            enabled: true
+            {% if not keycloak.production %}
+            email: admin@vantage6.org
+            firstName: Admin
+            lastName: Admin
+            {% endif %}
+            credentials:
+              - type: password
+                {% if keycloak.production %}
+                value: {{ keycloak.adminPassword | default('Admin123!') }}
+                {% else %}
+                value: {{ keycloak.adminPassword | default('admin') }}
+                {% endif %}
+            requiredActions:
+              {% if keycloak.production %}
+              # this requires all users to update their password on first login and
+              # configure two-factor authentication
+              - CONFIGURE_TOTP
+              - UPDATE_PASSWORD
+              {% elif keycloak.no_password_update_required%}
+              # enable configure OTP only if you want to use two-factor authentication
+              # - CONFIGURE_TOTP
+              # enable password update if you want every user to have to change their
+              # password on first login
+              # - UPDATE_PASSWORD
+              {% else %}
+              # enable configure OTP only if you want to use two-factor authentication
+              # - CONFIGURE_TOTP
+              # this requires all users to update their password on first login
+              - UPDATE_PASSWORD
+              {% endif %}
+          # create a service account user for the backend admin client. The
+          # serviceAccountClientId should match the value set in the client section
+          # below. The vantage6 server and store will use this user to create new
+          # accounts for users and nodes in keycloak - that is why it gets assigned some
+          # realm-management permissions.
+          - username: service-account-backend-admin-client
+            enabled: true
+            serviceAccountClientId: backend-admin-client
+            clientRoles:
+              realm-management:
+                - view-users
+                - manage-users
+                - view-clients
+                - manage-clients
+                - create-client
+
+        # clients to be created in the realm. This initializes the realm with a default
+        # public client and a backend admin client. The public client is used by users
+        # to authenticate in the browser. Either the UI or the Python client will
+        # redirect to this client.
+        clients:
+          - clientId: public_client
+            publicClient: true
+            # redirect URIs are the URIs that keycloak is allowed to redirect to after
+            # authentication. This should be set to the UI URL, and to the Keyloak
+            # service on port 7681. The latter is needed for authentication from outside
+            # the browser - if e.g. the Python client authenticates, it will open a
+            # browser window to authenticate that redirects to the Keycloak service on
+            # port 7681.
+            {% if keycloak.redirectUris %}
+            redirectUris:
+              {% for uri in keycloak.redirectUris %}
+              - "{{ uri }}/*"
+              {% endfor %}
+            {% else %}
+            redirectUris:
+              # allow logging in via a local UI
+              - "http://localhost:7600/*"
+              # allow logging in via the Python client (which spins up a local server
+              # on port 7681)
+              - "http://localhost:7681/*"
+            {% endif %}
+            # By setting webOrigins to "+", we allow the same origins as redirectUris.
+            webOrigins:
+              - "+"
+            # The public client is only for users, not for nodes. Therefore, map a
+            # constant claim to indicate the that the client is a user.
+            protocolMappers:
+              - name: vantage6_client_type
+                protocol: openid-connect
+                protocolMapper: oidc-hardcoded-claim-mapper
+                consentRequired: false
+                config:
+                  claim.name: vantage6_client_type
+                  claim.value: user
+                  access.token.claim: true
+          # create a client that will allow the backend to manage users and clients in
+          # keycloak.
+          - clientId: backend-admin-client
+            publicClient: false
+            # TODO v5+ configure secrets where necessary
+            secret: myadminclientsecret
+            serviceAccountsEnabled: true
+            standardFlowEnabled: false