vantage6 5.0.0a21__py3-none-any.whl → 5.0.0a26__py3-none-any.whl

vantage6/cli/configuration_wizard.py

@@ -1,25 +1,27 @@
 import os
 from pathlib import Path
+from typing import Any
 
 import questionary as q
 
-from vantage6.common import generate_apikey
+from vantage6.common import error, info, warning
+from vantage6.common.client.node_client import NodeClient
+from vantage6.common.context import AppContext
 from vantage6.common.globals import (
     DATABASE_TYPES,
+    DEFAULT_API_PATH,
     InstanceType,
     NodePolicy,
     Ports,
-    DEFAULT_API_PATH,
     RequiredNodeEnvVars,
 )
-from vantage6.common.client.node_client import NodeClient
-from vantage6.common.context import AppContext
-from vantage6.common import error, warning, info
-from vantage6.cli.context import select_context_class
+
+from vantage6.cli.config import CliConfig
 from vantage6.cli.configuration_manager import (
     NodeConfigurationManager,
     ServerConfigurationManager,
 )
+from vantage6.cli.context import select_context_class
 
 
 def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
@@ -368,115 +370,69 @@ def _get_common_server_config(instance_type: InstanceType, instance_name: str) -
     return config
 
 
-def server_configuration_questionaire(instance_name: str) -> dict:
+def server_configuration_questionaire(
+    instance_name: str,
+) -> dict[str, Any]:
     """
-    Questionary to generate a config file for the server instance.
+    Kubernetes-specific questionnaire to generate Helm values for server.
 
     Parameters
     ----------
     instance_name : str
-        Name of the server instance.
+        Name of the server instance
 
     Returns
     -------
-    dict
-        Dictionary with the new server configuration
+    dict[str, Any]
+        dictionary with Helm values for the server configuration
     """
+    # Get active kube namespace
+    cli_config = CliConfig()
+    kube_namespace = cli_config.get_last_namespace()
 
-    config = _get_common_server_config(InstanceType.SERVER, instance_name)
-
-    constant_jwt_secret = q.confirm("Do you want a constant JWT secret?").unsafe_ask()
-    if constant_jwt_secret:
-        config["jwt_secret_key"] = generate_apikey()
+    # Initialize config with basic structure
+    config = {"server": {}, "database": {}, "ui": {}}
 
-    is_mfa = q.confirm("Do you want to enforce two-factor authentication?").unsafe_ask()
-    if is_mfa:
-        config["two_factor_auth"] = is_mfa
+    # === Server settings ===
+    config["server"]["description"] = q.text(
+        "Enter a human-readable description:",
+        default=f"Vantage6 server {instance_name}",
+    ).unsafe_ask()
 
-    current_server_url = f"http://localhost:{config['port']}{config['api_path']}"
-    config["server_url"] = q.text(
-        "What is the server url exposed to the users? If you are running a"
-        " development server running locally, this is the same as the "
-        "server url. If you are running a production server, this is the "
-        "url that users will connect to.",
-        default=current_server_url,
+    config["server"]["image"] = q.text(
+        "Server Docker image:",
+        default="harbor2.vantage6.ai/infrastructure/server:latest",
     ).unsafe_ask()
 
-    is_add_vpn = q.confirm(
-        "Do you want to add a VPN server?", default=False
+    # === UI settings ===
+    config["ui"]["image"] = q.text(
+        "UI Docker image:",
+        default="harbor2.vantage6.ai/infrastructure/ui:latest",
     ).unsafe_ask()
-    if is_add_vpn:
-        vpn_config = q.unsafe_prompt(
-            [
-                {
-                    "type": "text",
-                    "name": "url",
-                    "message": "VPN server URL:",
-                },
-                {
-                    "type": "text",
-                    "name": "portal_username",
-                    "message": "VPN portal username:",
-                },
-                {
-                    "type": "password",
-                    "name": "portal_userpass",
-                    "message": "VPN portal password:",
-                },
-                {
-                    "type": "text",
-                    "name": "client_id",
-                    "message": "VPN client username:",
-                },
-                {
-                    "type": "password",
-                    "name": "client_secret",
-                    "message": "VPN client password:",
-                },
-                {
-                    "type": "text",
-                    "name": "redirect_url",
-                    "message": "Redirect url (should be local address of server)",
-                    "default": "http://localhost",
-                },
-            ]
-        )
-        config["vpn_server"] = vpn_config
 
-    is_add_rabbitmq = q.confirm(
-        "Do you want to add a RabbitMQ message queue?"
+    # === Database settings ===
+    config["database"]["volumePath"] = q.text(
+        "Where is your server database located on the host machine?",
+        default=f"{Path.cwd()}/dev/.db/db_pv_server",
     ).unsafe_ask()
-    if is_add_rabbitmq:
-        rabbit_uri = q.text(message="Enter the URI for your RabbitMQ:").unsafe_ask()
-        run_rabbit_locally = q.confirm(
-            "Do you want to run RabbitMQ locally? (Use only for testing)"
-        ).unsafe_ask()
-        config["rabbitmq"] = {
-            "uri": rabbit_uri,
-            "start_with_server": run_rabbit_locally,
-        }
-
-    # add algorithm stores to this server
-    is_add_community_store = q.confirm(
-        "Do you want to make the algorithms from the community algorithm store "
-        "available to your users?"
+
+    config["database"]["k8sNodeName"] = q.text(
+        "What is the name of the k8s node where the databases are running?",
+        default="docker-desktop",
     ).unsafe_ask()
-    algorithm_stores = []
-    if is_add_community_store:
-        algorithm_stores.append(
-            {"name": "Community store", "url": "https://store.cotopaxi.vantage6.ai"}
-        )
-    add_more_stores = q.confirm(
-        "Do you want to add more algorithm stores?", default=False
+
+    # === Keycloak settings ===
+    keycloak_url = f"http://vantage6-auth-keycloak.{kube_namespace}.svc.cluster.local"
+    config["server"]["keycloakUrl"] = keycloak_url
+
+    # === Other settings ===
+    log_level = q.select(
+        "Which level of logging would you like?",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        default="INFO",
     ).unsafe_ask()
-    while add_more_stores:
-        store_name = q.text(message="Enter the name of the store:").unsafe_ask()
-        store_url = q.text(message="Enter the URL of the store:").unsafe_ask()
-        algorithm_stores.append({"name": store_name, "url": store_url})
-        add_more_stores = q.confirm(
-            "Do you want to add more algorithm stores?", default=False
-        ).unsafe_ask()
-    config["algorithm_stores"] = algorithm_stores
+
+    config["server"]["logging"] = {"level": log_level}
 
     return config
 
@@ -520,7 +476,6 @@ def algo_store_configuration_questionaire(instance_name: str) -> dict:
     }
 
     # ask about openness of the algorithm store
-    config["policies"] = {"allow_localhost": False}
     is_open = q.confirm(
         "Do you want to open the algorithm store to the public? This will allow anyone "
         "to view the algorithms, but they cannot modify them.",
@@ -530,19 +485,21 @@ def algo_store_configuration_questionaire(instance_name: str) -> dict:
         open_algos_policy = "public"
     else:
         is_open_to_whitelist = q.confirm(
-            "Do you want to allow all users of whitelisted vantage6 servers to access "
+            "Do you want to allow all authenticated users to access "
             "the algorithms in the store? If not allowing this, you will have to assign"
             " the appropriate permissions to each user individually.",
             default=True,
         ).unsafe_ask()
-        open_algos_policy = "whitelisted" if is_open_to_whitelist else "private"
+        open_algos_policy = "authenticated" if is_open_to_whitelist else "private"
     config["policies"]["algorithm_view"] = open_algos_policy
 
     return config
 
 
 def configuration_wizard(
-    type_: InstanceType, instance_name: str, system_folders: bool
+    type_: InstanceType,
+    instance_name: str,
+    system_folders: bool,
 ) -> Path:
     """
     Create a configuration file for a node or server instance.
@@ -570,7 +527,9 @@ def configuration_wizard(
         config = node_configuration_questionaire(dirs, instance_name)
     elif type_ == InstanceType.SERVER:
         conf_manager = ServerConfigurationManager
-        config = server_configuration_questionaire(instance_name)
+        config = server_configuration_questionaire(
+            instance_name=instance_name,
+        )
     else:
         conf_manager = ServerConfigurationManager
         config = algo_store_configuration_questionaire(instance_name)

vantage6/cli/context/node.py

@@ -16,9 +16,6 @@ class NodeContext(AppContext):
     """
     Node context object for the host system.
 
-    See DockerNodeContext for the node instance mounts when running as a
-    dockerized service.
-
     Parameters
     ----------
     instance_name : str
@@ -188,48 +185,6 @@ class NodeContext(AppContext):
         """
         return os.environ.get("DATA_VOLUME_NAME", f"{self.docker_container_name}-vol")
 
-    @property
-    def docker_vpn_volume_name(self) -> str:
-        """
-        Docker volume in which the VPN configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "VPN_VOLUME_NAME", f"{self.docker_container_name}-vpn-vol"
-        )
-
-    @property
-    def docker_ssh_volume_name(self) -> str:
-        """
-        Docker volume in which the SSH configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "SSH_TUNNEL_VOLUME_NAME", f"{self.docker_container_name}-ssh-vol"
-        )
-
-    @property
-    def docker_squid_volume_name(self) -> str:
-        """
-        Docker volume in which the SSH configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "SSH_SQUID_VOLUME_NAME", f"{self.docker_container_name}-squid-vol"
-        )
-
     @property
     def proxy_log_file(self):
         return self.log_file_name(type_="proxy_server")
@@ -250,6 +205,40 @@ class NodeContext(AppContext):
         """
         return self.config["databases"][label]
 
+    def set_folders(
+        self, instance_type: InstanceType, instance_name: str, system_folders: bool
+    ) -> None:
+        """
+        Set the folders for the node.
+
+        Parameters
+        ----------
+        instance_type : InstanceType
+            Instance type
+        instance_name : str
+            Instance name
+        system_folders : bool
+            Whether to use system folders
+        """
+        dirs = self.instance_folders(instance_type, instance_name, system_folders)
+
+        self.log_dir = dirs.get("log")
+        self.data_dir = dirs.get("data")
+        self.config_dir = dirs.get("config")
+
+    @staticmethod
+    def instance_folders(*_args, **_kwargs) -> dict:
+        """Log, data and config folders are always mounted. The node manager
+        should take care of this.
+        """
+        mnt = Path("/mnt")
+
+        return {
+            "log": mnt / "log",
+            "data": mnt / "data",
+            "config": mnt / "config",
+        }
+
     def __create_node_identifier(self) -> str:
         """
         Create a unique identifier for the node.

vantage6/cli/context/server.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
+from pathlib import Path
 
 from vantage6.common.globals import APPNAME, InstanceType
 from vantage6.cli.configuration_manager import ServerConfigurationManager
 from vantage6.cli.globals import (
     DEFAULT_SERVER_SYSTEM_FOLDERS as S_FOL,
+    PROMETHEUS_DIR,
     ServerType,
     ServerGlobals,
 )
@@ -57,6 +59,30 @@ class ServerContext(BaseServerContext):
         """
         return f"{APPNAME}-{self.name}-{self.scope}-{ServerType.V6SERVER}"
 
+    @property
+    def prometheus_container_name(self) -> str:
+        """
+        Get the name of the Prometheus Docker container for this server.
+
+        Returns
+        -------
+        str
+            Prometheus container name, unique to this server instance.
+        """
+        return f"{APPNAME}-{self.name}-{self.scope}-prometheus"
+
+    @property
+    def prometheus_dir(self) -> Path:
+        """
+        Get the Prometheus directory path.
+
+        Returns
+        -------
+        Path
+            Path to the Prometheus directory
+        """
+        return self.data_dir / PROMETHEUS_DIR
+
     @classmethod
     def from_external_config_file(
         cls, path: str, system_folders: bool = S_FOL

vantage6/cli/dev/create.py

@@ -12,7 +12,7 @@ from vantage6.common import ensure_config_dir_writable, info, error, generate_ap
 
 import vantage6.cli.dev.data as data_dir
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
-from vantage6.cli.globals import PACKAGE_FOLDER
+from vantage6.cli.globals import PACKAGE_FOLDER, DefaultDatasets
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.server.common import get_server_context
@@ -20,7 +20,9 @@ from vantage6.cli.server.import_ import cli_server_import
 from vantage6.cli.utils import prompt_config_name
 
 
-def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
+def create_node_data_files(
+    num_nodes: int, server_name: str, dataset: tuple[str, Path]
+) -> list[tuple[str, Path]]:
     """Create data files for nodes.
 
     Parameters
@@ -29,15 +31,16 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         Number of nodes to create data files for.
     server_name : str
         Name of the server.
-
+    dataset : tuple[str, Path]
+        Tuple containing the name and the path to the dataset.
     Returns
     -------
-    list[Path]
-        List of paths to the created data files.
+    list[tuple[str, Path]]
+        List of the label and paths to the created data files.
     """
     info(f"Creating data files for {num_nodes} nodes.")
     data_files = []
-    full_df = pd.read_csv(impresources.files(data_dir) / "olympic_athletes_2016.csv")
+    full_df = pd.read_csv(dataset[1])
     length_df = len(full_df)
     for i in range(num_nodes):
         node_name = f"{server_name}_node_{i + 1}"
@@ -49,16 +52,20 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         start = i * length_df // num_nodes
         end = (i + 1) * length_df // num_nodes
         data = full_df[start:end]
-        data_file = data_folder / f"df_{node_name}.csv"
+        data_file = data_folder / f"df_{dataset[0]}_{node_name}.csv"
 
         # write data to file
         data.to_csv(data_file, index=False)
-        data_files.append(data_file)
+        data_files.append((dataset[0], data_file))
     return data_files
 
 
 def create_node_config_file(
-    server_url: str, port: int, config: dict, server_name: str, datafile: Path
+    server_url: str,
+    port: int,
+    config: dict,
+    server_name: str,
+    datasets: list[tuple[str, Path]] = (),
 ) -> None:
     """Create a node configuration file (YAML).
 
@@ -77,8 +84,8 @@ def create_node_config_file(
         additional user_defined_config.
     server_name : str
         Configuration name of the dummy server.
-    datafile : Path
-        Path to the data file for the node to use.
+    datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to the datasets
     """
     environment = Environment(
         loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
@@ -102,10 +109,12 @@ def create_node_config_file(
         error(f"Node configuration file already exists: {full_path}")
         exit(1)
 
+    databases = [{dataset[0]: dataset[1]} for dataset in datasets]
+
     node_config = template.render(
         {
             "api_key": config["api_key"],
-            "databases": {"default": datafile},
+            "databases": databases,
             "logging": {"file": f"{node_name}.log"},
             "port": port,
             "server_url": server_url,
@@ -124,8 +133,7 @@ def create_node_config_file(
         f.write(node_config)
 
     info(
-        f"Spawned node for organization {Fore.GREEN}{config['org_id']}"
-        f"{Style.RESET_ALL}"
+        f"Spawned node for organization {Fore.GREEN}{config['org_id']}{Style.RESET_ALL}"
     )
 
 
@@ -156,6 +164,7 @@ def generate_node_configs(
     port: int,
     server_name: str,
     extra_node_config: Path | None,
+    extra_datasets: list[tuple[str, Path]],
 ) -> list[dict]:
     """Generates ``num_nodes`` node configuration files.
 
@@ -171,6 +180,8 @@ def generate_node_configs(
         Configuration name of the dummy server.
     extra_node_config : Path | None
         Path to file with additional node configuration.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -178,8 +189,36 @@ def generate_node_configs(
         List of dictionaries containing node configurations.
     """
     configs = []
+    node_data_files = []
     extra_config = _read_extra_config_file(extra_node_config)
-    node_data_files = create_node_data_files(num_nodes, server_name)
+
+    data_directory = impresources.files(data_dir)
+
+    # Add default datasets to the list of dataset provided
+    for default_dataset in DefaultDatasets:
+        extra_datasets.append(
+            (default_dataset.name.lower(), data_directory / default_dataset.value)
+        )
+
+    # Check for duplicate dataset labels
+    seen_labels = set()
+    duplicates = [
+        label
+        for label in [dataset[0] for dataset in extra_datasets]
+        if (label in seen_labels or seen_labels.add(label))
+    ]
+
+    if len(duplicates) > 0:
+        error(
+            f"Duplicate dataset labels found: {duplicates}. "
+            f"Please make sure all dataset labels are unique."
+        )
+        exit(1)
+
+    # create the data files for the nodes and get the path and label for each dataset
+    for dataset in extra_datasets:
+        node_data_files.append(create_node_data_files(num_nodes, server_name, dataset))
+
     for i in range(num_nodes):
         config = {
             "org_id": i + 1,
@@ -188,7 +227,11 @@ def generate_node_configs(
             "user_defined_config": extra_config,
         }
         create_node_config_file(
-            server_url, port, config, server_name, node_data_files[i]
+            server_url,
+            port,
+            config,
+            server_name,
+            [files[i] for files in node_data_files],
         )
         configs.append(config)
 
@@ -421,6 +464,7 @@ def demo_network(
     ui_image: str,
     ui_port: int,
     algorithm_store_port: int,
+    extra_datasets: list[tuple[str, Path]],
 ) -> tuple[list[dict], Path, Path]:
     """Generates the demo network.
 
@@ -447,6 +491,8 @@ def demo_network(
         Port to run the UI on.
     algorithm_store_port : int
         Port to run the algorithm store on.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -454,7 +500,12 @@ def demo_network(
         Tuple containing node, server import and server configurations.
     """
     node_configs = generate_node_configs(
-        num_nodes, server_url, server_port, server_name, extra_node_config
+        num_nodes,
+        server_url,
+        server_port,
+        server_name,
+        extra_node_config,
+        extra_datasets,
     )
     server_import_config = create_vserver_import_config(node_configs, server_name)
     server_config = create_vserver_config(
@@ -548,6 +599,14 @@ def demo_network(
     help="YAML File with additional algorithm store configuration. This will be"
     " appended to the algorithm store configuration file",
 )
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=(),
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file.",
+)
 @click.pass_context
 def create_demo_network(
     click_ctx: click.Context,
@@ -562,6 +621,7 @@ def create_demo_network(
     extra_server_config: Path = None,
     extra_node_config: Path = None,
     extra_store_config: Path = None,
+    add_dataset: list[tuple[str, Path]] = (),
 ) -> dict:
     """Creates a demo network.
 
@@ -583,6 +643,7 @@ def create_demo_network(
             ui_image,
             ui_port,
             algorithm_store_port,
+            list(add_dataset),
         )
         info(
             f"Created {Fore.GREEN}{len(demo[0])}{Style.RESET_ALL} node "