vantage6 5.0.0a22__py3-none-any.whl → 5.0.0a29__py3-none-any.whl

vantage6/cli/context/node.py

@@ -2,23 +2,20 @@ from __future__ import annotations
 
 import hashlib
 import os.path
-
 from pathlib import Path
 
 from vantage6.common.context import AppContext
 from vantage6.common.globals import APPNAME, STRING_ENCODING, InstanceType
+
+from vantage6.cli._version import __version__
 from vantage6.cli.configuration_manager import NodeConfigurationManager
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli._version import __version__
 
 
 class NodeContext(AppContext):
     """
     Node context object for the host system.
 
-    See DockerNodeContext for the node instance mounts when running as a
-    dockerized service.
-
     Parameters
     ----------
     instance_name : str
@@ -43,12 +40,7 @@ class NodeContext(AppContext):
         logger_prefix: str = "",
     ):
         super().__init__(
-            InstanceType.NODE,
-            instance_name,
-            system_folders,
-            config_file,
-            print_log_header,
-            logger_prefix,
+            InstanceType.NODE, instance_name, system_folders=system_folders
         )
         if print_log_header:
             self.log.info("vantage6 version '%s'", __version__)
@@ -136,7 +128,7 @@ class NodeContext(AppContext):
         Path
             Path to the data folder
         """
-        return AppContext.type_data_folder(InstanceType.NODE.value, system_folders)
+        return AppContext.type_data_folder(InstanceType.NODE, system_folders)
 
     @property
     def databases(self) -> dict:
@@ -188,48 +180,6 @@ class NodeContext(AppContext):
         """
         return os.environ.get("DATA_VOLUME_NAME", f"{self.docker_container_name}-vol")
 
-    @property
-    def docker_vpn_volume_name(self) -> str:
-        """
-        Docker volume in which the VPN configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "VPN_VOLUME_NAME", f"{self.docker_container_name}-vpn-vol"
-        )
-
-    @property
-    def docker_ssh_volume_name(self) -> str:
-        """
-        Docker volume in which the SSH configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "SSH_TUNNEL_VOLUME_NAME", f"{self.docker_container_name}-ssh-vol"
-        )
-
-    @property
-    def docker_squid_volume_name(self) -> str:
-        """
-        Docker volume in which the SSH configuration is stored.
-
-        Returns
-        -------
-        str
-            Docker volume name
-        """
-        return os.environ.get(
-            "SSH_SQUID_VOLUME_NAME", f"{self.docker_container_name}-squid-vol"
-        )
-
     @property
     def proxy_log_file(self):
         return self.log_file_name(type_="proxy_server")
@@ -250,6 +200,40 @@ class NodeContext(AppContext):
         """
         return self.config["databases"][label]
 
+    def set_folders(
+        self, instance_type: InstanceType, instance_name: str, system_folders: bool
+    ) -> None:
+        """
+        Set the folders for the node.
+
+        Parameters
+        ----------
+        instance_type : InstanceType
+            Instance type
+        instance_name : str
+            Instance name
+        system_folders : bool
+            Whether to use system folders
+        """
+        dirs = self.instance_folders(instance_type, instance_name, system_folders)
+
+        self.log_dir = dirs.get("log")
+        self.data_dir = dirs.get("data")
+        self.config_dir = dirs.get("config")
+
+    @staticmethod
+    def instance_folders(*_args, **_kwargs) -> dict:
+        """Log, data and config folders are always mounted. The node manager
+        should take care of this.
+        """
+        mnt = Path("/mnt")
+
+        return {
+            "log": mnt / "log",
+            "data": mnt / "data",
+            "config": mnt / "config",
+        }
+
     def __create_node_identifier(self) -> str:
         """
         Create a unique identifier for the node.

vantage6/cli/context/server.py

@@ -1,14 +1,18 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from vantage6.common.globals import APPNAME, InstanceType
+
+from vantage6.cli._version import __version__
 from vantage6.cli.configuration_manager import ServerConfigurationManager
+from vantage6.cli.context.base_server import BaseServerContext
 from vantage6.cli.globals import (
     DEFAULT_SERVER_SYSTEM_FOLDERS as S_FOL,
-    ServerType,
+    PROMETHEUS_DIR,
     ServerGlobals,
+    ServerType,
 )
-from vantage6.cli._version import __version__
-from vantage6.cli.context.base_server import BaseServerContext
 
 
 class ServerContext(BaseServerContext):
@@ -43,7 +47,7 @@ class ServerContext(BaseServerContext):
         str
             string representation of the database uri
         """
-        return super().get_database_uri(ServerGlobals.DB_URI_ENV_VAR)
+        return super().get_database_uri(ServerGlobals.DB_URI_ENV_VAR.value)
 
     @property
     def docker_container_name(self) -> str:
@@ -57,6 +61,30 @@ class ServerContext(BaseServerContext):
         """
         return f"{APPNAME}-{self.name}-{self.scope}-{ServerType.V6SERVER}"
 
+    @property
+    def prometheus_container_name(self) -> str:
+        """
+        Get the name of the Prometheus Docker container for this server.
+
+        Returns
+        -------
+        str
+            Prometheus container name, unique to this server instance.
+        """
+        return f"{APPNAME}-{self.name}-{self.scope}-prometheus"
+
+    @property
+    def prometheus_dir(self) -> Path:
+        """
+        Get the Prometheus directory path.
+
+        Returns
+        -------
+        Path
+            Path to the Prometheus directory
+        """
+        return self.data_dir / PROMETHEUS_DIR
+
     @classmethod
     def from_external_config_file(
         cls, path: str, system_folders: bool = S_FOL
@@ -79,7 +107,10 @@ class ServerContext(BaseServerContext):
             Server context object
         """
         return super().from_external_config_file(
-            path, ServerType.V6SERVER, ServerGlobals.CONFIG_NAME_ENV_VAR, system_folders
+            path,
+            ServerType.V6SERVER,
+            ServerGlobals.CONFIG_NAME_ENV_VAR.value,
+            system_folders,
         )
 
     @classmethod

vantage6/cli/dev/create.py

@@ -1,26 +1,28 @@
-from pathlib import Path
 from importlib import resources as impresources
-import yaml
+from pathlib import Path
+
 import click
 import pandas as pd
-
-from jinja2 import Environment, FileSystemLoader
+import yaml
 from colorama import Fore, Style
+from jinja2 import Environment, FileSystemLoader
 
+from vantage6.common import ensure_config_dir_writable, error, generate_apikey, info
 from vantage6.common.globals import APPNAME, InstanceType, Ports
-from vantage6.common import ensure_config_dir_writable, info, error, generate_apikey
 
 import vantage6.cli.dev.data as data_dir
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
-from vantage6.cli.globals import PACKAGE_FOLDER
-from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
+from vantage6.cli.context.server import ServerContext
+from vantage6.cli.globals import PACKAGE_FOLDER, DefaultDatasets
 from vantage6.cli.server.common import get_server_context
 from vantage6.cli.server.import_ import cli_server_import
 from vantage6.cli.utils import prompt_config_name
 
 
-def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
+def create_node_data_files(
+    num_nodes: int, server_name: str, dataset: tuple[str, Path]
+) -> list[tuple[str, Path]]:
     """Create data files for nodes.
 
     Parameters
@@ -29,15 +31,16 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         Number of nodes to create data files for.
     server_name : str
         Name of the server.
-
+    dataset : tuple[str, Path]
+        Tuple containing the name and the path to the dataset.
     Returns
     -------
-    list[Path]
-        List of paths to the created data files.
+    list[tuple[str, Path]]
+        List of the label and paths to the created data files.
     """
     info(f"Creating data files for {num_nodes} nodes.")
     data_files = []
-    full_df = pd.read_csv(impresources.files(data_dir) / "olympic_athletes_2016.csv")
+    full_df = pd.read_csv(dataset[1])
     length_df = len(full_df)
     for i in range(num_nodes):
         node_name = f"{server_name}_node_{i + 1}"
@@ -49,16 +52,20 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         start = i * length_df // num_nodes
         end = (i + 1) * length_df // num_nodes
         data = full_df[start:end]
-        data_file = data_folder / f"df_{node_name}.csv"
+        data_file = data_folder / f"df_{dataset[0]}_{node_name}.csv"
 
         # write data to file
         data.to_csv(data_file, index=False)
-        data_files.append(data_file)
+        data_files.append((dataset[0], data_file))
     return data_files
 
 
 def create_node_config_file(
-    server_url: str, port: int, config: dict, server_name: str, datafile: Path
+    server_url: str,
+    port: int,
+    config: dict,
+    server_name: str,
+    datasets: list[tuple[str, Path]] = (),
 ) -> None:
     """Create a node configuration file (YAML).
 
@@ -77,8 +84,8 @@ def create_node_config_file(
         additional user_defined_config.
     server_name : str
         Configuration name of the dummy server.
-    datafile : Path
-        Path to the data file for the node to use.
+    datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to the datasets
     """
     environment = Environment(
         loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
@@ -102,10 +109,12 @@ def create_node_config_file(
         error(f"Node configuration file already exists: {full_path}")
         exit(1)
 
+    databases = [{dataset[0]: dataset[1]} for dataset in datasets]
+
     node_config = template.render(
         {
             "api_key": config["api_key"],
-            "databases": {"default": datafile},
+            "databases": databases,
             "logging": {"file": f"{node_name}.log"},
             "port": port,
             "server_url": server_url,
@@ -124,8 +133,7 @@ def create_node_config_file(
         f.write(node_config)
 
     info(
-        f"Spawned node for organization {Fore.GREEN}{config['org_id']}"
-        f"{Style.RESET_ALL}"
+        f"Spawned node for organization {Fore.GREEN}{config['org_id']}{Style.RESET_ALL}"
     )
 
 
@@ -156,6 +164,7 @@ def generate_node_configs(
     port: int,
     server_name: str,
     extra_node_config: Path | None,
+    extra_datasets: list[tuple[str, Path]],
 ) -> list[dict]:
     """Generates ``num_nodes`` node configuration files.
 
@@ -171,6 +180,8 @@ def generate_node_configs(
         Configuration name of the dummy server.
     extra_node_config : Path | None
         Path to file with additional node configuration.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -178,8 +189,36 @@ def generate_node_configs(
         List of dictionaries containing node configurations.
     """
     configs = []
+    node_data_files = []
     extra_config = _read_extra_config_file(extra_node_config)
-    node_data_files = create_node_data_files(num_nodes, server_name)
+
+    data_directory = impresources.files(data_dir)
+
+    # Add default datasets to the list of dataset provided
+    for default_dataset in DefaultDatasets:
+        extra_datasets.append(
+            (default_dataset.name.lower(), data_directory / default_dataset.value)
+        )
+
+    # Check for duplicate dataset labels
+    seen_labels = set()
+    duplicates = [
+        label
+        for label in [dataset[0] for dataset in extra_datasets]
+        if (label in seen_labels or seen_labels.add(label))
+    ]
+
+    if len(duplicates) > 0:
+        error(
+            f"Duplicate dataset labels found: {duplicates}. "
+            f"Please make sure all dataset labels are unique."
+        )
+        exit(1)
+
+    # create the data files for the nodes and get the path and label for each dataset
+    for dataset in extra_datasets:
+        node_data_files.append(create_node_data_files(num_nodes, server_name, dataset))
+
     for i in range(num_nodes):
         config = {
             "org_id": i + 1,
@@ -188,7 +227,11 @@ def generate_node_configs(
             "user_defined_config": extra_config,
         }
         create_node_config_file(
-            server_url, port, config, server_name, node_data_files[i]
+            server_url,
+            port,
+            config,
+            server_name,
+            [files[i] for files in node_data_files],
         )
         configs.append(config)
 
@@ -385,7 +428,7 @@ def create_algo_store_config(
     )
     folders = AlgorithmStoreContext.instance_folders(
         instance_type=InstanceType.ALGORITHM_STORE,
-        instance_name="{server_name}_store",
+        instance_name=f"{server_name}_store",
         system_folders=False,
     )
 
@@ -421,6 +464,7 @@ def demo_network(
     ui_image: str,
     ui_port: int,
     algorithm_store_port: int,
+    extra_datasets: list[tuple[str, Path]],
 ) -> tuple[list[dict], Path, Path]:
     """Generates the demo network.
 
@@ -447,6 +491,8 @@ def demo_network(
         Port to run the UI on.
     algorithm_store_port : int
         Port to run the algorithm store on.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -454,7 +500,12 @@ def demo_network(
         Tuple containing node, server import and server configurations.
     """
     node_configs = generate_node_configs(
-        num_nodes, server_url, server_port, server_name, extra_node_config
+        num_nodes,
+        server_url,
+        server_port,
+        server_name,
+        extra_node_config,
+        extra_datasets,
     )
     server_import_config = create_vserver_import_config(node_configs, server_name)
     server_config = create_vserver_config(
@@ -495,21 +546,19 @@ def demo_network(
     "--server-port",
     type=int,
     default=Ports.DEV_SERVER.value,
-    help=f"Port to run the server on. Default is {Ports.DEV_SERVER.value}.",
+    help=f"Port to run the server on. Default is {Ports.DEV_SERVER}.",
 )
 @click.option(
     "--ui-port",
     type=int,
     default=Ports.DEV_UI.value,
-    help=f"Port to run the UI on. Default is {Ports.DEV_UI.value}.",
+    help=f"Port to run the UI on. Default is {Ports.DEV_UI}.",
 )
 @click.option(
     "--algorithm-store-port",
     type=int,
     default=Ports.DEV_ALGO_STORE.value,
-    help=(
-        f"Port to run the algorithm store on. Default is {Ports.DEV_ALGO_STORE.value}."
-    ),
+    help=(f"Port to run the algorithm store on. Default is {Ports.DEV_ALGO_STORE}."),
 )
 @click.option(
     "-i",
@@ -548,6 +597,14 @@ def demo_network(
     help="YAML File with additional algorithm store configuration. This will be"
     " appended to the algorithm store configuration file",
 )
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=(),
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file.",
+)
 @click.pass_context
 def create_demo_network(
     click_ctx: click.Context,
@@ -562,6 +619,7 @@ def create_demo_network(
     extra_server_config: Path = None,
     extra_node_config: Path = None,
     extra_store_config: Path = None,
+    add_dataset: list[tuple[str, Path]] = (),
 ) -> dict:
     """Creates a demo network.
 
@@ -583,6 +641,7 @@ def create_demo_network(
             ui_image,
             ui_port,
             algorithm_store_port,
+            list(add_dataset),
         )
         info(
             f"Created {Fore.GREEN}{len(demo[0])}{Style.RESET_ALL} node "