vantage6 4.10.2__tar.gz → 4.11.0__tar.gz

{vantage6-4.10.2 → vantage6-4.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vantage6
-Version: 4.10.2
+Version: 4.11.0
 Summary: vantage6 command line interface
 Home-page: https://github.com/vantage6/vantage6
 Requires-Python: >=3.10

{vantage6-4.10.2 → vantage6-4.11.0}/setup.py

@@ -63,6 +63,8 @@ setup(
             "template/server_import_config.j2",
             "template/algo_store_config.j2",
             "dev/data/olympic_athletes_2016.csv",
+            "dev/data/km_dataset.csv",
+            "prometheus/prometheus.yml",
         ],
     },
     entry_points={

vantage6-4.11.0/tests_cli/test_client_script.py

@@ -0,0 +1,23 @@
+from click import UsageError
+from vantage6.cli.test.client_script import cli_test_client_script
+
+import click
+import unittest
+
+
+class TestScriptTest(unittest.TestCase):
+    def test_script_incorrect_usage(self):
+        ctx = click.Context(cli_test_client_script)
+
+        with self.assertRaises(UsageError):
+            ctx.invoke(
+                cli_test_client_script,
+                script="path/to/script.py",
+                task_arguments="{'my_arg': 1}",
+            )
+
+        with self.assertRaises(UsageError):
+            ctx.invoke(
+                cli_test_client_script,
+                task_arguments="not_a_json",
+            )

{vantage6-4.10.2 → vantage6-4.11.0}/vantage6/cli/_version.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "__build__")) as fp:
     __build__ = json.load(fp)
 
 # Module version
-version_info = (4, 10, 2, "final", __build__, 0)
+version_info = (4, 11, 0, "final", __build__, 0)
 
 # Module version stage suffix map
 _specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}

{vantage6-4.10.2 → vantage6-4.11.0}/vantage6/cli/cli.py

@@ -28,6 +28,7 @@ from vantage6.cli.dev.start import start_demo_network
 from vantage6.cli.dev.stop import stop_demo_network
 from vantage6.cli.algorithm.create import cli_algorithm_create
 from vantage6.cli.algorithm.update import cli_algorithm_update
+from vantage6.cli.test.client_script import cli_test_client_script
 from vantage6.cli.test.feature_tester import cli_test_features
 from vantage6.cli.test.integration_test import cli_test_integration
 from vantage6.cli.algostore.attach import cli_algo_store_attach
@@ -124,6 +125,7 @@ def cli_test() -> None:
 # Define the commands for the test group
 cli_test.add_command(cli_test_features, name="feature-test")
 cli_test.add_command(cli_test_integration, name="integration-test")
+cli_test.add_command(cli_test_client_script, name="client-script")
 
 
 # Define the algorithm-store group

{vantage6-4.10.2 → vantage6-4.11.0}/vantage6/cli/configuration_manager.py

@@ -52,6 +52,7 @@ class NodeConfiguration(Configuration):
         Optional("node_extra_env"): dict,
         Optional("node_extra_mounts"): [str],
         Optional("node_extra_hosts"): dict,
+        Optional("share_algorithm_logs"): Use(bool),
     }
 
 

{vantage6-4.10.2 → vantage6-4.11.0}/vantage6/cli/context/server.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
+from pathlib import Path
 
 from vantage6.common.globals import APPNAME, InstanceType
 from vantage6.cli.configuration_manager import ServerConfigurationManager
 from vantage6.cli.globals import (
     DEFAULT_SERVER_SYSTEM_FOLDERS as S_FOL,
+    PROMETHEUS_DIR,
     ServerType,
     ServerGlobals,
 )
@@ -57,6 +59,30 @@ class ServerContext(BaseServerContext):
         """
         return f"{APPNAME}-{self.name}-{self.scope}-{ServerType.V6SERVER}"
 
+    @property
+    def prometheus_container_name(self) -> str:
+        """
+        Get the name of the Prometheus Docker container for this server.
+
+        Returns
+        -------
+        str
+            Prometheus container name, unique to this server instance.
+        """
+        return f"{APPNAME}-{self.name}-{self.scope}-prometheus"
+
+    @property
+    def prometheus_dir(self) -> Path:
+        """
+        Get the Prometheus directory path.
+
+        Returns
+        -------
+        Path
+            Path to the Prometheus directory
+        """
+        return self.data_dir / PROMETHEUS_DIR
+
     @classmethod
     def from_external_config_file(
         cls, path: str, system_folders: bool = S_FOL

{vantage6-4.10.2 → vantage6-4.11.0}/vantage6/cli/dev/create.py

@@ -12,7 +12,7 @@ from vantage6.common import ensure_config_dir_writable, info, error, generate_ap
 
 import vantage6.cli.dev.data as data_dir
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
-from vantage6.cli.globals import PACKAGE_FOLDER
+from vantage6.cli.globals import PACKAGE_FOLDER, DefaultDatasets
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.server.common import get_server_context
@@ -20,7 +20,9 @@ from vantage6.cli.server.import_ import cli_server_import
 from vantage6.cli.utils import prompt_config_name
 
 
-def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
+def create_node_data_files(
+    num_nodes: int, server_name: str, dataset: tuple[str, Path]
+) -> list[tuple[str, Path]]:
     """Create data files for nodes.
 
     Parameters
@@ -29,15 +31,16 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         Number of nodes to create data files for.
     server_name : str
         Name of the server.
-
+    dataset : tuple[str, Path]
+        Tuple containing the name and the path to the dataset.
     Returns
     -------
-    list[Path]
-        List of paths to the created data files.
+    list[tuple[str, Path]]
+        List of the label and paths to the created data files.
     """
     info(f"Creating data files for {num_nodes} nodes.")
     data_files = []
-    full_df = pd.read_csv(impresources.files(data_dir) / "olympic_athletes_2016.csv")
+    full_df = pd.read_csv(dataset[1])
     length_df = len(full_df)
     for i in range(num_nodes):
         node_name = f"{server_name}_node_{i + 1}"
@@ -49,16 +52,20 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         start = i * length_df // num_nodes
         end = (i + 1) * length_df // num_nodes
         data = full_df[start:end]
-        data_file = data_folder / f"df_{node_name}.csv"
+        data_file = data_folder / f"df_{dataset[0]}_{node_name}.csv"
 
         # write data to file
         data.to_csv(data_file, index=False)
-        data_files.append(data_file)
+        data_files.append((dataset[0], data_file))
     return data_files
 
 
 def create_node_config_file(
-    server_url: str, port: int, config: dict, server_name: str, datafile: Path
+    server_url: str,
+    port: int,
+    config: dict,
+    server_name: str,
+    datasets: list[tuple[str, Path]] = (),
 ) -> None:
     """Create a node configuration file (YAML).
 
@@ -77,8 +84,8 @@ def create_node_config_file(
         additional user_defined_config.
     server_name : str
         Configuration name of the dummy server.
-    datafile : Path
-        Path to the data file for the node to use.
+    datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to the datasets
     """
     environment = Environment(
         loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
@@ -102,10 +109,12 @@ def create_node_config_file(
         error(f"Node configuration file already exists: {full_path}")
         exit(1)
 
+    databases = [{dataset[0]: dataset[1]} for dataset in datasets]
+
     node_config = template.render(
         {
             "api_key": config["api_key"],
-            "databases": {"default": datafile},
+            "databases": databases,
             "logging": {"file": f"{node_name}.log"},
             "port": port,
             "server_url": server_url,
@@ -124,8 +133,7 @@ def create_node_config_file(
         f.write(node_config)
 
     info(
-        f"Spawned node for organization {Fore.GREEN}{config['org_id']}"
-        f"{Style.RESET_ALL}"
+        f"Spawned node for organization {Fore.GREEN}{config['org_id']}{Style.RESET_ALL}"
     )
 
 
@@ -156,6 +164,7 @@ def generate_node_configs(
     port: int,
     server_name: str,
     extra_node_config: Path | None,
+    extra_datasets: list[tuple[str, Path]],
 ) -> list[dict]:
     """Generates ``num_nodes`` node configuration files.
 
@@ -171,6 +180,8 @@ def generate_node_configs(
         Configuration name of the dummy server.
     extra_node_config : Path | None
         Path to file with additional node configuration.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -178,8 +189,36 @@ def generate_node_configs(
         List of dictionaries containing node configurations.
     """
     configs = []
+    node_data_files = []
     extra_config = _read_extra_config_file(extra_node_config)
-    node_data_files = create_node_data_files(num_nodes, server_name)
+
+    data_directory = impresources.files(data_dir)
+
+    # Add default datasets to the list of dataset provided
+    for default_dataset in DefaultDatasets:
+        extra_datasets.append(
+            (default_dataset.name.lower(), data_directory / default_dataset.value)
+        )
+
+    # Check for duplicate dataset labels
+    seen_labels = set()
+    duplicates = [
+        label
+        for label in [dataset[0] for dataset in extra_datasets]
+        if (label in seen_labels or seen_labels.add(label))
+    ]
+
+    if len(duplicates) > 0:
+        error(
+            f"Duplicate dataset labels found: {duplicates}. "
+            f"Please make sure all dataset labels are unique."
+        )
+        exit(1)
+
+    # create the data files for the nodes and get the path and label for each dataset
+    for dataset in extra_datasets:
+        node_data_files.append(create_node_data_files(num_nodes, server_name, dataset))
+
     for i in range(num_nodes):
         config = {
             "org_id": i + 1,
@@ -188,7 +227,11 @@ def generate_node_configs(
             "user_defined_config": extra_config,
         }
         create_node_config_file(
-            server_url, port, config, server_name, node_data_files[i]
+            server_url,
+            port,
+            config,
+            server_name,
+            [files[i] for files in node_data_files],
         )
         configs.append(config)
 
@@ -422,6 +465,7 @@ def demo_network(
     ui_image: str,
     ui_port: int,
     algorithm_store_port: int,
+    extra_datasets: list[tuple[str, Path]],
 ) -> tuple[list[dict], Path, Path]:
     """Generates the demo network.
 
@@ -448,6 +492,8 @@ def demo_network(
         Port to run the UI on.
     algorithm_store_port : int
         Port to run the algorithm store on.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -455,7 +501,12 @@ def demo_network(
         Tuple containing node, server import and server configurations.
     """
     node_configs = generate_node_configs(
-        num_nodes, server_url, server_port, server_name, extra_node_config
+        num_nodes,
+        server_url,
+        server_port,
+        server_name,
+        extra_node_config,
+        extra_datasets,
     )
     server_import_config = create_vserver_import_config(node_configs, server_name)
     server_config = create_vserver_config(
@@ -549,6 +600,14 @@ def demo_network(
     help="YAML File with additional algorithm store configuration. This will be"
     " appended to the algorithm store configuration file",
 )
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=(),
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file.",
+)
 @click.pass_context
 def create_demo_network(
     click_ctx: click.Context,
@@ -563,6 +622,7 @@ def create_demo_network(
     extra_server_config: Path = None,
     extra_node_config: Path = None,
     extra_store_config: Path = None,
+    add_dataset: list[tuple[str, Path]] = (),
 ) -> dict:
     """Creates a demo network.
 
@@ -584,6 +644,7 @@ def create_demo_network(
             ui_image,
             ui_port,
             algorithm_store_port,
+            list(add_dataset),
         )
         info(
             f"Created {Fore.GREEN}{len(demo[0])}{Style.RESET_ALL} node "