vantage6 4.10.1rc1__py3-none-any.whl → 4.11.0rc2__py3-none-any.whl

tests_cli/test_client_script.py

@@ -0,0 +1,23 @@
+from click import UsageError
+from vantage6.cli.test.client_script import cli_test_client_script
+
+import click
+import unittest
+
+
+class TestScriptTest(unittest.TestCase):
+    def test_script_incorrect_usage(self):
+        ctx = click.Context(cli_test_client_script)
+
+        with self.assertRaises(UsageError):
+            ctx.invoke(
+                cli_test_client_script,
+                script="path/to/script.py",
+                task_arguments="{'my_arg': 1}",
+            )
+
+        with self.assertRaises(UsageError):
+            ctx.invoke(
+                cli_test_client_script,
+                task_arguments="not_a_json",
+            )

vantage6/cli/__build__

@@ -1 +1 @@
-1
+2

vantage6/cli/_version.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "__build__")) as fp:
     __build__ = json.load(fp)
 
 # Module version
-version_info = (4, 10, 1, "candidate", __build__, 0)
+version_info = (4, 11, 0, "candidate", __build__, 0)
 
 # Module version stage suffix map
 _specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}

vantage6/cli/cli.py

@@ -28,6 +28,7 @@ from vantage6.cli.dev.start import start_demo_network
 from vantage6.cli.dev.stop import stop_demo_network
 from vantage6.cli.algorithm.create import cli_algorithm_create
 from vantage6.cli.algorithm.update import cli_algorithm_update
+from vantage6.cli.test.client_script import cli_test_client_script
 from vantage6.cli.test.feature_tester import cli_test_features
 from vantage6.cli.test.integration_test import cli_test_integration
 from vantage6.cli.algostore.attach import cli_algo_store_attach
@@ -124,6 +125,7 @@ def cli_test() -> None:
 # Define the commands for the test group
 cli_test.add_command(cli_test_features, name="feature-test")
 cli_test.add_command(cli_test_integration, name="integration-test")
+cli_test.add_command(cli_test_client_script, name="client-script")
 
 
 # Define the algorithm-store group

vantage6/cli/configuration_manager.py

@@ -52,6 +52,7 @@ class NodeConfiguration(Configuration):
         Optional("node_extra_env"): dict,
         Optional("node_extra_mounts"): [str],
         Optional("node_extra_hosts"): dict,
+        Optional("share_algorithm_logs"): Use(bool),
     }
 
 

vantage6/cli/context/server.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
+from pathlib import Path
 
 from vantage6.common.globals import APPNAME, InstanceType
 from vantage6.cli.configuration_manager import ServerConfigurationManager
 from vantage6.cli.globals import (
     DEFAULT_SERVER_SYSTEM_FOLDERS as S_FOL,
+    PROMETHEUS_DIR,
     ServerType,
     ServerGlobals,
 )
@@ -57,6 +59,30 @@ class ServerContext(BaseServerContext):
         """
         return f"{APPNAME}-{self.name}-{self.scope}-{ServerType.V6SERVER}"
 
+    @property
+    def prometheus_container_name(self) -> str:
+        """
+        Get the name of the Prometheus Docker container for this server.
+
+        Returns
+        -------
+        str
+            Prometheus container name, unique to this server instance.
+        """
+        return f"{APPNAME}-prometheus"
+
+    @property
+    def prometheus_dir(self) -> Path:
+        """
+        Get the Prometheus directory path.
+
+        Returns
+        -------
+        Path
+            Path to the Prometheus directory
+        """
+        return self.data_dir / PROMETHEUS_DIR
+
     @classmethod
     def from_external_config_file(
         cls, path: str, system_folders: bool = S_FOL

vantage6/cli/dev/create.py

@@ -12,7 +12,7 @@ from vantage6.common import ensure_config_dir_writable, info, error, generate_ap
 
 import vantage6.cli.dev.data as data_dir
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
-from vantage6.cli.globals import PACKAGE_FOLDER
+from vantage6.cli.globals import PACKAGE_FOLDER, DefaultDatasets
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.server.common import get_server_context
@@ -20,7 +20,9 @@ from vantage6.cli.server.import_ import cli_server_import
 from vantage6.cli.utils import prompt_config_name
 
 
-def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
+def create_node_data_files(
+    num_nodes: int, server_name: str, dataset: tuple[str, Path]
+) -> list[tuple[str, Path]]:
     """Create data files for nodes.
 
     Parameters
@@ -29,15 +31,16 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         Number of nodes to create data files for.
     server_name : str
         Name of the server.
-
+    dataset : tuple[str, Path]
+        Tuple containing the name and the path to the dataset.
     Returns
     -------
-    list[Path]
-        List of paths to the created data files.
+    list[tuple[str, Path]]
+        List of the label and paths to the created data files.
     """
     info(f"Creating data files for {num_nodes} nodes.")
     data_files = []
-    full_df = pd.read_csv(impresources.files(data_dir) / "olympic_athletes_2016.csv")
+    full_df = pd.read_csv(dataset[1])
     length_df = len(full_df)
     for i in range(num_nodes):
         node_name = f"{server_name}_node_{i + 1}"
@@ -49,16 +52,20 @@ def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
         start = i * length_df // num_nodes
         end = (i + 1) * length_df // num_nodes
         data = full_df[start:end]
-        data_file = data_folder / f"df_{node_name}.csv"
+        data_file = data_folder / f"df_{dataset[0]}_{node_name}.csv"
 
         # write data to file
         data.to_csv(data_file, index=False)
-        data_files.append(data_file)
+        data_files.append((dataset[0], data_file))
     return data_files
 
 
 def create_node_config_file(
-    server_url: str, port: int, config: dict, server_name: str, datafile: Path
+    server_url: str,
+    port: int,
+    config: dict,
+    server_name: str,
+    datasets: list[tuple[str, Path]] = (),
 ) -> None:
     """Create a node configuration file (YAML).
 
@@ -77,8 +84,8 @@ def create_node_config_file(
         additional user_defined_config.
     server_name : str
         Configuration name of the dummy server.
-    datafile : Path
-        Path to the data file for the node to use.
+    datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to the datasets
     """
     environment = Environment(
         loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
@@ -102,10 +109,12 @@ def create_node_config_file(
         error(f"Node configuration file already exists: {full_path}")
         exit(1)
 
+    databases = [{dataset[0]: dataset[1]} for dataset in datasets]
+
     node_config = template.render(
         {
             "api_key": config["api_key"],
-            "databases": {"default": datafile},
+            "databases": databases,
             "logging": {"file": f"{node_name}.log"},
             "port": port,
             "server_url": server_url,
@@ -124,8 +133,7 @@ def create_node_config_file(
         f.write(node_config)
 
     info(
-        f"Spawned node for organization {Fore.GREEN}{config['org_id']}"
-        f"{Style.RESET_ALL}"
+        f"Spawned node for organization {Fore.GREEN}{config['org_id']}{Style.RESET_ALL}"
     )
 
 
@@ -156,6 +164,7 @@ def generate_node_configs(
     port: int,
     server_name: str,
     extra_node_config: Path | None,
+    extra_datasets: list[tuple[str, Path]],
 ) -> list[dict]:
     """Generates ``num_nodes`` node configuration files.
 
@@ -171,6 +180,8 @@ def generate_node_configs(
         Configuration name of the dummy server.
     extra_node_config : Path | None
         Path to file with additional node configuration.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -178,8 +189,36 @@ def generate_node_configs(
         List of dictionaries containing node configurations.
     """
     configs = []
+    node_data_files = []
     extra_config = _read_extra_config_file(extra_node_config)
-    node_data_files = create_node_data_files(num_nodes, server_name)
+
+    data_directory = impresources.files(data_dir)
+
+    # Add default datasets to the list of dataset provided
+    for default_dataset in DefaultDatasets:
+        extra_datasets.append(
+            (default_dataset.name.lower(), data_directory / default_dataset.value)
+        )
+
+    # Check for duplicate dataset labels
+    seen_labels = set()
+    duplicates = [
+        label
+        for label in [dataset[0] for dataset in extra_datasets]
+        if (label in seen_labels or seen_labels.add(label))
+    ]
+
+    if len(duplicates) > 0:
+        error(
+            f"Duplicate dataset labels found: {duplicates}. "
+            f"Please make sure all dataset labels are unique."
+        )
+        exit(1)
+
+    # create the data files for the nodes and get the path and label for each dataset
+    for dataset in extra_datasets:
+        node_data_files.append(create_node_data_files(num_nodes, server_name, dataset))
+
     for i in range(num_nodes):
         config = {
             "org_id": i + 1,
@@ -188,7 +227,11 @@ def generate_node_configs(
             "user_defined_config": extra_config,
         }
         create_node_config_file(
-            server_url, port, config, server_name, node_data_files[i]
+            server_url,
+            port,
+            config,
+            server_name,
+            [files[i] for files in node_data_files],
         )
         configs.append(config)
 
@@ -422,6 +465,7 @@ def demo_network(
     ui_image: str,
     ui_port: int,
     algorithm_store_port: int,
+    extra_datasets: list[tuple[str, Path]],
 ) -> tuple[list[dict], Path, Path]:
     """Generates the demo network.
 
@@ -448,6 +492,8 @@ def demo_network(
         Port to run the UI on.
     algorithm_store_port : int
         Port to run the algorithm store on.
+    extra_datasets : list[tuple[str, Path]]
+        List of tuples containing the labels and the paths to extra datasets
 
     Returns
     -------
@@ -455,7 +501,12 @@ def demo_network(
         Tuple containing node, server import and server configurations.
     """
     node_configs = generate_node_configs(
-        num_nodes, server_url, server_port, server_name, extra_node_config
+        num_nodes,
+        server_url,
+        server_port,
+        server_name,
+        extra_node_config,
+        extra_datasets,
     )
     server_import_config = create_vserver_import_config(node_configs, server_name)
     server_config = create_vserver_config(
@@ -549,6 +600,14 @@ def demo_network(
     help="YAML File with additional algorithm store configuration. This will be"
     " appended to the algorithm store configuration file",
 )
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=(),
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file.",
+)
 @click.pass_context
 def create_demo_network(
     click_ctx: click.Context,
@@ -563,6 +622,7 @@ def create_demo_network(
     extra_server_config: Path = None,
     extra_node_config: Path = None,
     extra_store_config: Path = None,
+    add_dataset: list[tuple[str, Path]] = (),
 ) -> dict:
     """Creates a demo network.
 
@@ -584,6 +644,7 @@ def create_demo_network(
             ui_image,
             ui_port,
             algorithm_store_port,
+            list(add_dataset),
         )
         info(
             f"Created {Fore.GREEN}{len(demo[0])}{Style.RESET_ALL} node "

vantage6/cli/globals.py

@@ -42,6 +42,18 @@ DIAGNOSTICS_IMAGE = "harbor2.vantage6.ai/algorithms/diagnostic"
 # Address of community algorithm store
 COMMUNITY_STORE = "https://store.cotopaxi.vantage6.ai/api"
 
+DEFAULT_PROMETHEUS_IMAGE = "prom/prometheus"
+PROMETHEUS_CONFIG = "prometheus.yml"
+PROMETHEUS_DIR = "prometheus"
+
+
+# datasets included in the nodes of the dev network
+class DefaultDatasets(str, Enum):
+    """Enum containing default datasets"""
+
+    OLYMPIC_ATHLETES = "olympic_athletes_2016.csv"
+    KAPLAN_MEIER_TEST = "km_dataset.csv"
+
 
 class ServerType(str, Enum):
     """Enum containing server types"""

vantage6/cli/prometheus/monitoring_manager.py

@@ -0,0 +1,145 @@
+import yaml
+import docker
+from pathlib import Path
+
+from vantage6.common import info, error
+from vantage6.common.docker.network_manager import NetworkManager
+from vantage6.common.globals import DEFAULT_PROMETHEUS_EXPORTER_PORT
+from vantage6.cli.context.server import ServerContext
+from vantage6.cli.globals import (
+    DEFAULT_PROMETHEUS_IMAGE,
+    PROMETHEUS_CONFIG,
+)
+
+
+class PrometheusServer:
+    """
+    Manages the Prometheus Docker container
+    """
+
+    def __init__(
+        self, ctx: ServerContext, network_mgr: NetworkManager, image: str = None
+    ):
+        """
+        Initialize the PrometheusServer instance.
+
+        Parameters
+        ----------
+        ctx : ServerContext
+            The server context containing configuration and paths.
+        network_mgr : NetworkManager
+            The network manager responsible for managing Docker networks.
+        image : str, optional
+            The Docker image to use for the Prometheus container. If not provided,
+            the default Prometheus image will be used.
+        """
+        self.ctx = ctx
+        self.network_mgr = network_mgr
+        self.docker = docker.from_env()
+        self.image = image if image else DEFAULT_PROMETHEUS_IMAGE
+        self.config_file = Path(self.ctx.data_dir / PROMETHEUS_CONFIG)
+        self.data_dir = self.ctx.prometheus_dir
+
+    def start(self):
+        """
+        Start a Docker container running Prometheus
+        """
+        self._prepare_config()
+
+        volumes = {
+            str(self.config_file): {
+                "bind": "/etc/prometheus/prometheus.yml",
+                "mode": "ro",
+            },
+            str(self.data_dir): {"bind": "/prometheus", "mode": "rw"},
+        }
+        ports = {"9090/tcp": 9090}
+
+        if self._is_container_running():
+            info("Prometheus is already running!")
+            return
+
+        self.docker.containers.run(
+            name=self.ctx.prometheus_container_name,
+            image=self.image,
+            volumes=volumes,
+            ports=ports,
+            detach=True,
+            restart_policy={"Name": "unless-stopped"},
+            network=self.network_mgr.network_name,
+        )
+        info("Prometheus container started successfully!")
+
+    def _prepare_config(self):
+        """
+        Prepare the Prometheus configuration and data directories
+        """
+        if not self.config_file.exists():
+            error(f"Prometheus configuration file {self.config_file} not found!")
+            raise FileNotFoundError(f"{self.config_file} not found!")
+
+        if not self.data_dir.exists():
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+
+        self._update_prometheus_config()
+
+    def _update_prometheus_config(self):
+        """
+        Update the Prometheus configuration file with the server address.
+        """
+
+        try:
+            prometheus_exporter_port = self.ctx.config.get("prometheus", {}).get(
+                "exporter_port", DEFAULT_PROMETHEUS_EXPORTER_PORT
+            )
+            server_hostname = self.ctx.prometheus_container_name
+            server_address = f"{server_hostname}:{prometheus_exporter_port}"
+
+            info(
+                f"Using Docker container hostname '{server_hostname}' for Prometheus target. "
+                "Ensure Prometheus is in the same Docker network to resolve this address."
+            )
+
+            with open(self.config_file, "r") as f:
+                config = yaml.safe_load(f)
+
+            job_name = "vantage6_server_metrics"
+            job_exists = any(
+                job.get("job_name") == job_name
+                for job in config.get("scrape_configs", [])
+            )
+
+            if not job_exists:
+                new_job = {
+                    "job_name": job_name,
+                    "static_configs": [{"targets": [server_address]}],
+                }
+                config.setdefault("scrape_configs", []).append(new_job)
+            else:
+                for job in config["scrape_configs"]:
+                    if job.get("job_name") == job_name:
+                        job["static_configs"] = [{"targets": [server_address]}]
+
+            with open(self.config_file, "w") as f:
+                yaml.dump(config, f)
+
+            info(f"Prometheus configuration updated with target: {server_address}")
+
+        except Exception as e:
+            error(f"Failed to update Prometheus configuration: {e}")
+            raise
+
+    def _is_container_running(self) -> bool:
+        """
+        Check if a Prometheus container is already running.
+
+        Returns
+        -------
+        bool
+            True if the Prometheus container is running, False otherwise.
+        """
+        try:
+            container = self.docker.containers.get(self.ctx.prometheus_container_name)
+            return container.status == "running"
+        except docker.errors.NotFound:
+            return False

vantage6/cli/server/start.py

@@ -11,11 +11,12 @@ from vantage6.common.globals import (
     InstanceType,
 )
 
-from vantage6.common.globals import Ports
+from vantage6.common.globals import Ports, DEFAULT_PROMETHEUS_EXPORTER_PORT
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.rabbitmq.queue_manager import RabbitMQManager
 from vantage6.cli.server.common import stop_ui
 from vantage6.cli.common.decorator import click_insert_context
+from vantage6.cli.prometheus.monitoring_manager import PrometheusServer
 from vantage6.cli.common.start import (
     attach_logs,
     check_for_start,
@@ -49,6 +50,14 @@ from vantage6.cli.common.start import (
     "container - use in development only",
 )
 @click.option("--rabbitmq-image", default=None, help="RabbitMQ docker image to use")
+@click.option(
+    "--with-prometheus",
+    "start_prometheus",
+    flag_value=True,
+    default=False,
+    help="Start Prometheus monitoring as a local container",
+)
+@click.option("--prometheus-image", default=None, help="Prometheus docker image to use")
 @click.option(
     "--keep/--auto-remove",
     default=False,
@@ -75,6 +84,8 @@ def cli_server_start(
     ui_port: int,
     start_rabbitmq: bool,
     rabbitmq_image: str,
+    start_prometheus: bool,
+    prometheus_image: str,
     keep: bool,
     mount_src: str,
     attach: bool,
@@ -135,6 +146,24 @@ def cli_server_start(
             "cannot be scaled horizontally!"
         )
 
+    if (
+        start_prometheus
+        or ctx.config.get("prometheus")
+        and ctx.config["prometheus"].get("start_with_server", False)
+    ):
+        info("Starting Prometheus container")
+        _start_prometheus(ctx, prometheus_image, server_network_mgr)
+    elif ctx.config.get("prometheus"):
+        info(
+            "Prometheus is provided in the config file as external service."
+            "Assuming this service is up and running."
+        )
+    else:
+        warning(
+            "Monitoring is not set up! This means that the vantage6 server "
+            "cannot be monitored with Prometheus!"
+        )
+
     # start the UI if requested
     if start_ui or ctx.config.get("ui") and ctx.config["ui"].get("enabled"):
         _start_ui(docker_client, ctx, ui_port)
@@ -152,6 +181,9 @@ def cli_server_start(
 
     info("Run Docker container")
     port_ = str(port or ctx.config["port"] or Ports.DEV_SERVER.value)
+    prometheus_exporter_port = ctx.config.get("prometheus", {}).get(
+        "exporter_port", DEFAULT_PROMETHEUS_EXPORTER_PORT
+    )
     container = docker_client.containers.run(
         image,
         command=cmd,
@@ -162,7 +194,10 @@ def cli_server_start(
             "name": ctx.config_file_name,
         },
         environment=environment_vars,
-        ports={f"{internal_port}/tcp": (ip, port_)},
+        ports={
+            f"{internal_port}/tcp": (ip, port_),  # API port
+            f"{prometheus_exporter_port}/tcp": prometheus_exporter_port,
+        },
         name=ctx.docker_container_name,
         auto_remove=not keep,
         tty=True,
@@ -202,6 +237,27 @@ def _start_rabbitmq(
     rabbit_mgr.start()
 
 
+def _start_prometheus(
+    ctx: ServerContext, prometheus_image: str, network_mgr: NetworkManager
+) -> None:
+    """
+    Start the Prometheus container if it is not already running.
+
+    Parameters
+    ----------
+    ctx : ServerContext
+        Server context object
+    prometheus_image : str
+        Prometheus image to use
+    network_mgr : NetworkManager
+        Network manager object
+    """
+    prometheus_server = PrometheusServer(
+        ctx=ctx, network_mgr=network_mgr, image=prometheus_image
+    )
+    prometheus_server.start()
+
+
 def _start_ui(client: DockerClient, ctx: ServerContext, ui_port: int) -> None:
     """
     Start the UI container.

vantage6/cli/server/stop.py

@@ -117,3 +117,10 @@ def _stop_server_containers(
                 f"Stopped the {Fore.GREEN}{rabbit_container_name}"
                 f"{Style.RESET_ALL} container."
             )
+
+    if ctx.config.get("prometheus", {}).get("enabled"):
+        remove_container_if_exists(client, name=ctx.prometheus_container_name)
+        info(
+            f"Stopped the {Fore.GREEN}{ctx.prometheus_container_name}"
+            f"{Style.RESET_ALL} container."
+        )

vantage6/cli/template/node_config.j2

@@ -1,11 +1,13 @@
 api_key: {{ api_key }}
 api_path: /api
 databases:
-  {% for label, path in databases.items() %}
+  {% for db in databases %}
+  {% for label, path in db.items() %}
   - label: {{ label }}
     uri: {{ path }}
     type: csv
   {% endfor %}
+  {% endfor %}
 encryption:
   enabled: false
   private_key: null
@@ -31,4 +33,4 @@ logging:
 port: {{ port }}
 server_url: {{ server_url }}
 task_dir: {{ task_dir}}
-{{ user_provided_config }}
+{{- user_provided_config -}}

vantage6/cli/test/algo_test_scripts/algo_test_arguments.py

@@ -0,0 +1,33 @@
+average = {
+    "collaboration": 1,
+    "organizations": [1],
+    "name": "test_average_task",
+    "image": "harbor2.vantage6.ai/demo/average",
+    "description": "",
+    "input_": {
+        "method": "central_average",
+        "args": [],
+        "kwargs": {"column_name": "Age"},
+    },
+    "databases": [{"label": "olympic_athletes"}],
+}
+
+kaplan_meier = {
+    "collaboration": 1,
+    "organizations": [1],
+    "name": "test_average_task",
+    "image": "harbor2.vantage6.ai/algorithms/kaplan-meier",
+    "description": "",
+    "input_": {
+        "method": "kaplan_meier_central",
+        "args": [],
+        "kwargs": {
+            "time_column_name": "days",
+            "censor_column_name": "censor",
+            "organizations_to_include": [1, 2, 3],
+        },
+    },
+    "databases": [{"label": "kaplan_meier_test"}],
+}
+
+args = {"average": average, "kaplan_meier": kaplan_meier}

vantage6/cli/test/algo_test_scripts/algo_test_script.py

@@ -0,0 +1,89 @@
+import logging
+import algo_test_arguments as arguments
+import json
+import sys
+
+import vantage6.common.task_status as task_status
+
+from vantage6.client import Client
+from vantage6.common import error
+from vantage6.common.globals import Ports
+
+
+def create_and_run_task(client: Client, task_args: dict, algo_name: str = "algorithm"):
+    """
+    Create and run a task using the provided client and task arguments.
+
+    Parameters
+    ----------
+        client: Client
+            The client instance to use for creating and running the task.
+        task_args: dict
+            The arguments to pass to the task creation method.
+        algo_name: str, optional
+            The name of the algorithm for logging purposes. Default is "algorithm".
+
+    Raises
+    ------
+        AssertionError: If the task fails.
+    """
+    task = client.task.create(**task_args)
+    task_id = task["id"]
+    client.wait_for_results(task_id)
+
+    try:
+        # check if the task has failed
+        assert not task_status.has_task_failed(client.task.get(task_id)["status"])
+
+        logging.info(f"Task for {algo_name} completed successfully.")
+
+    except AssertionError:
+        error(
+            f"Task for {algo_name} failed. Check the log file of the task "
+            f"{task_id} for more information."
+        )
+        exit(1)
+
+
+def run_test(custom_args: dict | None = None):
+    """
+    Run a test by creating and running tasks using the provided arguments.
+
+    Parameters
+    ----------
+    custom_args: dict, optional
+        The arguments to pass to the task creation method. If not provided,
+        the arguments from the `arguments` module will be used.
+    """
+    # Create a client and authenticate
+    client = Client("http://localhost", Ports.DEV_SERVER.value, "/api")
+    try:
+        client.authenticate("dev_admin", "password")
+    except ConnectionError:
+        error(
+            "Could not connect to the server. Please check if a dev network is running."
+        )
+        exit(1)
+
+    # if custom arguments are provided, use them for running the task
+    if custom_args:
+        create_and_run_task(client, custom_args)
+
+    else:
+        # Run the task for each algorithm in the arguments file
+        for algo in arguments.args:
+            logging.info(f"Running task for {algo}")
+
+            task_args = arguments.args[algo]
+            create_and_run_task(client, task_args, algo)
+
+
+if __name__ == "__main__":
+    # check if arguments are provided
+    if len(sys.argv) > 1:
+        input_string = sys.argv[1].replace("'", '"')
+        json_input = json.loads(input_string)
+    else:
+        json_input = None
+
+    run_test(json_input)

vantage6/cli/test/client_script.py

@@ -0,0 +1,143 @@
+import click
+import json
+import subprocess
+import sys
+
+from pathlib import Path
+
+from rich.console import Console
+
+from vantage6.cli.dev.create import create_demo_network
+from vantage6.cli.dev.remove import remove_demo_network
+from vantage6.cli.dev.start import start_demo_network
+from vantage6.cli.dev.stop import stop_demo_network
+from vantage6.common.globals import Ports
+
+TEST_FILE_PATH = Path(__file__).parent / "algo_test_scripts" / "algo_test_script.py"
+
+
+@click.command()
+@click.option(
+    "--script",
+    type=click.Path(),
+    default=TEST_FILE_PATH,
+    help="Path of the script to test the algorithm. If a script is not provided, the default script is used.",
+)
+@click.option(
+    "--task-arguments",
+    type=str,
+    default=None,
+    help="Arguments to be provided to Task.create function. If --script is provided, this should not be set.",
+)
+@click.option(
+    "--create-dev-network",
+    is_flag=True,
+    help="Create a new dev network to run the test",
+)
+@click.option(
+    "--start-dev-network",
+    is_flag=True,
+    help="Start a dev network to run the test",
+)
+@click.option(
+    "-n", "--name", default=None, type=str, help="Name for your development setup"
+)
+@click.option(
+    "--server-url",
+    type=str,
+    default="http://host.docker.internal",
+    help="Server URL to point to. If you are using Docker Desktop, "
+    "the default http://host.docker.internal should not be changed.",
+)
+@click.option(
+    "-i", "--image", type=str, default=None, help="Server Docker image to use"
+)
+@click.option(
+    "--keep",
+    type=bool,
+    default=False,
+    help="Keep the dev network after finishing the test",
+)
+@click.option(
+    "--add-dataset",
+    type=(str, click.Path()),
+    default=[],
+    multiple=True,
+    help="Add a dataset to the nodes. The first argument is the label of the database, "
+    "the second is the path to the dataset file.",
+)
+@click.pass_context
+def cli_test_client_script(
+    click_ctx: click.Context,
+    script: Path,
+    task_arguments: str,
+    name: str,
+    server_url: str,
+    create_dev_network: bool,
+    start_dev_network: bool,
+    image: str,
+    keep: bool,
+    add_dataset: list[tuple[str, Path]] = (),
+) -> int:
+    """
+    Run a script for testing an algorithm on a dev network.
+    The path to the script must be provided as an argument.
+    """
+    if not (script or task_arguments):
+        raise click.UsageError("--script or --task-arguments must be set.")
+    elif script != TEST_FILE_PATH and task_arguments:
+        raise click.UsageError("--script and --task-arguments cannot be set together.")
+
+    # Check if the task_arguments is a valid JSON string
+    if task_arguments:
+        try:
+            json.loads(task_arguments.replace("'", '"'))
+        except json.JSONDecodeError:
+            raise click.UsageError("task-arguments must be a valid JSON string.")
+
+    # create the network
+    if create_dev_network:
+        click_ctx.invoke(
+            create_demo_network,
+            name=name,
+            num_nodes=3,
+            server_url=server_url,
+            server_port=Ports.DEV_SERVER.value,
+            image=image,
+            extra_server_config=None,
+            extra_node_config=None,
+            add_dataset=add_dataset,
+        )
+
+    # start the server and nodes
+    if create_dev_network or start_dev_network:
+        click_ctx.invoke(
+            start_demo_network,
+            name=name,
+            server_image=image,
+            node_image=image,
+        )
+
+    # run the test script and get the result
+    if not task_arguments:
+        subprocess_args = ["python", script]
+    else:
+        subprocess_args = ["python", script, task_arguments]
+
+    result = subprocess.run(subprocess_args, stdout=sys.stdout, stderr=sys.stderr)
+
+    # check the exit code. If the test passed, it should be 0
+    if result.returncode == 0:
+        msg = ":heavy_check_mark: [green]Test passed[/green]"
+    else:
+        msg = ":x: [red]Test failed[/red]"
+
+    console = Console()
+    console.print(msg)
+
+    # clean up the test resources
+    if not keep:
+        click_ctx.invoke(stop_demo_network, name=name)
+        click_ctx.invoke(remove_demo_network, name=name)
+
+    return result.returncode

{vantage6-4.10.1rc1.dist-info → vantage6-4.11.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vantage6
-Version: 4.10.1rc1
+Version: 4.11.0rc2
 Summary: vantage6 command line interface
 Home-page: https://github.com/vantage6/vantage6
 Requires-Python: >=3.10
@@ -16,8 +16,8 @@ Requires-Dist: questionary==1.10.0
 Requires-Dist: rich==13.5.2
 Requires-Dist: schema==0.7.5
 Requires-Dist: SQLAlchemy==1.4.46
-Requires-Dist: vantage6-common==4.10.1rc1
-Requires-Dist: vantage6-client==4.10.1rc1
+Requires-Dist: vantage6-common==4.11.0rc2
+Requires-Dist: vantage6-client==4.11.0rc2
 Provides-Extra: dev
 Requires-Dist: coverage==6.4.4; extra == "dev"
 Requires-Dist: black; extra == "dev"

{vantage6-4.10.1rc1.dist-info → vantage6-4.11.0rc2.dist-info}/RECORD

@@ -1,15 +1,16 @@
 tests_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests_cli/test_client_script.py,sha256=pJiCk0Yu5xyAcj_FzdUCa4av0dKKrXvpr_Ybz9fW0ew,647
 tests_cli/test_example.py,sha256=0fw_v-lgZEacshWSDwLNyLMA1_xc48bKUGM3ll-n1L0,146
 tests_cli/test_node_cli.py,sha256=XAKRS1-SXdeBRz3ob25q467soCIB178pKVz8MJVZbCQ,17534
 tests_cli/test_server_cli.py,sha256=DLcOWM0ZCRTs-XMNsNKCOEELUMh22Ayow1FvAFLI0aA,5879
 tests_cli/test_wizard.py,sha256=NIj59eiCBuVNJXwhofrWLmLIKAsD45gSOzqOFWLmWhY,4916
-vantage6/cli/__build__,sha256=a4ayc_80_OGda4BO_1o_V0etpOqiLx1JwB5S3beHW0s,1
+vantage6/cli/__build__,sha256=1HNeOiZeFu7gP1lxi5tdAwGcB9i2xR-Q2jpmbuwTqzU,1
 vantage6/cli/__init__.py,sha256=ZXbeQ_-g2-M4XYteWZkoO5lMFYhqjm5doQgGy1fq8i0,125
-vantage6/cli/_version.py,sha256=nMTSumCNc9_aCqofYJOR0p4Siz14Bqne4oF2KVFvzJk,701
-vantage6/cli/cli.py,sha256=NHpiPcljADaVzwGYrG4FWVMfbmDRguSZWBeGxcdQxDo,6184
-vantage6/cli/configuration_manager.py,sha256=QuR8Lvgjfp36aLnMV3PdbjddVJHOCvOv2Vci1bYHDHY,3642
+vantage6/cli/_version.py,sha256=7g1VYEN766Mf9wGlm8bPgQO8QzE0d-l9gt02DbZwjWE,701
+vantage6/cli/cli.py,sha256=L_t0sHUwVkEumkYg6MsXm_sdAEWEjiz5DkUa51ihGtU,6318
+vantage6/cli/configuration_manager.py,sha256=YqObxbV4jl3K_0GiJcE0V7r2pFE2nETLEgP3yieYZaU,3695
 vantage6/cli/configuration_wizard.py,sha256=ifqvrVqHkxoM0ZVUVIwlYXFByzAbuVlahNjmwFGLVRU,20874
-vantage6/cli/globals.py,sha256=8AWI55FBbumVQTuI1bJzKp5hiRWtiwsVgTTKqWgRBes,1616
+vantage6/cli/globals.py,sha256=TBvO_ywYXRIyX1GlKUND_s6s9dPAEpdlh7p8i6aKSYA,1953
 vantage6/cli/utils.py,sha256=Jfr6IeHMQDk_wU5X7rJ1dRY118dhVVX8PwzwMYMv9Vw,2481
 vantage6/cli/algorithm/create.py,sha256=kRT1BlBcb0fDaB2Q988WxtA6EyAZmOW5QoU2uhbwBIo,2075
 vantage6/cli/algorithm/update.py,sha256=WwAfTnq0kTOgePUsBzGoo1AJQqGMn82E9Bjk1wf61CQ,1338
@@ -27,8 +28,8 @@ vantage6/cli/context/__init__.py,sha256=e8rfY2tCyu6_SLQ-rbVzEHkDtmbnGCZRHFN_HH-2
 vantage6/cli/context/algorithm_store.py,sha256=RimxNcoqfWeu2WQede6wsOu1rx-azzXIPVkCDqVJLWs,3944
 vantage6/cli/context/base_server.py,sha256=paKSzNrKWD-J6eakHAtGELk2cD05A8NqoCAuQfF7c2s,2972
 vantage6/cli/context/node.py,sha256=4R7X__u_pyZVvRSNO3ojTnVQFTWhuIemoHcGP6hsD5I,7444
-vantage6/cli/context/server.py,sha256=vBGJWNsJoVcIryX5OLiWnFklNRcjOVkhqm2U5tqW5b0,3946
-vantage6/cli/dev/create.py,sha256=6LiK0MUZjZK_W932WnlMMVeCqX1L11F87Rk1UkU6O-4,19347
+vantage6/cli/context/server.py,sha256=4BiccAakyXviGiV1SisZ0w0-0hrKAmCGVneZCRsSne8,4563
+vantage6/cli/dev/create.py,sha256=ZN4BfExJVypl28ZKQEua0PIXpQCfcP8AC3RUhp1Cj-I,21280
 vantage6/cli/dev/remove.py,sha256=R_OU_LXLDCnoD-2xnegg4lh0B3t8EgpqzDqueLx16io,3730
 vantage6/cli/dev/start.py,sha256=fUMoPAEpmXoDAJidAmjIziaHZX1yqEErcrTKEXqPik8,4471
 vantage6/cli/dev/stop.py,sha256=gPy87r8T3nqe7RFJjlYE9Bns8N3RiPPcdzNIFCoqGRY,1430
@@ -47,6 +48,7 @@ vantage6/cli/node/start.py,sha256=mV4vDBBtFcxva4MJdg5ymsstefDM4bV4pmhSsVX1o4k,12
 vantage6/cli/node/stop.py,sha256=-jw0DChIoUScVVwY4SBi_AvmdXT0XlMsGVuO9_vrpJI,4085
 vantage6/cli/node/version.py,sha256=X921xyIvIPYObPac2Si5msZ2tay5ySidnPWmGj1ilZw,1959
 vantage6/cli/node/common/__init__.py,sha256=ziGS3skkX6Kf4uvFqe22kdFbSck7mubNK5a-KgDAxX8,3467
+vantage6/cli/prometheus/monitoring_manager.py,sha256=fbzzUkXKogoHgfR-pKn7DASMBfXZnio8ft77VMBSjDE,4889
 vantage6/cli/rabbitmq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vantage6/cli/rabbitmq/definitions.py,sha256=CcS9jG7ZGB6LjzHQqZ2FliDurPItUvNSjHrOYptORZg,637
 vantage6/cli/rabbitmq/queue_manager.py,sha256=KGDGHy4NBN8O9xhjzfI7mh65i9lOQIqQwrOFqvGFdHI,7545
@@ -58,19 +60,22 @@ vantage6/cli/server/list.py,sha256=_kVBe7TrtUfT2ZLWPkVvKj4xx5AvS17e99Bwp0ajc_E,4
 vantage6/cli/server/new.py,sha256=JnUMMD0UUSCOdO0K7v19l5HOV7vnMd9D38WUoEJ2F00,2094
 vantage6/cli/server/remove.py,sha256=6tfKfVa5dYnZAKQYo_VlGZTuiugi7sh2F3U2cZ7mCmQ,1627
 vantage6/cli/server/shell.py,sha256=7cSdBOwvloWyCz3RXHaFZUnxgNRV4gZUDmIFecV8Hks,1589
-vantage6/cli/server/start.py,sha256=mPfsfkihJWf8bObNoln4YuBwOCVx3EstXtcNft8Pigo,7787
-vantage6/cli/server/stop.py,sha256=DY3r9VsUk_r3cqIm1iL-U-kstLVb9pZsiGDSZyAMrKA,4147
+vantage6/cli/server/start.py,sha256=u3wWHlS7FjAkOHndCrebC6uCPuvRUQFci7dSlGDYMTs,9614
+vantage6/cli/server/stop.py,sha256=CP8WtMLkxRhKRr6PnATkCzjJoSVzy9XR9B1ETWtarmU,4422
 vantage6/cli/server/version.py,sha256=aXAztHEky_F2jPbfPdHPfsAY7rdTurl0_3S6bL94_QQ,1318
 vantage6/cli/server/common/__init__.py,sha256=htv0mFYa4GhIHdzA2xqUUgKhHcMh09UQERlIjIgrwOM,2062
 vantage6/cli/template/algo_store_config.j2,sha256=XR-ly-47p6egH8lVh4lZZDh3YSV4kFnkZprdsfSkS2Y,552
-vantage6/cli/template/node_config.j2,sha256=XHJm5x5KEwuBAZERzWzzVKJxcw7Px5k-LYSMET_8dqU,743
+vantage6/cli/template/node_config.j2,sha256=1uK21oLan2r48-v_HyIW2TeO5bR9ER_Wy0PICHQ-xXo,781
 vantage6/cli/template/server_config.j2,sha256=3gEPY8YlqUMAQEgfR7a1HTU8WaCRhVzTS-IwPhsU1Gg,802
 vantage6/cli/template/server_import_config.j2,sha256=9WT2XeG9-ADoYLb4ahXhof3i9Fcvg0oqwNPyFwLJpvc,1827
+vantage6/cli/test/client_script.py,sha256=yKZECAMox8sgnHZQyxWTBxILnqQIAbf6v884wgaMkuE,4195
 vantage6/cli/test/feature_tester.py,sha256=M8hvebupPwYjcBZoUB8GB3qb8G1-d3ipNzRMc_3-Z8E,2761
 vantage6/cli/test/integration_test.py,sha256=yQVG72XKDNH_eOPTsf3pb65FCBwJzMxn5VNfUGemJBM,3808
+vantage6/cli/test/algo_test_scripts/algo_test_arguments.py,sha256=PfJUwXXbOUvs8W2i3XqIoNNkqvnXmRTv19kKcpg9ouk,888
+vantage6/cli/test/algo_test_scripts/algo_test_script.py,sha256=F1meN4vI4Nz7_RXD_-xmclfw3M5MzBKGINgzbZmDaN8,2652
 vantage6/cli/test/common/diagnostic_runner.py,sha256=x_4ikihgoSTKI914pqlgVziBSg5LpV6MheO6O_GBCeA,6657
-vantage6-4.10.1rc1.dist-info/METADATA,sha256=SOdtW3t66t8QFaPpgkZet9CQYIboQxydipcaDqQDpy4,10890
-vantage6-4.10.1rc1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-vantage6-4.10.1rc1.dist-info/entry_points.txt,sha256=YFBvwjxoeAGxYyPC-YevEgOBBYRGaXkS6jiOGGCLNy0,157
-vantage6-4.10.1rc1.dist-info/top_level.txt,sha256=CYDIBS8jEfFq5YCs_Fuit54K9-3wdosZppTrsymIoUk,19
-vantage6-4.10.1rc1.dist-info/RECORD,,
+vantage6-4.11.0rc2.dist-info/METADATA,sha256=0ATXMr019zRsaUuKNRhin-W_4AUlfbNB3SOT1st4vAg,10890
+vantage6-4.11.0rc2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+vantage6-4.11.0rc2.dist-info/entry_points.txt,sha256=YFBvwjxoeAGxYyPC-YevEgOBBYRGaXkS6jiOGGCLNy0,157
+vantage6-4.11.0rc2.dist-info/top_level.txt,sha256=CYDIBS8jEfFq5YCs_Fuit54K9-3wdosZppTrsymIoUk,19
+vantage6-4.11.0rc2.dist-info/RECORD,,