vantage6 5.0.0a22__py3-none-any.whl → 5.0.0a29__py3-none-any.whl

vantage6/cli/dev/remove.py

@@ -1,111 +1,112 @@
-import subprocess
-import itertools
-from shutil import rmtree
-from pathlib import Path
+# import subprocess
+# import itertools
+# from shutil import rmtree
+# from pathlib import Path
 
-import click
-import docker
-from colorama import Fore, Style
+# import click
+# import docker
+# from colorama import Fore, Style
 
-from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
-from vantage6.common import info, error
-from vantage6.common.globals import APPNAME
-from vantage6.cli.context.server import ServerContext
-from vantage6.cli.context.node import NodeContext
-from vantage6.cli.server.remove import cli_server_remove
-from vantage6.cli.utils import remove_file
-from vantage6.common.globals import InstanceType
-from vantage6.cli.dev.utils import get_dev_server_context
+# from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+# from vantage6.common import info, error
+# from vantage6.common.globals import APPNAME
+# from vantage6.cli.context.server import ServerContext
+# from vantage6.cli.context.node import NodeContext
+# from vantage6.cli.server.remove import cli_server_remove
+# from vantage6.cli.utils import remove_file
+# from vantage6.common.globals import InstanceType
 
+# # from vantage6.cli.dev.utils import get_dev_server_context
 
-@click.command()
-@click.option("-n", "--name", default=None, help="Name of the configuration.")
-@click.option(
-    "-c",
-    "--config",
-    default=None,
-    help="Path to configuration-file; overrides --name",
-)
-@click.pass_context
-def remove_demo_network(
-    click_ctx: click.Context, name: str | None, config: str | None
-) -> None:
-    """Remove all related demo network files and folders.
 
-    Select a server configuration to remove that server and the nodes attached
-    to it.
-    """
-    ctx = get_dev_server_context(config, name)
+# @click.command()
+# @click.option("-n", "--name", default=None, help="Name of the configuration.")
+# @click.option(
+#     "-c",
+#     "--config",
+#     default=None,
+#     help="Path to configuration-file; overrides --name",
+# )
+# @click.pass_context
+# def remove_demo_network(
+#     click_ctx: click.Context, name: str | None, config: str | None
+# ) -> None:
+#     """Remove all related demo network files and folders.
 
-    # check that the server is not running
-    client = docker.from_env()
-    running_servers = client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.SERVER.value}"}
-    )
-    running_server_names = [server.name for server in running_servers]
-    container_name = f"{APPNAME}-{name}-user-{InstanceType.SERVER.value}"
-    if container_name in running_server_names:
-        error(
-            f"Server {Fore.RED}{name}{Style.RESET_ALL} is still running! First stop "
-            "the network with 'v6 dev stop-demo-network'."
-        )
-        return
+#     Select a server configuration to remove that server and the nodes attached
+#     to it.
+#     """
+#     ctx = get_dev_server_context(config, name)
 
-    # remove the server
-    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-        handler.close()
-    click_ctx.invoke(cli_server_remove, ctx=ctx, force=True)
+#     # check that the server is not running
+#     client = docker.from_env()
+#     running_servers = client.containers.list(
+#         filters={"label": f"{APPNAME}-type={InstanceType.SERVER}"}
+#     )
+#     running_server_names = [server.name for server in running_servers]
+#     container_name = f"{APPNAME}-{name}-user-{InstanceType.SERVER}"
+#     if container_name in running_server_names:
+#         error(
+#             f"Server {Fore.RED}{name}{Style.RESET_ALL} is still running! First stop "
+#             "the network with 'v6 dev stop-demo-network'."
+#         )
+#         return
 
-    # removing the server import config
-    info("Deleting demo import config file")
-    server_configs = ServerContext.instance_folders(
-        InstanceType.SERVER, ctx.name, system_folders=False
-    )
-    import_config_to_del = Path(server_configs["dev"]) / f"{ctx.name}.yaml"
-    remove_file(import_config_to_del, "import_configuration")
+#     # remove the server
+#     for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
+#         handler.close()
+#     click_ctx.invoke(cli_server_remove, ctx=ctx, force=True)
 
-    # also remove the server folder
-    server_configs = ServerContext.instance_folders(
-        InstanceType.SERVER, ctx.name, system_folders=False
-    )
-    server_folder = server_configs["data"]
-    if server_folder.is_dir():
-        rmtree(server_folder)
+#     # removing the server import config
+#     info("Deleting demo import config file")
+#     server_configs = ServerContext.instance_folders(
+#         InstanceType.SERVER, ctx.name, system_folders=False
+#     )
+#     import_config_to_del = Path(server_configs["dev"]) / f"{ctx.name}.yaml"
+#     remove_file(import_config_to_del, "import_configuration")
 
-    # remove the store folder
-    store_configs = AlgorithmStoreContext.instance_folders(
-        InstanceType.ALGORITHM_STORE, f"{ctx.name}_store", system_folders=False
-    )
-    store_folder = store_configs["data"]
-    if store_folder.is_dir():
-        rmtree(store_folder)
+#     # also remove the server folder
+#     server_configs = ServerContext.instance_folders(
+#         InstanceType.SERVER, ctx.name, system_folders=False
+#     )
+#     server_folder = server_configs["data"]
+#     if server_folder.is_dir():
+#         rmtree(server_folder)
 
-    # remove the store config file
-    subprocess.run(
-        [
-            "v6",
-            "algorithm-store",
-            "remove",
-            "-n",
-            f"{ctx.name}_store",
-            "--force",
-            "--user",
-        ]
-    )
+#     # remove the store folder
+#     store_configs = AlgorithmStoreContext.instance_folders(
+#         InstanceType.ALGORITHM_STORE, f"{ctx.name}_store", system_folders=False
+#     )
+#     store_folder = store_configs["data"]
+#     if store_folder.is_dir():
+#         rmtree(store_folder)
 
-    # remove the nodes
-    configs, _ = NodeContext.available_configurations(system_folders=False)
-    node_names = [
-        config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
-    ]
-    for name in node_names:
-        node_ctx = NodeContext(name, False)
-        for handler in itertools.chain(
-            node_ctx.log.handlers, node_ctx.log.root.handlers
-        ):
-            handler.close()
-        subprocess.run(["v6", "node", "remove", "-n", name, "--user", "--force"])
+#     # remove the store config file
+#     subprocess.run(
+#         [
+#             "v6",
+#             "algorithm-store",
+#             "remove",
+#             "-n",
+#             f"{ctx.name}_store",
+#             "--force",
+#             "--user",
+#         ]
+#     )
 
-    # remove data files attached to the network
-    data_dirs_nodes = NodeContext.instance_folders("node", "", False)["dev"]
-    rmtree(Path(data_dirs_nodes / ctx.name))
+#     # remove the nodes
+#     configs, _ = NodeContext.available_configurations(system_folders=False)
+#     node_names = [
+#         config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
+#     ]
+#     for name in node_names:
+#         node_ctx = NodeContext(name, False)
+#         for handler in itertools.chain(
+#             node_ctx.log.handlers, node_ctx.log.root.handlers
+#         ):
+#             handler.close()
+#         subprocess.run(["v6", "node", "remove", "-n", name, "--user", "--force"])
+
+#     # remove data files attached to the network
+#     data_dirs_nodes = NodeContext.instance_folders("node", "", False)["dev"]
+#     rmtree(Path(data_dirs_nodes / ctx.name))

vantage6/cli/globals.py

@@ -2,10 +2,18 @@
 This module contains global variables that are used throughout the CLI.
 """
 
-from enum import Enum
 from pathlib import Path
+
+from vantage6.common.enum import StrEnumBase
 from vantage6.common.globals import APPNAME
 
+#
+#  CLI SETTINGS
+#
+
+DEFAULT_CLI_CONFIG_FOLDER = Path.home() / ".vantage6"
+DEFAULT_CLI_CONFIG_FILE = DEFAULT_CLI_CONFIG_FOLDER / "config.yaml"
+
 #
 #   SERVER SETTINGS
 #
@@ -44,22 +52,34 @@ DIAGNOSTICS_IMAGE = "harbor2.vantage6.ai/algorithms/diagnostic"
 # Address of community algorithm store
 COMMUNITY_STORE = "https://store.cotopaxi.vantage6.ai/api"
 
+DEFAULT_PROMETHEUS_IMAGE = "prom/prometheus"
+PROMETHEUS_CONFIG = "prometheus.yml"
+PROMETHEUS_DIR = "prometheus"
+
+
+# datasets included in the nodes of the dev network
+class DefaultDatasets(StrEnumBase):
+    """Enum containing default datasets"""
+
+    OLYMPIC_ATHLETES = "olympic_athletes_2016.csv"
+    KAPLAN_MEIER_TEST = "km_dataset.csv"
+
 
-class ServerType(str, Enum):
+class ServerType(StrEnumBase):
     """Enum containing server types"""
 
     V6SERVER = "server"
     ALGORITHM_STORE = "algorithm-store"
 
 
-class ServerGlobals(str, Enum):
+class ServerGlobals(StrEnumBase):
     """Enum containing server environment variables"""
 
     DB_URI_ENV_VAR = "VANTAGE6_DB_URI"
     CONFIG_NAME_ENV_VAR = "VANTAGE6_CONFIG_NAME"
 
 
-class AlgoStoreGlobals(str, Enum):
+class AlgoStoreGlobals(StrEnumBase):
     """Enum containing algorithm store environment variables"""
 
     DB_URI_ENV_VAR = "VANTAGE6_ALGO_STORE_DB_URI"

vantage6/cli/node/common/__init__.py

@@ -3,15 +3,17 @@ Common functions that are used in node CLI commands
 """
 
 import os
+
 import docker
 from colorama import Fore, Style
 
-from vantage6.common import error, info, debug
+from vantage6.common import debug, error, info
 from vantage6.common.globals import APPNAME, InstanceType, RequiredNodeEnvVars
+
 from vantage6.client import UserClient
 
-from vantage6.cli.context.node import NodeContext
 from vantage6.cli.configuration_wizard import select_configuration_questionaire
+from vantage6.cli.context.node import NodeContext
 
 
 def create_client(ctx: NodeContext) -> UserClient:
@@ -86,8 +88,7 @@ def select_node(name: str, system_folders: bool) -> tuple[str, str]:
     # raise error if config could not be found
     if not NodeContext.config_exists(name, system_folders):
         error(
-            f"The configuration {Fore.RED}{name}{Style.RESET_ALL} could "
-            f"not be found."
+            f"The configuration {Fore.RED}{name}{Style.RESET_ALL} could not be found."
         )
         exit(1)
     return name
@@ -108,6 +109,6 @@ def find_running_node_names(client: docker.DockerClient) -> list[str]:
         List of names of running nodes
     """
     running_nodes = client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.NODE.value}"}
+        filters={"label": f"{APPNAME}-type={InstanceType.NODE}"}
     )
     return [node.name for node in running_nodes]

vantage6/cli/node/new.py

@@ -1,12 +1,13 @@
 import click
 from colorama import Fore, Style
 
-from vantage6.common import error, info, ensure_config_dir_writable
+from vantage6.common import ensure_config_dir_writable, error, info
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.configuration_wizard import configuration_wizard
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
-from vantage6.cli.configuration_wizard import configuration_wizard
 from vantage6.cli.utils import check_config_name_allowed, prompt_config_name
-from vantage6.common.globals import InstanceType
 
 
 @click.command()

vantage6/cli/node/remove.py

@@ -14,14 +14,16 @@ from vantage6.common import (
 )
 from vantage6.common.globals import APPNAME
 
-from vantage6.common.globals import VPN_CONFIG_FILE
-
 
 from vantage6.cli.context.node import NodeContext
 from vantage6.cli.globals import DEFAULT_NODE_SYSTEM_FOLDERS as N_FOL
 from vantage6.cli.utils import check_if_docker_daemon_is_running, remove_file
 from vantage6.cli.node.common import select_node, find_running_node_names
 
+# TODO v5+ remove this - just a dummy to prevent import issues from v4 CLI
+# from vantage6.common.globals import VPN_CONFIG_FILE
+VPN_CONFIG_FILE = "vpn.conf"
+
 
 @click.command()
 @click.option("-n", "--name", default=None, help="Configuration name")

vantage6/cli/node/start.py

@@ -1,15 +1,18 @@
+import os.path
+import time
 from pathlib import Path
 from threading import Thread
-import time
-import os.path
 
 import click
 import docker
-
 from colorama import Fore, Style
 
-from vantage6.cli.common.start import pull_infra_image
-from vantage6.common import warning, error, info, debug, get_database_config
+from vantage6.common import debug, error, info, warning
+from vantage6.common.dataclass import TaskDB
+from vantage6.common.docker.addons import (
+    check_docker_running,
+    remove_container_if_exists,
+)
 from vantage6.common.globals import (
     APPNAME,
     DEFAULT_DOCKER_REGISTRY,
@@ -17,17 +20,14 @@ from vantage6.common.globals import (
     DEFAULT_NODE_IMAGE_WO_TAG,
     InstanceType,
 )
-from vantage6.common.docker.addons import (
-    remove_container_if_exists,
-    check_docker_running,
-)
 
+from vantage6.cli import __version__
 from vantage6.cli.common.decorator import click_insert_context
-from vantage6.cli.context.node import NodeContext
+from vantage6.cli.common.start import pull_infra_image
 from vantage6.cli.common.utils import print_log_worker
+from vantage6.cli.context.node import NodeContext
 from vantage6.cli.node.common import create_client
 from vantage6.cli.utils import check_config_name_allowed
-from vantage6.cli import __version__
 
 
 @click.command()
@@ -40,18 +40,17 @@ from vantage6.cli import __version__
 @click.option(
     "--force-db-mount",
     is_flag=True,
-    help="Always mount node databases; skip the check if they are " "existing files.",
+    help="Always mount node databases; skip the check if they are existing files.",
 )
 @click.option(
     "--attach/--detach",
     default=False,
-    help="Show node logs on the current console after starting the " "node",
+    help="Show node logs on the current console after starting the node",
 )
 @click.option(
     "--mount-src",
     default="",
-    help="Override vantage6 source code in container with the source"
-    " code in this path",
+    help="Override vantage6 source code in container with the source code in this path",
 )
 @click_insert_context(InstanceType.NODE, include_name=True, include_system_folders=True)
 def cli_node_start(
@@ -78,7 +77,7 @@ def cli_node_start(
 
     # check that this node is not already running
     running_nodes = docker_client.containers.list(
-        filters={"label": f"{APPNAME}-type={InstanceType.NODE.value}"}
+        filters={"label": f"{APPNAME}-type={InstanceType.NODE}"}
     )
 
     suffix = "system" if system_folders else "user"
@@ -116,9 +115,7 @@ def cli_node_start(
                     f":{major_minor}"
                 )
             except Exception:
-                warning(
-                    "Could not determine server version. Using default " "node image"
-                )
+                warning("Could not determine server version. Using default node image")
 
             if major_minor and not __version__.startswith(major_minor):
                 warning(
@@ -213,29 +210,25 @@ def cli_node_start(
 
     # only mount the DB if it is a file
     info("Setting up databases")
-    db_labels = [db["label"] for db in ctx.databases]
-    for label in db_labels:
+    dbs = [TaskDB.from_dict(db) for db in ctx.databases]
+    for db in dbs:
         # check that label contains only valid characters
-        if not label.isidentifier():
+        if not db.label.isidentifier():
             error(
-                f"Database label {Fore.RED}{label}{Style.RESET_ALL} contains"
+                f"Database label {Fore.RED}{db.label}{Style.RESET_ALL} contains"
                 " invalid characters. Only letters, numbers, and underscores"
                 " are allowed, and it cannot start with a number."
             )
             exit(1)
 
-        db_config = get_database_config(ctx.databases, label)
-        uri = db_config["uri"]
-        db_type = db_config["type"]
-
         info(
-            f"  Processing {Fore.GREEN}{db_type}{Style.RESET_ALL} database "
-            f"{Fore.GREEN}{label}:{uri}{Style.RESET_ALL}"
+            f"  Processing {Fore.GREEN}{db.type}{Style.RESET_ALL} database "
+            f"{Fore.GREEN}{db.label}:{db.uri}{Style.RESET_ALL}"
         )
-        label_capitals = label.upper()
+        label_capitals = db.label.upper()
 
         try:
-            db_file_exists = Path(uri).exists()
+            db_file_exists = Path(db.uri).exists()
         except Exception:
             # If the database uri cannot be parsed, it is definitely not a
             # file. In case of http servers or sql servers, checking the path
@@ -243,22 +236,22 @@ def cli_node_start(
             # we catch all exceptions here.
             db_file_exists = False
 
-        if db_type in ["folder", "csv", "parquet", "excel"] and not db_file_exists:
+        if db.type.is_file_based() and not db_file_exists:
             error(
-                f"Database {Fore.RED}{uri}{Style.RESET_ALL} not found. Databases of "
-                f"type '{db_type}' must be present on the harddrive. Please update "
-                "your node configuration file."
+                f"Database {Fore.RED}{db.uri}{Style.RESET_ALL} not found. Databases of "
+                f"type '{db.type}' must be present on the harddrive. Please "
+                "update your node configuration file."
             )
             exit(1)
 
         if not db_file_exists and not force_db_mount:
             debug("  - non file-based database added")
-            env[f"{label_capitals}_DATABASE_URI"] = uri
+            env[f"{label_capitals}_DATABASE_URI"] = db.uri
         else:
             debug("  - file-based database added")
-            suffix = Path(uri).suffix
-            env[f"{label_capitals}_DATABASE_URI"] = f"{label}{suffix}"
-            mounts.append((f"/mnt/{label}{suffix}", str(uri)))
+            suffix = Path(db.uri).suffix
+            env[f"{label_capitals}_DATABASE_URI"] = f"{db.label}{suffix}"
+            mounts.append((f"/mnt/{db.label}{suffix}", str(db.uri)))
 
     system_folders_option = "--system" if system_folders else "--user"
     cmd = (
@@ -286,9 +279,7 @@ def cli_node_start(
     # we won't accept overwrites of existing env vars
     env_overwrites = extra_env.keys() & env.keys()
     if env_overwrites:
-        error(
-            "Cannot overwrite existing node environment variables: " f"{env_overwrites}"
-        )
+        error(f"Cannot overwrite existing node environment variables: {env_overwrites}")
         exit(1)
     env.update(extra_env)
 

vantage6/cli/prometheus/monitoring_manager.py

@@ -0,0 +1,146 @@
+import yaml
+import docker
+from pathlib import Path
+
+from vantage6.common import info, error
+from vantage6.common.docker.network_manager import NetworkManager
+from vantage6.common.globals import DEFAULT_PROMETHEUS_EXPORTER_PORT
+from vantage6.cli.context.server import ServerContext
+from vantage6.cli.globals import (
+    DEFAULT_PROMETHEUS_IMAGE,
+    PROMETHEUS_CONFIG,
+)
+
+
+class PrometheusServer:
+    """
+    Manages the Prometheus Docker container
+    """
+
+    def __init__(
+        self, ctx: ServerContext, network_mgr: NetworkManager, image: str = None
+    ):
+        """
+        Initialize the PrometheusServer instance.
+
+        Parameters
+        ----------
+        ctx : ServerContext
+            The server context containing configuration and paths.
+        network_mgr : NetworkManager
+            The network manager responsible for managing Docker networks.
+        image : str, optional
+            The Docker image to use for the Prometheus container. If not provided,
+            the default Prometheus image will be used.
+        """
+        self.ctx = ctx
+        self.network_mgr = network_mgr
+        self.docker = docker.from_env()
+        self.image = image if image else DEFAULT_PROMETHEUS_IMAGE
+        self.config_file = Path(__file__).parent / PROMETHEUS_CONFIG
+        self.data_dir = self.ctx.prometheus_dir
+
+    def start(self):
+        """
+        Start a Docker container running Prometheus
+        """
+        self._prepare_config()
+
+        volumes = {
+            str(self.config_file): {
+                "bind": "/etc/prometheus/prometheus.yml",
+                "mode": "ro",
+            },
+            str(self.data_dir): {"bind": "/prometheus", "mode": "rw"},
+        }
+        ports = {"9090/tcp": 9090}
+
+        if self._is_container_running():
+            info("Prometheus is already running!")
+            return
+
+        self.docker.containers.run(
+            name=self.ctx.prometheus_container_name,
+            image=self.image,
+            volumes=volumes,
+            ports=ports,
+            detach=True,
+            restart_policy={"Name": "unless-stopped"},
+            network=self.network_mgr.network_name,
+        )
+        info("Prometheus container started successfully!")
+
+    def _prepare_config(self):
+        """
+        Prepare the Prometheus configuration and data directories
+        """
+        if not self.config_file.exists():
+            error(f"Prometheus configuration file {self.config_file} not found!")
+            raise FileNotFoundError(f"{self.config_file} not found!")
+
+        if not self.data_dir.exists():
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+
+        self._update_prometheus_config()
+
+    def _update_prometheus_config(self):
+        """
+        Update the Prometheus configuration file with the server address.
+        """
+
+        try:
+            prometheus_exporter_port = self.ctx.config.get("prometheus", {}).get(
+                "exporter_port", DEFAULT_PROMETHEUS_EXPORTER_PORT
+            )
+            server_hostname = self.ctx.prometheus_container_name
+            server_address = f"{server_hostname}:{prometheus_exporter_port}"
+
+            info(
+                f"Using Docker container hostname '{server_hostname}' for Prometheus "
+                "target. Ensure Prometheus is in the same Docker network to resolve "
+                "this address."
+            )
+
+            with open(self.config_file, "r") as f:
+                config = yaml.safe_load(f)
+
+            job_name = "vantage6_server_metrics"
+            job_exists = any(
+                job.get("job_name") == job_name
+                for job in config.get("scrape_configs", [])
+            )
+
+            if not job_exists:
+                new_job = {
+                    "job_name": job_name,
+                    "static_configs": [{"targets": [server_address]}],
+                }
+                config.setdefault("scrape_configs", []).append(new_job)
+            else:
+                for job in config["scrape_configs"]:
+                    if job.get("job_name") == job_name:
+                        job["static_configs"] = [{"targets": [server_address]}]
+
+            with open(self.config_file, "w") as f:
+                yaml.dump(config, f)
+
+            info(f"Prometheus configuration updated with target: {server_address}")
+
+        except Exception as e:
+            error(f"Failed to update Prometheus configuration: {e}")
+            raise
+
+    def _is_container_running(self) -> bool:
+        """
+        Check if a Prometheus container is already running.
+
+        Returns
+        -------
+        bool
+            True if the Prometheus container is running, False otherwise.
+        """
+        try:
+            container = self.docker.containers.get(self.ctx.prometheus_container_name)
+            return container.status == "running"
+        except docker.errors.NotFound:
+            return False

vantage6/cli/prometheus/prometheus.yml

@@ -0,0 +1,5 @@
+global:
+  # How frequently to scrape targets by default.
+  scrape_interval: 1m
+
+scrape_configs: []