vantage6 5.0.0a36__py3-none-any.whl → 5.0.0a38__py3-none-any.whl

vantage6/cli/sandbox/populate/helpers/load_fixtures.py

@@ -0,0 +1,476 @@
+"""
+Development script to populate the server
+"""
+
+import traceback
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from vantage6.client import Client
+
+from vantage6.cli.globals import APPNAME, PACKAGE_FOLDER
+from vantage6.cli.sandbox.populate.helpers.utils import (
+    NodeConfigCreationDetails,
+    replace_wsl_path,
+)
+
+
+def clear_dev_folder(dev_dir: Path, name: str) -> None:
+    node_dev_dir = dev_dir / name
+    if node_dev_dir.exists():
+        for file_ in node_dev_dir.iterdir():
+            file_.unlink()
+        node_dev_dir.rmdir()
+        print(f"===> Dev folder for node `{name}` cleared")
+
+
+def create_organizations(
+    client: Client, number_of_nodes: int
+) -> tuple[list[dict], dict]:
+    """
+    Create organizations. If the organization already exists, it is added to the
+    existing organizations list. If the organization is the root organization, it is
+    patched so that the admin user is also in the organization.
+
+    Returns
+    -------
+    tuple[list[dict], dict]
+        A tuple containing the list of organizations and the creation details.
+    """
+    organizations = []
+    creation_details = {
+        "created": [],
+        "existing": [],
+        "root_org_patched": [],
+    }
+
+    existing_organizations = client.organization.list()["data"]
+    for i in range(1, number_of_nodes + 1):
+        name = f"org_{i}"
+        if org := next(
+            iter([org for org in existing_organizations if org["name"] == name]), None
+        ):
+            creation_details["existing"].append({"name": name, "domain": org["domain"]})
+            organizations.append(org)
+        elif i == 1:
+            # Patch the root organization so that admin user is also in the org
+            org = client.organization.update(
+                id_=1,
+                name=name,
+            )
+            creation_details["root_org_patched"].append(
+                {"name": name, "domain": org["domain"]}
+            )
+            organizations.append(org)
+        else:
+            org = client.organization.create(
+                name=name,
+                address1=f"First address line {i}",
+                address2=f"Second address line {i}",
+                zipcode="1234AB",
+                country="Earthland",
+                domain=f"org{i}.org",
+            )
+            creation_details["created"].append({"name": name, "domain": org["domain"]})
+            organizations.append(org)
+
+    return organizations, creation_details
+
+
+def create_collaborations(
+    client: Client, organizations: list[dict]
+) -> tuple[list[dict], dict]:
+    """
+    Create collaborations. If the collaboration already exists, it is added to the
+    existing collaborations list.
+
+    Returns
+    -------
+    tuple[dict, dict]
+        A tuple containing the collaboration and the creation details.
+    """
+    creation_details = {"created": [], "existing": []}
+    collab_name = "demo"
+    existing_collaborations = client.collaboration.list(
+        scope="global", name=collab_name
+    )["data"]
+    if collab := next(iter(existing_collaborations), None):
+        creation_details["existing"].append({"name": collab_name, "id": collab["id"]})
+    else:
+        collab = client.collaboration.create(
+            name=collab_name,
+            organizations=[org["id"] for org in organizations],
+            encrypted=False,
+        )
+        creation_details["created"].append({"name": collab_name, "id": collab["id"]})
+    return collab, creation_details
+
+
+def create_users(client: Client, organizations: list[dict]) -> dict:
+    """
+    Create users. If the user already exists, it is added to the existing users list.
+
+    Returns
+    -------
+    dict
+        The creation details.
+    """
+    creation_details = {"created": [], "existing": []}
+    existing_users = client.user.list()["data"]
+    for index, org in enumerate(organizations):
+        username = f"user_{index + 1}"
+        if next(
+            iter([user for user in existing_users if user["username"] == username]),
+            None,
+        ):
+            creation_details["existing"].append(
+                {
+                    "username": username,
+                    "organization": org["name"],
+                }
+            )
+        else:
+            password = "Password123!"
+            client.user.create(
+                username=username,
+                password=password,
+                organization=org["id"],
+                roles=[1],  # TODO assign proper roles
+            )
+            creation_details["created"].append(
+                {
+                    "username": username,
+                    "password": password,
+                    "organization": org["name"],
+                }
+            )
+    return creation_details
+
+
+def register_node(
+    client: Client,
+    node_name: str,
+    collaboration: dict,
+    organization: dict,
+) -> dict:
+    """
+    Register a node at the server.
+
+    Returns
+    -------
+    dict
+        The node registration details.
+    """
+    return client.node.create(
+        collaboration=collaboration["id"],
+        organization=organization["id"],
+        name=node_name,
+    )
+
+
+def create_node_config(
+    node_number: int,
+    node_name: str,
+    dev_dir: Path,
+    task_directory: str,
+    task_namespace: str,
+    node_starting_port_number: int,
+    organization: dict,
+    node: dict,
+) -> dict:
+    """
+    Create a node configuration file.
+
+    Returns
+    -------
+    dict
+        The node configuration details.
+    """
+    # Create a folder for all config files for a single node
+    node_dev_dir = dev_dir / node_name
+    node_dev_dir.mkdir(exist_ok=True)
+
+    # Generate node configuration
+    environment = Environment(
+        loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
+        trim_blocks=True,
+        lstrip_blocks=True,
+        autoescape=True,
+    )
+    template = environment.get_template("node_config_nonk8s.j2")
+
+    node_config = template.render(
+        {
+            "logging": {"file": f"node_{node_number}.log"},
+            "port": 7601,
+            "server_url": "http://vantage6-server-vantage6-server-service",
+            "task_dir": f"{task_directory}/node_{node_number}",
+            "task_dir_extension": f"node_{node_number}",
+            "api_path": "/server",
+            "task_namespace": task_namespace,
+            "node_proxy_port": node_starting_port_number + (node_number - 1),
+        }
+    )
+    config_file = node_dev_dir / f"node_org_{node_number}.yaml"
+    with open(config_file, "w") as f:
+        f.write(node_config)
+
+    # also make sure the task directory exists
+    task_dir = Path(f"{task_directory}/node_{node_number}")
+    task_dir = replace_wsl_path(task_dir)
+    task_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create .env file for the node
+    env_file = node_dev_dir / ".env"
+    with open(env_file, "w") as f:
+        f.write(f"V6_API_KEY={node['api_key']}\n")
+        f.write(f"V6_NODE_NAME={node_name}\n")
+
+    return {
+        "name": f"node-{node_number}",
+        "organization": organization["name"],
+        "api_key": node["api_key"],
+        "config_file": str(config_file),
+        "env_file": str(env_file),
+    }
+
+
+def create_session(client: Client, collaboration: dict) -> dict:
+    """
+    Create a session.
+
+    Returns
+    -------
+    dict
+        The session creation details.
+    """
+    session = client.session.create(
+        collaboration=collaboration["id"],
+        name="session (collaboration scope)",
+        scope="collaboration",
+    )
+    creation_details = {
+        "created": [
+            {
+                "name": "session (collaboration scope)",
+                "id": session["id"],
+            }
+        ],
+    }
+    return creation_details
+
+
+def print_creation_details(creation_details: dict) -> str:
+    """
+    Print the creation details.
+
+    Returns
+    -------
+    str
+        The creation summary.
+    """
+    summary = "=== Creation Summary ===\n"
+
+    summary += f"\nOrganizations: {len(creation_details['organizations']['created'])} "
+    summary += f"created, {len(creation_details['organizations']['existing'])} existing"
+    summary += f", {len(creation_details['organizations']['root_org_patched'])} root "
+    summary += "org patched"
+    if creation_details["organizations"]["created"]:
+        summary += "\n  Created:"
+        for org in creation_details["organizations"]["created"]:
+            summary += f"\n    - {org['name']} ({org['domain']})"
+    if creation_details["organizations"]["existing"]:
+        summary += "\n  Existing:"
+        for org in creation_details["organizations"]["existing"]:
+            summary += f"\n    - {org['name']} ({org['domain']})"
+    if creation_details["organizations"]["root_org_patched"]:
+        summary += "\n  Root org patched:"
+        for org in creation_details["organizations"]["root_org_patched"]:
+            summary += f"\n    - {org['name']} ({org['domain']})"
+
+    summary += f"\n\nUsers: {len(creation_details['users']['created'])} created, "
+    summary += f"{len(creation_details['users']['existing'])} existing"
+    if creation_details["users"]["created"]:
+        summary += "\n  Created:"
+        for user in creation_details["users"]["created"]:
+            summary += f"\n    - {user['username']} - Password: "
+            summary += f"{user['password']} - Org: {user['organization']}"
+    if creation_details["users"]["existing"]:
+        summary += "\n  Existing:"
+        for user in creation_details["users"]["existing"]:
+            summary += f"\n    - {user['username']} - Org: "
+            summary += f"{user['organization']}"
+
+    summary += f"\n\nNodes: {len(creation_details['nodes']['created'])} created, "
+    summary += f"{len(creation_details['nodes']['existing'])} existing"
+    if creation_details["nodes"]["created"]:
+        summary += "\n  Created:"
+        for node in creation_details["nodes"]["created"]:
+            summary += f"\n    - {node['name']} (Org: {node['organization']})"
+            summary += f"\n      API Key: {node['api_key']}"
+            if "config_file" in node:
+                summary += f"\n      Config: {node['config_file']}"
+            if "env_file" in node:
+                summary += f"\n      Env: {node['env_file']}"
+    if creation_details["nodes"]["existing"]:
+        summary += "\n  Existing:"
+        for node in creation_details["nodes"]["existing"]:
+            summary += f"\n    - {node['name']} (Org: {node['organization']})"
+
+    summary += (
+        f"\n\nCollaborations: {len(creation_details['collaborations']['created'])} "
+    )
+    summary += (
+        f"created, {len(creation_details['collaborations']['existing'])} existing"
+    )
+    if creation_details["collaborations"]["created"]:
+        summary += "\n  Created:"
+        for collab in creation_details["collaborations"]["created"]:
+            summary += f"\n    - {collab['name']} (ID: {collab['id']})"
+    if creation_details["collaborations"]["existing"]:
+        summary += "\n  Existing:"
+        for collab in creation_details["collaborations"]["existing"]:
+            summary += f"\n    - {collab['name']} (ID: {collab['id']})"
+
+    summary += f"\n\nSessions: {len(creation_details['sessions']['created'])} created"
+    if creation_details["sessions"]["created"]:
+        summary += "\n  Created:"
+        for session in creation_details["sessions"]["created"]:
+            summary += f"\n    - {session['name']} (ID: {session['id']})"
+
+    if creation_details["dev_folders_cleared"]:
+        summary += (
+            f"\n\nDev folders cleared: {len(creation_details['dev_folders_cleared'])}"
+        )
+        for folder in creation_details["dev_folders_cleared"]:
+            summary += f"\n  - {folder}"
+
+    summary += "\n\n======================="
+    print(summary)
+    return summary
+
+
+def create_fixtures(
+    client: Client,
+    number_of_nodes: int,
+    return_as_dict: bool = False,
+    node_config_creation_details: NodeConfigCreationDetails | None = None,
+    clear_dev_folders: bool = False,
+) -> str | dict:
+    """
+    Create the fixtures for the server.
+
+    Parameters
+    ----------
+    client: Client
+        The client to use to create the fixtures.
+    number_of_nodes: int
+        The number of nodes to create.
+    return_as_dict: bool
+        Whether to return the creation details as a dictionary or as a summary string.
+        Default is False.
+    node_config_creation_details: NodeConfigCreationDetails | None
+        The details to use to create the node configs. If not provided, the node configs
+        will not be created.
+    clear_dev_folders: bool
+        Whether to clear the dev folders.
+
+    Returns
+    -------
+    str | dict
+        The creation summary or the creation details as a dictionary.
+    """
+
+    # Track creation details
+    creation_details = {
+        "organizations": {"created": [], "existing": [], "root_org_patched": []},
+        "users": {"created": [], "existing": []},
+        "nodes": {"created": [], "existing": []},
+        "collaborations": {"created": [], "existing": []},
+        "sessions": {"created": []},
+        "dev_folders_cleared": [],
+    }
+
+    # Remove old config files
+    if clear_dev_folders and node_config_creation_details:
+        for node_dir in [
+            d for d in node_config_creation_details.dev_dir.iterdir() if d.is_dir()
+        ]:
+            clear_dev_folder(node_config_creation_details.dev_dir, node_dir.name)
+            creation_details["dev_folders_cleared"].append(node_dir.name)
+
+    # Create organizations
+    organizations, creation_details["organizations"] = create_organizations(
+        client, number_of_nodes
+    )
+
+    # Create collaboration
+    collaboration, creation_details["collaborations"] = create_collaborations(
+        client, organizations
+    )
+
+    # Create users
+    creation_details["users"] = create_users(client, organizations)
+
+    # create collaboration session
+    creation_details["sessions"] = create_session(client, collaboration)
+
+    # Create nodes
+    for i in range(1, number_of_nodes + 1):
+        name = f"node-{i}"
+        if next(
+            iter(
+                [
+                    node
+                    for node in client.node.list(name=name)["data"]
+                    if node["name"] == name
+                ]
+            ),
+            None,
+        ):
+            creation_details["nodes"]["existing"].append(
+                {"name": name, "organization": organizations[i - 1]["name"]}
+            )
+        else:
+            try:
+                node = register_node(
+                    client,
+                    node_name=name,
+                    collaboration=collaboration,
+                    organization=organizations[i - 1],
+                )
+                if node_config_creation_details:
+                    creation_details["nodes"]["created"].append(
+                        create_node_config(
+                            node_number=i,
+                            node_name=name,
+                            dev_dir=node_config_creation_details.dev_dir,
+                            task_directory=node_config_creation_details.task_directory,
+                            task_namespace=node_config_creation_details.task_namespace,
+                            node_starting_port_number=(
+                                node_config_creation_details.node_starting_port_number
+                            ),
+                            node=node,
+                            organization=organizations[i - 1],
+                        )
+                    )
+                else:
+                    creation_details["nodes"]["created"].append(
+                        {
+                            "name": name,
+                            "organization": organizations[i - 1]["name"],
+                            "api_key": node["api_key"],
+                        }
+                    )
+
+            except Exception as e:
+                traceback.print_exc()
+                print(f"Error creating node {name}: {str(e)}")
+
+    # Print creation details
+    printed_summary = print_creation_details(creation_details)
+    if return_as_dict:
+        return creation_details
+    else:
+        return printed_summary

vantage6/cli/sandbox/populate/helpers/utils.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class NodeConfigCreationDetails:
+    node_starting_port_number: int
+    dev_dir: Path
+    task_directory: str
+    task_namespace: str
+
+
+def replace_wsl_path(path: Path, to_mnt_wsl: bool = True) -> Path:
+    """
+    Replace the WSL path with the regular path.
+
+    If the directory contains /run/desktop/mnt/host/wsl, this will be replaced
+    by /mnt/wsl: this is an idiosyncrasy of WSL (for more details, see
+    https://dev.to/nsieg/use-k8s-hostpath-volumes-in-docker-desktop-on-wsl2-4dcl)
+
+    Parameters
+    ----------
+    path: Path
+        Path to replace.
+    to_mnt_wsl: bool
+        If True, the path will be replaced from the /run/desktop/mnt/host/wsl path to
+        the /mnt/wsl path. If false, vice versa. By default, it is False.
+    """
+    wsl_reference_path = "/run/desktop/mnt/host/wsl"
+    wsl_regular_path = "/mnt/wsl"
+    if to_mnt_wsl and str(path).startswith(wsl_reference_path):
+        path = Path(wsl_regular_path) / path.relative_to(wsl_reference_path)
+    elif not to_mnt_wsl and str(path).startswith(wsl_regular_path):
+        path = Path(wsl_reference_path) / path.relative_to(wsl_regular_path)
+    return path

vantage6/cli/sandbox/remove.py

@@ -0,0 +1,155 @@
+import itertools
+from pathlib import Path
+from shutil import rmtree
+
+import click
+
+from vantage6.common import error, warning
+from vantage6.common.globals import InstanceType
+
+from vantage6.cli.auth.remove import auth_remove
+from vantage6.cli.common.remove import execute_remove
+from vantage6.cli.configuration_create import select_configuration_questionnaire
+from vantage6.cli.context import get_context
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
+from vantage6.cli.context.auth import AuthContext
+from vantage6.cli.context.node import NodeContext
+from vantage6.cli.globals import InfraComponentName
+from vantage6.cli.server.remove import cli_server_remove
+
+
+@click.command()
+@click.option("-n", "--name", default=None, help="Name of the configuration.")
+@click.option(
+    "-c",
+    "--config",
+    default=None,
+    help="Path to configuration-file; overrides --name",
+)
+@click.option(
+    "--data-dir",
+    "custom_data_dir",
+    type=click.Path(exists=True),
+    default=None,
+    help="Path to a custom data directory to use. This option is especially useful "
+    "on WSL because of mount issues for default directories. Use the same value as "
+    "was provided when creating the sandbox.",
+)
+@click.pass_context
+def cli_sandbox_remove(
+    click_ctx: click.Context,
+    name: str | None,
+    config: str | None,
+    custom_data_dir: Path | None,
+) -> None:
+    """Remove all related demo network files and folders.
+
+    Select a server configuration to remove that server and the nodes attached
+    to it.
+    """
+
+    if not name:
+        try:
+            name = select_configuration_questionnaire(
+                type_=InstanceType.SERVER, system_folders=False, is_sandbox=True
+            )
+        except Exception:
+            error("No configurations could be found!")
+            exit()
+
+    ctx = get_context(InstanceType.SERVER, name, system_folders=False, is_sandbox=True)
+
+    # remove the store folder
+    store_configs = AlgorithmStoreContext.instance_folders(
+        InstanceType.ALGORITHM_STORE,
+        f"{ctx.name}-store",
+        system_folders=False,
+    )
+    store_folder = store_configs["data"]
+    if store_folder.is_dir():
+        rmtree(store_folder)
+
+    # remove the store config file
+    AlgorithmStoreContext.LOGGING_ENABLED = False
+    store_ctx = AlgorithmStoreContext(
+        instance_name=f"{ctx.name}-store",
+        system_folders=False,
+        is_sandbox=True,
+    )
+    execute_remove(
+        store_ctx,
+        InstanceType.ALGORITHM_STORE,
+        InfraComponentName.ALGORITHM_STORE,
+        f"{ctx.name}-store",
+        system_folders=False,
+        force=True,
+    )
+
+    # remove the auth folder
+    AuthContext.LOGGING_ENABLED = False
+    auth_configs = AuthContext.instance_folders(
+        InstanceType.AUTH, f"{ctx.name}-auth", system_folders=False
+    )
+    auth_folder = auth_configs["data"]
+    if auth_folder.is_dir():
+        rmtree(auth_folder)
+
+    # remove the auth service
+    auth_ctx = AuthContext(
+        instance_name=f"{ctx.name}-auth",
+        system_folders=False,
+        is_sandbox=True,
+    )
+    auth_remove(auth_ctx, f"{ctx.name}-auth", system_folders=False, force=True)
+
+    # remove the nodes
+    NodeContext.LOGGING_ENABLED = False
+    configs, _ = NodeContext.available_configurations(system_folders=False)
+    node_names = [
+        config.name for config in configs if config.name.startswith(f"{ctx.name}-node-")
+    ]
+    for name in node_names:
+        # Context clases are singletons, so we need to clear the cache to force
+        # creation of a new instance. Otherwise, we *think* we get the ctx of another
+        # node but we actually get the one from the previous node in the loop.
+        if NodeContext in NodeContext._instances:
+            del NodeContext._instances[NodeContext]
+
+        node_ctx = NodeContext(
+            instance_name=name,
+            system_folders=False,
+            is_sandbox=True,
+            print_log_header=False,
+            logger_prefix="",
+            in_container=False,
+        )
+        for handler in itertools.chain(
+            node_ctx.log.handlers, node_ctx.log.root.handlers
+        ):
+            handler.close()
+        execute_remove(
+            node_ctx,
+            InstanceType.NODE,
+            InfraComponentName.NODE,
+            name,
+            system_folders=False,
+            force=True,
+        )
+
+    # remove data files attached to the network
+    data_dirs_nodes = NodeContext.instance_folders("node", "", False)["dev"]
+    try:
+        rmtree(Path(data_dirs_nodes / ctx.name))
+    except Exception as e:
+        warning(f"Failed to delete data directory {data_dirs_nodes / ctx.name}: {e}")
+
+    # remove the server last - if anything goes wrong, the server is still there so the
+    # user can still retry the removal.
+    # Note that this also checks if the server is running. Therefore, it is prevented
+    # that a running sandbox is removed.
+    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
+        handler.close()
+    click_ctx.invoke(
+        cli_server_remove, ctx=ctx, name=name, system_folders=False, force=True
+    )
+    # TODO remove the right data in the custom data directory if it is provided