vantage6 4.5.4__tar.gz → 4.6.0rc3__tar.gz

{vantage6-4.5.4 → vantage6-4.6.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vantage6
-Version: 4.5.4
+Version: 4.6.0rc3
 Summary: vantage6 command line interface
 Home-page: https://github.com/vantage6/vantage6
 Requires-Python: >=3.10

{vantage6-4.5.4 → vantage6-4.6.0rc3}/tests_cli/test_node_cli.py

@@ -207,7 +207,7 @@ class NodeCLITest(unittest.TestCase):
         self.assertNotEqual(result.exit_code, 0)
 
     @patch("docker.DockerClient.volumes")
-    @patch("vantage6.cli.node.start.pull_image")
+    @patch("vantage6.cli.node.start.pull_infra_image")
     @patch("vantage6.cli.common.decorator.get_context")
     @patch("docker.DockerClient.containers")
     @patch("vantage6.cli.node.start.check_docker_running", return_value=True)

{vantage6-4.5.4 → vantage6-4.6.0rc3}/tests_cli/test_server_cli.py

@@ -18,7 +18,7 @@ class ServerCLITest(unittest.TestCase):
     @patch("vantage6.cli.server.start.NetworkManager")
     @patch("vantage6.cli.server.start.docker.types.Mount")
     @patch("os.makedirs")
-    @patch("vantage6.cli.server.start.pull_image")
+    @patch("vantage6.cli.server.start.pull_infra_image")
     @patch("vantage6.cli.common.decorator.get_context")
     @patch("vantage6.cli.server.start.docker.from_env")
     @patch("vantage6.cli.common.start.check_docker_running", return_value=True)

{vantage6-4.5.4 → vantage6-4.6.0rc3}/tests_cli/test_wizard.py

@@ -9,7 +9,7 @@ from vantage6.cli.configuration_wizard import (
     configuration_wizard,
     select_configuration_questionaire,
 )
-from vantage6.common.globals import InstanceType
+from vantage6.common.globals import InstanceType, NodePolicy
 
 module_path = "vantage6.cli.configuration_wizard"
 
@@ -39,8 +39,14 @@ class WizardTest(unittest.TestCase):
                 False,  # don't enable two-factor authentication
                 True,  # add VPN server
                 True,  # add algorithm policies
+                True,  # add single algorithms to allowed_algorithms
                 "some-image",  # algorithm image to whitelist
                 False,  # don't add another algorithm image
+                True,  # add algorithm stores to allowed_algorithm_stores
+                "some-store",  # algorithm store to whitelist
+                False,  # don't add another algorithm store
+                False,  # answer question on combining policies on store level and
+                # single algorithm level
                 False,  # don't abort if no server connection is made to pull
                 # collaboration settings
                 True,  # Enable encryption
@@ -61,7 +67,10 @@ class WizardTest(unittest.TestCase):
         ]
         for key in keys:
             self.assertIn(key, config)
-        nested_keys = [["policies", "allowed_algorithms"]]
+        nested_keys = [
+            ["policies", NodePolicy.ALLOWED_ALGORITHMS],
+            ["policies", NodePolicy.ALLOWED_ALGORITHM_STORES],
+        ]
         for nesting in nested_keys:
             current_config = config
             for key in nesting:

vantage6-4.6.0rc3/vantage6/cli/__build__

@@ -0,0 +1 @@
+3

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/_version.py

@@ -7,7 +7,7 @@ with open(os.path.join(here, "__build__")) as fp:
     __build__ = json.load(fp)
 
 # Module version
-version_info = (4, 5, 4, "final", __build__, 0)
+version_info = (4, 6, 0, "candidate", __build__, 0)
 
 # Module version stage suffix map
 _specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/algostore/start.py

@@ -2,7 +2,6 @@ import click
 
 from vantage6.common import info
 from vantage6.common.globals import APPNAME, DEFAULT_ALGO_STORE_IMAGE, InstanceType
-from vantage6.common.docker.addons import pull_image
 from vantage6.cli.common.start import (
     attach_logs,
     check_for_start,
@@ -10,6 +9,7 @@ from vantage6.cli.common.start import (
     mount_config_file,
     mount_database,
     mount_source,
+    pull_infra_image,
 )
 from vantage6.cli.globals import AlgoStoreGlobals
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
@@ -55,7 +55,7 @@ def cli_algo_store_start(
     image = get_image(image, ctx, "algorithm-store", DEFAULT_ALGO_STORE_IMAGE)
 
     info("Pulling algorithm store image...")
-    pull_image(docker_client, image)
+    pull_infra_image(docker_client, image, InstanceType.ALGORITHM_STORE)
 
     config_file = "/mnt/config.yaml"
     mounts = mount_config_file(ctx, config_file)

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/common/start.py

@@ -10,8 +10,16 @@ from sqlalchemy.engine.url import make_url
 
 from vantage6.common import error, info, warning
 from vantage6.common.context import AppContext
-from vantage6.common.globals import InstanceType, APPNAME, DEFAULT_DOCKER_REGISTRY
-from vantage6.common.docker.addons import check_docker_running
+from vantage6.common.globals import (
+    DEFAULT_ALGO_STORE_IMAGE,
+    DEFAULT_NODE_IMAGE,
+    DEFAULT_SERVER_IMAGE,
+    DEFAULT_UI_IMAGE,
+    InstanceType,
+    APPNAME,
+    DEFAULT_DOCKER_REGISTRY,
+)
+from vantage6.common.docker.addons import check_docker_running, pull_image
 from vantage6.cli.context import AlgorithmStoreContext, ServerContext
 from vantage6.cli.common.utils import print_log_worker
 from vantage6.cli.utils import check_config_name_allowed
@@ -88,6 +96,88 @@ def get_image(
     return image
 
 
+def pull_infra_image(
+    client: DockerClient, image: str, instance_type: InstanceType
+) -> None:
+    """
+    Try to pull an infrastructure image. If the image is a default infrastructure image,
+    exit if in cannot be pulled. If it is not a default image, exit if it cannot be
+    pulled and it is also not available locally. If a local image is available, a
+    warning is printed.
+
+    Parameters
+    ----------
+    client : DockerClient
+        The Docker client
+    image : str
+        The image name to pull
+    instance_type : InstanceType
+        The type of instance to pull the image for
+    """
+    try:
+        pull_image(client, image, suppress_error=True)
+    except docker.errors.APIError:
+        if not is_default_infra_image(image, instance_type):
+            if image_exists_locally(client, image):
+                warning("Failed to pull infrastructure image! Will use local image...")
+            else:
+                error("Failed to pull infrastructure image!")
+                error("Image also not found locally. Exiting...")
+                exit(1)
+        else:
+            error("Failed to pull infrastructure image! Exiting...")
+            exit(1)
+
+
+def is_default_infra_image(image: str, instance_type: InstanceType) -> bool:
+    """
+    Check if an infrastructure image is the default image.
+
+    Parameters
+    ----------
+    image : str
+        The image name to check
+    instance_type : InstanceType
+        The type of instance to check the image for
+
+    Returns
+    -------
+    bool
+        True if the image is the default image, False otherwise
+    """
+    if instance_type == InstanceType.SERVER:
+        return image == f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_SERVER_IMAGE}"
+    elif instance_type == InstanceType.ALGORITHM_STORE:
+        return image == f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_ALGO_STORE_IMAGE}"
+    elif instance_type == InstanceType.UI:
+        return image == f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_UI_IMAGE}"
+    elif instance_type == InstanceType.NODE:
+        return image == f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
+
+
+def image_exists_locally(client: DockerClient, image: str) -> bool:
+    """
+    Check if the image exists locally.
+
+    Parameters
+    ----------
+    client : DockerClient
+        The Docker client
+    image : str
+        The image name to check
+
+    Returns
+    -------
+    bool
+        True if the image exists locally, False otherwise
+    """
+    try:
+        client.images.get(image)
+    except docker.errors.ImageNotFound:
+        return False
+    return True
+
+
 def mount_config_file(ctx: AppContext, config_file: str) -> list[docker.types.Mount]:
     """
     Mount the config file in the container.

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/configuration_wizard.py

@@ -3,7 +3,7 @@ import questionary as q
 from pathlib import Path
 
 from vantage6.common import generate_apikey
-from vantage6.common.globals import DATABASE_TYPES, InstanceType
+from vantage6.common.globals import DATABASE_TYPES, InstanceType, NodePolicy
 from vantage6.common.client.node_client import NodeClient
 from vantage6.common.context import AppContext
 from vantage6.common import error, warning, info
@@ -82,11 +82,6 @@ def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
             {"label": db_label.get("label"), "uri": db_path.get("uri"), "type": db_type}
         )
 
-    res = q.select(
-        "Which level of logging would you like?",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "NOTSET"],
-    ).ask()
-
     is_add_vpn = q.confirm(
         "Do you want to connect to a VPN server?", default=False
     ).ask()
@@ -101,33 +96,41 @@ def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
         "should always be done for production scenarios.",
         default=True,
     ).ask()
+    policies = {}
     if is_policies:
-        allowed_algorithms = []
-        info("Below you can add algorithms that are allowed to run on your node.")
-        info(
-            "You can use regular expressions to match multiple algorithms, or you can "
-            "use strings to provide one algorithm at a time."
-        )
-        info("Examples:")
-        # pylint: disable=W1401
-        # flake8: noqa: W605
-        info("^harbor2\.vantage6\.ai/demo/average$    Allow the demo average algorithm")
-        info(
-            "^harbor2\.vantage6\.ai/algorithms/.*   Allow all algorithms from "
-            "harbor2.vantage6.ai/algorithms"
-        )
         info(
-            "^harbor2\.vantage6\.ai/demo/average:@sha256:82becede...$    Allow a "
-            "specific hash of average algorithm"
+            "You can limit the algorithms that can run on your node in two ways: by "
+            "allowing specific algorithms or by allowing all algorithms in a given "
+            "algorithm store."
         )
-        while True:
-            algo = q.text(message="Enter your algorithm expression:").ask()
-            allowed_algorithms.append(algo)
-            if not q.confirm(
-                "Do you want to add another algorithm expression?", default=True
-            ).ask():
-                break
-        config["policies"] = {"allowed_algorithms": allowed_algorithms}
+        ask_single_algorithms = q.confirm(
+            "Do you want to enter a list of allowed algorithms?"
+        ).ask()
+        if ask_single_algorithms:
+            policies[NodePolicy.ALLOWED_ALGORITHMS] = _get_allowed_algorithms()
+        ask_algorithm_stores = q.confirm(
+            "Do you want to allow algorithms from specific algorithm stores?"
+        ).ask()
+        if ask_algorithm_stores:
+            policies[NodePolicy.ALLOWED_ALGORITHM_STORES] = (
+                _get_allowed_algorithm_stores()
+            )
+        if ask_single_algorithms and ask_algorithm_stores:
+            require_both_whitelists = q.confirm(
+                "Do you want to allow only algorithms that are both in the list of "
+                "allowed algorithms *AND* are part of one of the allowed algorithm "
+                "stores? If not, algorithms will be allowed if they are in either the "
+                "list of allowed algorithms or one of the allowed algorithm stores.",
+                default=True,
+            ).ask()
+            policies["allow_either_whitelist_or_store"] = not require_both_whitelists
+    if policies:
+        config["policies"] = policies
+
+    res = q.select(
+        "Which level of logging would you like?",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "NOTSET"],
+    ).ask()
 
     config["logging"] = {
         "level": res,
@@ -182,6 +185,77 @@ def node_configuration_questionaire(dirs: dict, instance_name: str) -> dict:
     return config
 
 
+def _get_allowed_algorithms() -> list[str]:
+    """
+    Prompt the user for the allowed algorithms on their node
+
+    Returns
+    -------
+    list[str]
+        List of allowed algorithms or regular expressions to match them
+    """
+    info("Below you can add algorithms that are allowed to run on your node.")
+    info(
+        "You can use regular expressions to match multiple algorithms, or you can "
+        "use strings to provide one algorithm at a time."
+    )
+    info("Examples:")
+    # pylint: disable=W1401
+    # flake8: noqa: W605
+    info("^harbor2\.vantage6\.ai/demo/average$    Allow the demo average algorithm")
+    info(
+        "^harbor2\.vantage6\.ai/algorithms/.*   Allow all algorithms from "
+        "harbor2.vantage6.ai/algorithms"
+    )
+    info(
+        "^harbor2\.vantage6\.ai/demo/average@sha256:82becede...$    Allow a "
+        "specific hash of average algorithm"
+    )
+    allowed_algorithms = []
+    while True:
+        algo = q.text(message="Enter your algorithm expression:").ask()
+        allowed_algorithms.append(algo)
+        if not q.confirm(
+            "Do you want to add another algorithm expression?", default=True
+        ).ask():
+            break
+    return allowed_algorithms
+
+
+def _get_allowed_algorithm_stores() -> list[str]:
+    """
+    Prompt the user for the allowed algorithm stores on their node
+
+    Returns
+    -------
+    list[str]
+        List of allowed algorithm stores
+    """
+    info("Below you can add algorithm stores that are allowed to run on your node.")
+    info(
+        "You can use regular expressions to match multiple algorithm stores, or you can"
+        " use strings to provide one algorithm store at a time."
+    )
+    info("Examples:")
+    info(
+        "https://store.cotopaxi.vantage6.ai    Allow all algorithms from the "
+        "community store"
+    )
+    info(
+        "^https://*\.vantage6\.ai$               Allow all algorithms from any "
+        "store hosted on vantage6.ai"
+    )
+    allowed_algorithm_stores = []
+    while True:
+        store = q.text(message="Enter the URL of the algorithm store:").ask()
+        allowed_algorithm_stores.append(store)
+        if not q.confirm(
+            "Do you want to add another algorithm store?", default=True
+        ).ask():
+            break
+    return allowed_algorithm_stores
+
+
 def _get_common_server_config(
     instance_type: InstanceType, instance_name: str, include_api_path: bool = True
 ) -> dict:
@@ -435,13 +509,14 @@ def algo_store_configuration_questionaire(instance_name: str) -> dict:
     }
 
     # ask about openness of the algorithm store
+    config["policies"] = {"allow_localhost": False}
     is_open = q.confirm(
         "Do you want to open the algorithm store to the public? This will allow anyone "
         "to view the algorithms, but they cannot modify them.",
         default=False,
     ).ask()
     if is_open:
-        config["policies"] = {"algorithms_open": True}
+        open_algos_policy = "public"
     else:
         is_open_to_whitelist = q.confirm(
             "Do you want to allow all users of whitelisted vantage6 servers to access "
@@ -449,10 +524,8 @@ def algo_store_configuration_questionaire(instance_name: str) -> dict:
             " the appropriate permissions to each user individually.",
             default=True,
         ).ask()
-        config["policies"] = {
-            "algorithms_open": False,
-            "algorithms_open_to_whitelisted": is_open_to_whitelist,
-        }
+        open_algos_policy = "whitelisted" if is_open_to_whitelist else "private"
+    config["policies"]["algorithm_view"] = open_algos_policy
 
     return config
 

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/node/start.py

@@ -8,8 +8,8 @@ import docker
 
 from colorama import Fore, Style
 
+from vantage6.cli.common.start import pull_infra_image
 from vantage6.common import warning, error, info, debug, get_database_config
-from vantage6.common.docker.addons import pull_image
 from vantage6.common.globals import (
     APPNAME,
     DEFAULT_DOCKER_REGISTRY,
@@ -135,7 +135,7 @@ def cli_node_start(
             image = f"{DEFAULT_DOCKER_REGISTRY}/{DEFAULT_NODE_IMAGE}"
 
     info(f"Pulling latest node image '{image}'")
-    pull_image(docker_client, image)
+    pull_infra_image(docker_client, image, InstanceType.NODE)
 
     data_volume = docker_client.volumes.create(ctx.docker_volume_name)
     vpn_volume = docker_client.volumes.create(ctx.docker_vpn_volume_name)

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6/cli/server/start.py

@@ -4,7 +4,6 @@ from docker.client import DockerClient
 
 from vantage6.common import info, warning, error
 from vantage6.common.docker.network_manager import NetworkManager
-from vantage6.common.docker.addons import pull_image
 from vantage6.common.globals import (
     APPNAME,
     DEFAULT_SERVER_IMAGE,
@@ -23,6 +22,7 @@ from vantage6.cli.common.start import (
     get_image,
     mount_database,
     mount_source,
+    pull_infra_image,
 )
 
 
@@ -91,7 +91,7 @@ def cli_server_start(
     ctx.log_dir.mkdir(parents=True, exist_ok=True)
 
     info("Pulling server image...")
-    pull_image(docker_client, image)
+    pull_infra_image(docker_client, image, InstanceType.SERVER)
 
     info("Creating mounts")
     config_file = "/mnt/config.yaml"
@@ -229,7 +229,7 @@ def _start_ui(client: DockerClient, ctx: ServerContext, ui_port: int) -> None:
     # find image to use
     image = get_image(None, ctx, "ui", DEFAULT_UI_IMAGE)
 
-    pull_image(client, image)
+    pull_infra_image(client, image, InstanceType.UI)
 
     # set environment variables
     env_vars = {

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vantage6
-Version: 4.5.4
+Version: 4.6.0rc3
 Summary: vantage6 command line interface
 Home-page: https://github.com/vantage6/vantage6
 Requires-Python: >=3.10

{vantage6-4.5.4 → vantage6-4.6.0rc3}/vantage6.egg-info/requires.txt

@@ -8,8 +8,8 @@ questionary==1.10.0
 rich==13.5.2
 schema==0.7.5
 SQLAlchemy==1.4.46
-vantage6-common==4.5.4
-vantage6-client==4.5.4
+vantage6-common==4.6.0rc3
+vantage6-client==4.6.0rc3
 
 [dev]
 coverage==6.4.4

vantage6-4.5.4/vantage6/cli/__build__

@@ -1 +0,0 @@
-0