vantage6 4.6.1__py3-none-any.whl → 4.7.0rc1__py3-none-any.whl

vantage6/cli/dev/remove.py

@@ -5,24 +5,34 @@ from pathlib import Path
 
 import click
 
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
 from vantage6.common import info
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.server.remove import cli_server_remove
 from vantage6.cli.utils import remove_file
 from vantage6.common.globals import InstanceType
+from vantage6.cli.dev.utils import get_dev_server_context
 
 
 @click.command()
-@click_insert_context(type_="server")
+@click.option("-n", "--name", default=None, help="Name of the configuration.")
+@click.option(
+    "-c",
+    "--config",
+    default=None,
+    help="Path to configuration-file; overrides --name",
+)
 @click.pass_context
-def remove_demo_network(click_ctx: click.Context, ctx: ServerContext) -> None:
+def remove_demo_network(
+    click_ctx: click.Context, name: str | None, config: str | None
+) -> None:
     """Remove all related demo network files and folders.
 
     Select a server configuration to remove that server and the nodes attached
     to it.
     """
+    ctx = get_dev_server_context(config, name)
 
     # remove the server
     for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
@@ -39,16 +49,37 @@ def remove_demo_network(click_ctx: click.Context, ctx: ServerContext) -> None:
 
     # also remove the server folder
     server_configs = ServerContext.instance_folders(
-        InstanceType.SERVER, ctx.name, system_folders=True
+        InstanceType.SERVER, ctx.name, system_folders=False
     )
     server_folder = server_configs["data"]
     if server_folder.is_dir():
         rmtree(server_folder)
 
+    # remove the store folder
+    store_configs = AlgorithmStoreContext.instance_folders(
+        InstanceType.ALGORITHM_STORE, f"{ctx.name}_store", system_folders=False
+    )
+    store_folder = store_configs["data"]
+    if store_folder.is_dir():
+        rmtree(store_folder)
+
+    # remove the store config file
+    subprocess.run(
+        [
+            "v6",
+            "algorithm-store",
+            "remove",
+            "-n",
+            f"{ctx.name}_store",
+            "--force",
+            "--user",
+        ]
+    )
+
     # remove the nodes
     configs, _ = NodeContext.available_configurations(system_folders=False)
     node_names = [
-        config.name for config in configs if f"{ctx.name}_node_" in config.name
+        config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
     ]
     for name in node_names:
         node_ctx = NodeContext(name, False)

vantage6/cli/dev/start.py

@@ -1,52 +1,121 @@
 import subprocess
 import click
 
-from vantage6.cli.context.server import ServerContext
+from vantage6.common import info
+from vantage6.client import Client
+from vantage6.cli.globals import COMMUNITY_STORE
+from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.server.start import cli_server_start
+from vantage6.cli.dev.utils import get_dev_server_context
 
 
 @click.command()
-@click_insert_context(type_="server")
+@click.option("-n", "--name", default=None, help="Name of the configuration.")
+@click.option(
+    "-c",
+    "--config",
+    default=None,
+    help="Path to configuration-file; overrides --name",
+)
 @click.option(
     "--server-image", type=str, default=None, help="Server Docker image to use"
 )
 @click.option("--node-image", type=str, default=None, help="Node Docker image to use")
+@click.option(
+    "--store-image", type=str, default=None, help="Algorithm Store Docker image to use"
+)
 @click.pass_context
 def start_demo_network(
-    click_ctx: click.Context, ctx: ServerContext, server_image: str, node_image: str
+    click_ctx: click.Context,
+    name: str | None,
+    config: str | None,
+    server_image: str | None,
+    node_image: str | None,
+    store_image: str | None,
 ) -> None:
     """Starts running a demo-network.
 
     Select a server configuration to run its demo network. You should choose a
     server configuration that you created earlier for a demo network. If you
-    have not created a demo network, you can run `vdev create-demo-network` to
+    have not created a demo network, you can run `v6 dev create-demo-network` to
     create one.
     """
+    ctx = get_dev_server_context(config, name)
+
     # run the server
+    info("Starting server...")
     click_ctx.invoke(
         cli_server_start,
         ctx=ctx,
         ip=None,
         port=None,
         image=server_image,
-        start_ui=False,
+        start_ui=True,
         ui_port=None,
         start_rabbitmq=False,
         rabbitmq_image=None,
         keep=True,
         mount_src="",
         attach=False,
+        system_folders=False,
     )
 
+    # run the store
+    info("Starting algorithm store...")
+    cmd = ["v6", "algorithm-store", "start", "--name", f"{ctx.name}_store", "--user"]
+    if store_image:
+        cmd.extend(["--image", store_image])
+    subprocess.run(cmd)
+
     # run all nodes that belong to this server
     configs, _ = NodeContext.available_configurations(system_folders=False)
     node_names = [
-        config.name for config in configs if f"{ctx.name}_node_" in config.name
+        config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
     ]
     for name in node_names:
         cmd = ["v6", "node", "start", "--name", name]
         if node_image:
             cmd.extend(["--image", node_image])
         subprocess.run(cmd)
+
+    # now that both server and store have been started, couple them
+    info("Linking local algorithm store to server...")
+    store_ctxs, _ = AlgorithmStoreContext.available_configurations(system_folders=False)
+    store_ctx = [c for c in store_ctxs if c.name == f"{ctx.name}_store"][0]
+    client = Client(
+        "http://localhost",
+        ctx.config["port"],
+        ctx.config["api_path"],
+        log_level="warn",
+    )
+    # TODO these credentials are hardcoded and may change if changed elsewhere. Link
+    # them together so that they are guaranteed to be the same.
+    USERNAME = "dev_admin"
+    PASSWORD = "password"
+    client.authenticate(USERNAME, PASSWORD)
+    existing_stores = client.store.list().get("data", [])
+    existing_urls = [store["url"] for store in existing_stores]
+    local_store_url = f"http://localhost:{store_ctx.config['port']}"
+    if not local_store_url in existing_urls:
+        client.store.create(
+            algorithm_store_url=local_store_url,
+            name="local store",
+            all_collaborations=True,
+            force=True,  # required to link localhost store
+        )
+        # note that we do not need to register the user as root of the store: this is
+        # already handled in the store config file and is executed on store startup (and
+        # successful because server is already started up at that point)
+    info("Done!")
+
+    # link the community store also to the server
+    info("Linking community algorithm store to local server...")
+    if not COMMUNITY_STORE in existing_urls:
+        client.store.create(
+            algorithm_store_url=COMMUNITY_STORE,
+            name="Community store (read-only)",
+            all_collaborations=True,
+            force=True,  # required to continue when linking localhost server
+        )
+    info("Done!")

vantage6/cli/dev/stop.py

@@ -1,30 +1,45 @@
 import click
 
-from vantage6.cli.context.server import ServerContext
 from vantage6.cli.context.node import NodeContext
-from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.server.stop import cli_server_stop
 from vantage6.cli.node.stop import cli_node_stop
+from vantage6.cli.algostore.stop import cli_algo_store_stop
+from vantage6.cli.dev.utils import get_dev_server_context
 
 
 @click.command()
-@click_insert_context(type_="server")
+@click.option("-n", "--name", default=None, help="Name of the configuration.")
+@click.option(
+    "-c",
+    "--config",
+    default=None,
+    help="Path to configuration-file; overrides --name",
+)
 @click.pass_context
-def stop_demo_network(click_ctx: click.Context, ctx: ServerContext) -> None:
+def stop_demo_network(
+    click_ctx: click.Context, name: str | None, config: str | None
+) -> None:
     """Stops a demo network's server and nodes.
 
     Select a server configuration to stop that server and the nodes attached
     to it.
     """
+    ctx = get_dev_server_context(config, name)
+
     # stop the server
     click_ctx.invoke(
-        cli_server_stop, name=ctx.name, system_folders=True, all_servers=False
+        cli_server_stop, name=ctx.name, system_folders=False, all_servers=False
+    )
+
+    # stop the algorithm store
+    click_ctx.invoke(
+        cli_algo_store_stop, name=f"{ctx.name}_store", system_folders=False
     )
 
     # stop the nodes
     configs, _ = NodeContext.available_configurations(False)
     node_names = [
-        config.name for config in configs if f"{ctx.name}_node_" in config.name
+        config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
     ]
     for name in node_names:
         click_ctx.invoke(

vantage6/cli/dev/utils.py

@@ -0,0 +1,24 @@
+from vantage6.common.globals import InstanceType
+from vantage6.cli.configuration_wizard import select_configuration_questionaire
+from vantage6.cli.context import get_context
+from vantage6.cli.context.server import ServerContext
+
+
+def get_dev_server_context(config: str | None, name: str | None) -> ServerContext:
+    """
+    Get the server context for the development server.
+
+    Parameters
+    ----------
+    config : str | None
+        Path to the configuration file. If None, the name will be used.
+    name : str | None
+        Name of the configuration. If None, a questionaire will be shown.
+    """
+    if config:
+        return ServerContext.from_external_config_file(config)
+    if not name:
+        name = select_configuration_questionaire(
+            InstanceType.SERVER, system_folders=False
+        )
+    return get_context(InstanceType.SERVER, name, system_folders=False)

vantage6/cli/globals.py

@@ -42,6 +42,9 @@ ALGORITHM_TEMPLATE_REPO = "gh:vantage6/v6-algorithm-template.git"
 # image to use for diagnostics in `v6 test` commands
 DIAGNOSTICS_IMAGE = "harbor2.vantage6.ai/algorithms/diagnostic"
 
+# Address of community algorithm store
+COMMUNITY_STORE = "https://store.cotopaxi.vantage6.ai"
+
 
 class ServerType(str, Enum):
     """Enum containing server types"""

vantage6/cli/node/remove.py

@@ -99,13 +99,12 @@ def cli_node_remove(name: str, system_folders: bool, force: bool) -> None:
 
     # remove the log file. As this process opens the log file above, the log
     # handlers need to be closed before deleting
-    info(f"Removing log file {ctx.log_file}")
+    log_dir = Path(ctx.log_file.parent)
+    info(f"Removing log file {log_dir}")
     for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
         handler.close()
-    # remove_file(ctx.log_file, 'log')
-
-    # removes the whole folder
-    rmtree(Path(ctx.log_file.parent))
+    # remove the whole folder with all the log files
+    rmtree(log_dir)
 
     # remove the folder: if it hasn't been started yet this won't exist...
     if Path.exists(ctx.config_dir / name):

vantage6/cli/node/start.py

@@ -309,7 +309,11 @@ def cli_node_start(
         extra_hosts=extra_hosts,
     )
 
-    info("Node container was successfully started!")
+    info("Node container was started!")
+    info(
+        "Please check the node logs to see if the node successfully connects to the "
+        "server."
+    )
 
     if attach:
         logs = container.attach(stream=True, logs=True)

vantage6/cli/rabbitmq/__init__.py

@@ -1,28 +0,0 @@
-"""RabbitMQ utilities."""
-
-
-def split_rabbitmq_uri(rabbit_uri: str) -> dict:
-    """
-    Get details (user, pass, host, vhost, port) from a RabbitMQ uri.
-
-    Parameters
-    ----------
-    rabbit_uri: str
-        URI of RabbitMQ service ('amqp://$user:$pass@$host:$port/$vhost')
-
-    Returns
-    -------
-    dict[str]
-        The vhost defined in the RabbitMQ URI
-    """
-    (user_details, location_details) = rabbit_uri.split("@", 1)
-    (user, password) = user_details.split("/")[-1].split(":", 1)
-    (host, remainder) = location_details.split(":", 1)
-    port, vhost = remainder.split("/", 1)
-    return {
-        "user": user,
-        "password": password,
-        "host": host,
-        "port": port,
-        "vhost": vhost,
-    }

vantage6/cli/rabbitmq/queue_manager.py

@@ -12,10 +12,10 @@ from vantage6.common.globals import APPNAME
 from vantage6.common import debug, info, error
 from vantage6.common.docker.addons import get_container
 from vantage6.common.docker.network_manager import NetworkManager
+from vantage6.common import split_rabbitmq_uri
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.rabbitmq.definitions import RABBITMQ_DEFINITIONS
 from vantage6.cli.globals import RABBIT_TIMEOUT
-from vantage6.cli.rabbitmq import split_rabbitmq_uri
 
 DEFAULT_RABBIT_IMAGE = "harbor2.vantage6.ai/infrastructure/rabbitmq"
 RABBIT_CONFIG = "rabbitmq.config"

vantage6/cli/server/import_.py

@@ -47,7 +47,7 @@ def cli_server_import(
     ctx: ServerContext,
     file: str,
     drop_all: bool,
-    image: str,
+    image: str | None,
     mount_src: str,
     keep: bool,
     wait: bool,
@@ -122,7 +122,7 @@ def cli_server_import(
         info("Consider using the docker-compose method to start a server")
 
     drop_all_ = "--drop-all" if drop_all else ""
-    cmd = f"vserver-local import -c /mnt/config.yaml {drop_all_} " "/mnt/import.yaml"
+    cmd = f"vserver-local import -c /mnt/config.yaml {drop_all_} /mnt/import.yaml"
 
     info(cmd)
 

vantage6/cli/server/remove.py

@@ -1,16 +1,20 @@
 import itertools
+from pathlib import Path
+from shutil import rmtree
+
 import click
 import questionary as q
 
 from vantage6.common import info
 from vantage6.common.docker.addons import check_docker_running
+from vantage6.common.globals import InstanceType
 from vantage6.cli.common.decorator import click_insert_context
 from vantage6.cli.context import ServerContext
 from vantage6.cli.utils import remove_file
 
 
 @click.command()
-@click_insert_context(type_="server")
+@click_insert_context(type_=InstanceType.SERVER)
 @click.option("-f", "--force", "force", flag_value=True)
 def cli_server_remove(ctx: ServerContext, force: bool) -> None:
     """
@@ -37,6 +41,12 @@ def cli_server_remove(ctx: ServerContext, force: bool) -> None:
     # now remove the folders...
     remove_file(ctx.config_file, "configuration")
 
-    for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
-        handler.close()
-    remove_file(ctx.log_file, "log")
+    # ensure log files are closed before removing
+    log_dir = Path(ctx.log_file.parent)
+    if log_dir.exists():
+        info(f"Removing log directory: {log_dir}")
+        for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
+            handler.close()
+        # remove the whole folder with all the log files. This may also still contain other
+        # files like RabbitMQ configuration etc
+        rmtree(log_dir)

vantage6/cli/server/stop.py

@@ -16,7 +16,7 @@ from vantage6.common.docker.addons import (
     remove_container_if_exists,
 )
 from vantage6.common.globals import APPNAME, InstanceType
-from vantage6.cli.rabbitmq import split_rabbitmq_uri
+from vantage6.common import split_rabbitmq_uri
 from vantage6.cli.context.server import ServerContext
 from vantage6.cli.globals import DEFAULT_SERVER_SYSTEM_FOLDERS
 from vantage6.cli.server.common import get_server_context, stop_ui

vantage6/cli/template/node_config.j2

@@ -17,6 +17,17 @@ logging:
   level: DEBUG
   max_size: 1024
   use_console: true
+  loggers:
+  - level: warning
+    name: urllib3
+  - level: warning
+    name: requests
+  - level: warning
+    name: engineio.client
+  - level: warning
+    name: docker.utils.config
+  - level: warning
+    name: docker.auth
 port: {{ port }}
 server_url: {{ server_url }}
 task_dir: {{ task_dir}}

vantage6/cli/template/server_config.j2

@@ -10,7 +10,26 @@ logging:
   level: DEBUG
   max_size: 1024
   use_console: true
+  loggers:
+  - level: warning
+    name: urllib3
+  - level: warning
+    name: socketIO-client
+  - level: warning
+    name: socketio.server
+  - level: warning
+    name: engineio.server
+  - level: warning
+    name: sqlalchemy.engine
+  - level: warning
+    name: requests_oauthlib.oauth2_session
 port: {{ port }}
 uri: sqlite:///default.sqlite
 jwt_secret_key: {{ jwt_secret_key }}
+dev:
+  host_uri: {{ host_uri }}
+server_url: http://localhost:{{ port }}/api
+ui:
+  enable: true
+  port: {{ ui_port }}
 {{ user_provided_config }}

vantage6/cli/template/server_import_config.j2

@@ -3,12 +3,12 @@ organizations:
 - name: {{ org['name'] }}
   domain: iknl.nl
   address1: Godebaldkwartier 419
-  address2: 
+  address2:
   zipcode: 3511DT
   country: Netherlands
   {% if org['make_admin'] %}
   users:
-    - username: {{ org['name'] }}-admin
+    - username: dev_admin
       firstname: admin
       lastname: robot
       password: password
@@ -19,7 +19,7 @@ collaborations:
 - name: {{ collaboration['name'] }}
   domain: iknl.nl
   address1: Godebaldkwartier 419
-  address2: 
+  address2:
   zipcode: 3511DT
   country: Netherlands
   participants:
@@ -31,4 +31,3 @@ collaborations:
   {% endif %}
   tasks: ['hello-world']
   encrypted: false
-

vantage6/cli/test/integration_test.py

@@ -100,7 +100,7 @@ def cli_test_integration(
         host="http://localhost",
         port=5000,
         api_path="/api",
-        username="org_1-admin",
+        username="dev_admin",
         password="password",
         collaboration=1,
         organizations=None,

{vantage6-4.6.1.dist-info → vantage6-4.7.0rc1.dist-info}/METADATA

@@ -1,26 +1,26 @@
 Metadata-Version: 2.1
 Name: vantage6
-Version: 4.6.1
+Version: 4.7.0rc1
 Summary: vantage6 command line interface
 Home-page: https://github.com/vantage6/vantage6
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: click ==8.1.3
-Requires-Dist: colorama ==0.4.6
-Requires-Dist: copier ==9.2.0
-Requires-Dist: docker ==7.1.0
-Requires-Dist: ipython ==8.10.0
-Requires-Dist: jinja2 ==3.1.4
-Requires-Dist: questionary ==1.10.0
-Requires-Dist: rich ==13.5.2
-Requires-Dist: schema ==0.7.5
-Requires-Dist: SQLAlchemy ==1.4.46
-Requires-Dist: vantage6-common ==4.6.1
-Requires-Dist: vantage6-client ==4.6.1
+Requires-Dist: click==8.1.3
+Requires-Dist: colorama==0.4.6
+Requires-Dist: copier==9.2.0
+Requires-Dist: docker==7.1.0
+Requires-Dist: ipython==8.10.0
+Requires-Dist: jinja2==3.1.4
+Requires-Dist: questionary==1.10.0
+Requires-Dist: rich==13.5.2
+Requires-Dist: schema==0.7.5
+Requires-Dist: SQLAlchemy==1.4.46
+Requires-Dist: vantage6-common==4.7.0rc1
+Requires-Dist: vantage6-client==4.7.0rc1
 Provides-Extra: dev
-Requires-Dist: coverage ==6.4.4 ; extra == 'dev'
-Requires-Dist: black ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: coverage==6.4.4; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
 
 <h1 align="center">
   <br>
@@ -103,7 +103,7 @@ v6 server attach
 
 From here you can use the [vantage6-client](https://pypi.org/project/vantage6-client) to interact with the server. The demo network has a pre-configured organization with the following credentials:
 
-- Username: `org_1-admin`
+- Username: `dev_admin`
 - Password: `password`
 
 For example, you can create a new organization by running:
@@ -112,7 +112,7 @@ For example, you can create a new organization by running:
 from vantage6.client import Client
 
 client = Client('http://127.0.0.1', 5000, '/api', log_level='debug')
-client.authenticate('org_1-admin', 'password')
+client.authenticate('dev_admin', 'password')
 client.setup_encryption(None)
 
 client.organization.create(