vantage6 5.0.0a14__py3-none-any.whl → 5.0.0a16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of vantage6 might be problematic. Click here for more details.
- vantage6/cli/__build__ +1 -1
- vantage6/cli/configuration_manager.py +0 -2
- vantage6/cli/context/node.py +1 -1
- vantage6/cli/dev/create.py +635 -0
- vantage6/cli/dev/remove.py +111 -0
- vantage6/cli/template/node_config.j2 +0 -7
- {vantage6-5.0.0a14.dist-info → vantage6-5.0.0a16.dist-info}/METADATA +3 -3
- {vantage6-5.0.0a14.dist-info → vantage6-5.0.0a16.dist-info}/RECORD +11 -9
- {vantage6-5.0.0a14.dist-info → vantage6-5.0.0a16.dist-info}/WHEEL +0 -0
- {vantage6-5.0.0a14.dist-info → vantage6-5.0.0a16.dist-info}/entry_points.txt +0 -0
- {vantage6-5.0.0a14.dist-info → vantage6-5.0.0a16.dist-info}/top_level.txt +0 -0
vantage6/cli/__build__
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
16
|
|
@@ -40,12 +40,10 @@ class NodeConfiguration(Configuration):
|
|
|
40
40
|
"""
|
|
41
41
|
|
|
42
42
|
VALIDATORS = {
|
|
43
|
-
"api_key": Use(str),
|
|
44
43
|
"server_url": Use(str),
|
|
45
44
|
"port": Or(Use(int), None),
|
|
46
45
|
"task_dir": Use(str),
|
|
47
46
|
# TODO: remove `dict` validation from databases
|
|
48
|
-
"databases": Or([Use(dict)], dict, None),
|
|
49
47
|
"api_path": Use(str),
|
|
50
48
|
"logging": LOGGING_VALIDATORS,
|
|
51
49
|
"encryption": {"enabled": bool, Optional("private_key"): Use(str)},
|
vantage6/cli/context/node.py
CHANGED
|
@@ -0,0 +1,635 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from importlib import resources as impresources
|
|
3
|
+
import yaml
|
|
4
|
+
import click
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from jinja2 import Environment, FileSystemLoader
|
|
8
|
+
from colorama import Fore, Style
|
|
9
|
+
|
|
10
|
+
from vantage6.common.globals import APPNAME, InstanceType, Ports
|
|
11
|
+
from vantage6.common import ensure_config_dir_writable, info, error, generate_apikey
|
|
12
|
+
|
|
13
|
+
import vantage6.cli.dev.data as data_dir
|
|
14
|
+
from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
|
|
15
|
+
from vantage6.cli.globals import PACKAGE_FOLDER
|
|
16
|
+
from vantage6.cli.context.server import ServerContext
|
|
17
|
+
from vantage6.cli.context.node import NodeContext
|
|
18
|
+
from vantage6.cli.server.common import get_server_context
|
|
19
|
+
from vantage6.cli.server.import_ import cli_server_import
|
|
20
|
+
from vantage6.cli.utils import prompt_config_name
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def create_node_data_files(num_nodes: int, server_name: str) -> list[Path]:
|
|
24
|
+
"""Create data files for nodes.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
num_nodes : int
|
|
29
|
+
Number of nodes to create data files for.
|
|
30
|
+
server_name : str
|
|
31
|
+
Name of the server.
|
|
32
|
+
|
|
33
|
+
Returns
|
|
34
|
+
-------
|
|
35
|
+
list[Path]
|
|
36
|
+
List of paths to the created data files.
|
|
37
|
+
"""
|
|
38
|
+
info(f"Creating data files for {num_nodes} nodes.")
|
|
39
|
+
data_files = []
|
|
40
|
+
full_df = pd.read_csv(impresources.files(data_dir) / "olympic_athletes_2016.csv")
|
|
41
|
+
length_df = len(full_df)
|
|
42
|
+
for i in range(num_nodes):
|
|
43
|
+
node_name = f"{server_name}_node_{i + 1}"
|
|
44
|
+
dev_folder = NodeContext.instance_folders("node", node_name, False)["dev"]
|
|
45
|
+
data_folder = Path(dev_folder / server_name)
|
|
46
|
+
data_folder.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
|
|
48
|
+
# Split the data over the nodes
|
|
49
|
+
start = i * length_df // num_nodes
|
|
50
|
+
end = (i + 1) * length_df // num_nodes
|
|
51
|
+
data = full_df[start:end]
|
|
52
|
+
data_file = data_folder / f"df_{node_name}.csv"
|
|
53
|
+
|
|
54
|
+
# write data to file
|
|
55
|
+
data.to_csv(data_file, index=False)
|
|
56
|
+
data_files.append(data_file)
|
|
57
|
+
return data_files
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def create_node_config_file(
|
|
61
|
+
server_url: str, port: int, config: dict, server_name: str, datafile: Path
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Create a node configuration file (YAML).
|
|
64
|
+
|
|
65
|
+
Creates a node configuration for a simulated organization. Organization ID
|
|
66
|
+
is used for generating both the organization name and node_name as each
|
|
67
|
+
organization only houses one node.
|
|
68
|
+
|
|
69
|
+
Parameters
|
|
70
|
+
----------
|
|
71
|
+
server_url : str
|
|
72
|
+
Url of the dummy server.
|
|
73
|
+
port : int
|
|
74
|
+
Port of the dummy server.
|
|
75
|
+
config : dict
|
|
76
|
+
Configuration dictionary containing org_id, api_key, node name and
|
|
77
|
+
additional user_defined_config.
|
|
78
|
+
server_name : str
|
|
79
|
+
Configuration name of the dummy server.
|
|
80
|
+
datafile : Path
|
|
81
|
+
Path to the data file for the node to use.
|
|
82
|
+
"""
|
|
83
|
+
environment = Environment(
|
|
84
|
+
loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
|
|
85
|
+
trim_blocks=True,
|
|
86
|
+
lstrip_blocks=True,
|
|
87
|
+
autoescape=True,
|
|
88
|
+
)
|
|
89
|
+
template = environment.get_template("node_config.j2")
|
|
90
|
+
|
|
91
|
+
# TODO: make this name specific to the server it connects
|
|
92
|
+
node_name = config["node_name"]
|
|
93
|
+
folders = NodeContext.instance_folders("node", node_name, False)
|
|
94
|
+
path_to_dev_dir = Path(folders["dev"] / server_name)
|
|
95
|
+
path_to_dev_dir.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
|
|
97
|
+
path_to_data_dir = Path(folders["data"])
|
|
98
|
+
path_to_data_dir.mkdir(parents=True, exist_ok=True)
|
|
99
|
+
full_path = Path(folders["config"] / f"{node_name}.yaml")
|
|
100
|
+
|
|
101
|
+
if full_path.exists():
|
|
102
|
+
error(f"Node configuration file already exists: {full_path}")
|
|
103
|
+
exit(1)
|
|
104
|
+
|
|
105
|
+
node_config = template.render(
|
|
106
|
+
{
|
|
107
|
+
"api_key": config["api_key"],
|
|
108
|
+
"databases": {"default": datafile},
|
|
109
|
+
"logging": {"file": f"{node_name}.log"},
|
|
110
|
+
"port": port,
|
|
111
|
+
"server_url": server_url,
|
|
112
|
+
"task_dir": str(path_to_data_dir),
|
|
113
|
+
"user_provided_config": config["user_defined_config"],
|
|
114
|
+
}
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# Check that we can write the node config
|
|
118
|
+
if not ensure_config_dir_writable():
|
|
119
|
+
error("Cannot write configuration file. Exiting...")
|
|
120
|
+
exit(1)
|
|
121
|
+
|
|
122
|
+
Path(full_path).parent.mkdir(parents=True, exist_ok=True)
|
|
123
|
+
with open(full_path, "x", encoding="utf-8") as f:
|
|
124
|
+
f.write(node_config)
|
|
125
|
+
|
|
126
|
+
info(
|
|
127
|
+
f"Spawned node for organization {Fore.GREEN}{config['org_id']}"
|
|
128
|
+
f"{Style.RESET_ALL}"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _read_extra_config_file(extra_config_file: Path | None) -> str:
|
|
133
|
+
"""Reads extra configuration file.
|
|
134
|
+
|
|
135
|
+
Parameters
|
|
136
|
+
----------
|
|
137
|
+
extra_config_file : Path | None
|
|
138
|
+
Path to file with additional configuration.
|
|
139
|
+
|
|
140
|
+
Returns
|
|
141
|
+
-------
|
|
142
|
+
str
|
|
143
|
+
Extra configuration file content
|
|
144
|
+
"""
|
|
145
|
+
if extra_config_file:
|
|
146
|
+
# read the YAML file as string, so it can be appended to the
|
|
147
|
+
# configuration easily
|
|
148
|
+
with open(extra_config_file, "r", encoding="utf-8") as f:
|
|
149
|
+
return f.read()
|
|
150
|
+
return ""
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def generate_node_configs(
|
|
154
|
+
num_nodes: int,
|
|
155
|
+
server_url: str,
|
|
156
|
+
port: int,
|
|
157
|
+
server_name: str,
|
|
158
|
+
extra_node_config: Path | None,
|
|
159
|
+
) -> list[dict]:
|
|
160
|
+
"""Generates ``num_nodes`` node configuration files.
|
|
161
|
+
|
|
162
|
+
Parameters
|
|
163
|
+
----------
|
|
164
|
+
num_nodes : int
|
|
165
|
+
Integer to determine how many configurations to create.
|
|
166
|
+
server_url : str
|
|
167
|
+
Url of the dummy server.
|
|
168
|
+
port : int
|
|
169
|
+
Port of the dummy server.
|
|
170
|
+
server_name : str
|
|
171
|
+
Configuration name of the dummy server.
|
|
172
|
+
extra_node_config : Path | None
|
|
173
|
+
Path to file with additional node configuration.
|
|
174
|
+
|
|
175
|
+
Returns
|
|
176
|
+
-------
|
|
177
|
+
list[dict]
|
|
178
|
+
List of dictionaries containing node configurations.
|
|
179
|
+
"""
|
|
180
|
+
configs = []
|
|
181
|
+
extra_config = _read_extra_config_file(extra_node_config)
|
|
182
|
+
node_data_files = create_node_data_files(num_nodes, server_name)
|
|
183
|
+
for i in range(num_nodes):
|
|
184
|
+
config = {
|
|
185
|
+
"org_id": i + 1,
|
|
186
|
+
"api_key": generate_apikey(),
|
|
187
|
+
"node_name": f"{server_name}_node_{i + 1}",
|
|
188
|
+
"user_defined_config": extra_config,
|
|
189
|
+
}
|
|
190
|
+
create_node_config_file(
|
|
191
|
+
server_url, port, config, server_name, node_data_files[i]
|
|
192
|
+
)
|
|
193
|
+
configs.append(config)
|
|
194
|
+
|
|
195
|
+
return configs
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def create_vserver_import_config(node_configs: list[dict], server_name: str) -> Path:
|
|
199
|
+
"""Create server configuration import file (YAML).
|
|
200
|
+
|
|
201
|
+
Utilized by the ``v6 server import`` command.
|
|
202
|
+
|
|
203
|
+
Parameters
|
|
204
|
+
----------
|
|
205
|
+
node_configs : list[dict]
|
|
206
|
+
List of dictionaries containing the node configurations, returned from
|
|
207
|
+
``generate_node_configs()``.
|
|
208
|
+
server_name : str
|
|
209
|
+
Server name.
|
|
210
|
+
|
|
211
|
+
Returns
|
|
212
|
+
-------
|
|
213
|
+
Path
|
|
214
|
+
Path object where the server import configuration is stored.
|
|
215
|
+
"""
|
|
216
|
+
environment = Environment(
|
|
217
|
+
loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
|
|
218
|
+
trim_blocks=True,
|
|
219
|
+
lstrip_blocks=True,
|
|
220
|
+
autoescape=True,
|
|
221
|
+
)
|
|
222
|
+
template = environment.get_template("server_import_config.j2")
|
|
223
|
+
|
|
224
|
+
organizations = []
|
|
225
|
+
collaboration = {"name": "demo", "participants": []}
|
|
226
|
+
for config in node_configs:
|
|
227
|
+
org_id = config["org_id"]
|
|
228
|
+
org_data = {"name": f"org_{org_id}"}
|
|
229
|
+
|
|
230
|
+
organizations.append(org_data)
|
|
231
|
+
collaboration["participants"].append(
|
|
232
|
+
{"name": f"org_{org_id}", "api_key": config["api_key"]}
|
|
233
|
+
)
|
|
234
|
+
organizations[0]["make_admin"] = True
|
|
235
|
+
info(
|
|
236
|
+
f"Organization {Fore.GREEN}{node_configs[0]['org_id']}"
|
|
237
|
+
f"{Style.RESET_ALL} is the admin"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
server_import_config = template.render(
|
|
241
|
+
organizations=organizations, collaboration=collaboration
|
|
242
|
+
)
|
|
243
|
+
folders = ServerContext.instance_folders(InstanceType.SERVER, server_name, False)
|
|
244
|
+
|
|
245
|
+
demo_dir = Path(folders["dev"])
|
|
246
|
+
demo_dir.mkdir(parents=True, exist_ok=True)
|
|
247
|
+
full_path = demo_dir / f"{server_name}.yaml"
|
|
248
|
+
if full_path.exists():
|
|
249
|
+
error(f"Server configuration file already exists: {full_path}")
|
|
250
|
+
exit(1)
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
with open(full_path, "x") as f:
|
|
254
|
+
f.write(server_import_config)
|
|
255
|
+
info(
|
|
256
|
+
"Server import configuration ready, writing to "
|
|
257
|
+
f"{Fore.GREEN}{full_path}{Style.RESET_ALL}"
|
|
258
|
+
)
|
|
259
|
+
except Exception as e:
|
|
260
|
+
error(f"Could not write server import configuration file: {e}")
|
|
261
|
+
exit(1)
|
|
262
|
+
|
|
263
|
+
return full_path
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def create_vserver_config(
|
|
267
|
+
server_name: str,
|
|
268
|
+
port: int,
|
|
269
|
+
server_url: str,
|
|
270
|
+
extra_config_file: Path,
|
|
271
|
+
ui_image: str | None,
|
|
272
|
+
ui_port: int,
|
|
273
|
+
store_port: int,
|
|
274
|
+
) -> Path:
|
|
275
|
+
"""Creates server configuration file (YAML).
|
|
276
|
+
|
|
277
|
+
Parameters
|
|
278
|
+
----------
|
|
279
|
+
server_name : str
|
|
280
|
+
Server name.
|
|
281
|
+
port : int
|
|
282
|
+
Server port.
|
|
283
|
+
server_url : str
|
|
284
|
+
Url of the server this
|
|
285
|
+
extra_config_file : Path
|
|
286
|
+
Path to file with additional server configuration.
|
|
287
|
+
ui_image : str | None
|
|
288
|
+
UI docker image to specify in configuration files. Will be used on startup of
|
|
289
|
+
the network.
|
|
290
|
+
ui_port : int
|
|
291
|
+
Port to run the UI on.
|
|
292
|
+
store_port : int
|
|
293
|
+
Port to run the algorithm store on.
|
|
294
|
+
|
|
295
|
+
Returns
|
|
296
|
+
-------
|
|
297
|
+
Path
|
|
298
|
+
Path object where server configuration is stored.
|
|
299
|
+
"""
|
|
300
|
+
environment = Environment(
|
|
301
|
+
loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
|
|
302
|
+
trim_blocks=True,
|
|
303
|
+
lstrip_blocks=True,
|
|
304
|
+
autoescape=True,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
extra_config = _read_extra_config_file(extra_config_file)
|
|
308
|
+
if ui_image is not None:
|
|
309
|
+
if extra_config:
|
|
310
|
+
extra_config += "\n"
|
|
311
|
+
extra_config += f"images:\n ui: {ui_image}"
|
|
312
|
+
|
|
313
|
+
template = environment.get_template("server_config.j2")
|
|
314
|
+
server_config = template.render(
|
|
315
|
+
port=port,
|
|
316
|
+
host_uri=server_url,
|
|
317
|
+
jwt_secret_key=generate_apikey(),
|
|
318
|
+
user_provided_config=extra_config,
|
|
319
|
+
ui_port=ui_port,
|
|
320
|
+
store_port=store_port,
|
|
321
|
+
)
|
|
322
|
+
folders = ServerContext.instance_folders(
|
|
323
|
+
instance_type=InstanceType.SERVER,
|
|
324
|
+
instance_name=server_name,
|
|
325
|
+
system_folders=False,
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
config_dir = Path(folders["config"] / server_name)
|
|
329
|
+
config_dir.mkdir(parents=True, exist_ok=True)
|
|
330
|
+
full_path = folders["config"] / f"{server_name}.yaml"
|
|
331
|
+
if full_path.exists():
|
|
332
|
+
error(f"Server configuration file already exists: {full_path}")
|
|
333
|
+
exit(1)
|
|
334
|
+
|
|
335
|
+
try:
|
|
336
|
+
with open(full_path, "x") as f:
|
|
337
|
+
f.write(server_config)
|
|
338
|
+
info(
|
|
339
|
+
"Server configuration read, writing to "
|
|
340
|
+
f"{Fore.GREEN}{full_path}{Style.RESET_ALL}"
|
|
341
|
+
)
|
|
342
|
+
except Exception as e:
|
|
343
|
+
error(f"Could not write server configuration file: {e}")
|
|
344
|
+
exit(1)
|
|
345
|
+
|
|
346
|
+
return full_path
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def create_algo_store_config(
|
|
350
|
+
server_name: str,
|
|
351
|
+
server_url: str,
|
|
352
|
+
server_port: int,
|
|
353
|
+
store_port: int,
|
|
354
|
+
extra_config_file: Path,
|
|
355
|
+
) -> Path:
|
|
356
|
+
"""Create algorithm store configuration file (YAML).
|
|
357
|
+
|
|
358
|
+
Parameters
|
|
359
|
+
----------
|
|
360
|
+
server_name : str
|
|
361
|
+
Server name.
|
|
362
|
+
server_url : str
|
|
363
|
+
Url of the server this store connects to.
|
|
364
|
+
server_port : int
|
|
365
|
+
Port of the server this store connects to.
|
|
366
|
+
port : int
|
|
367
|
+
Port of the algorithm store.
|
|
368
|
+
extra_config_file : Path
|
|
369
|
+
Path to file with additional algorithm store configuration.
|
|
370
|
+
"""
|
|
371
|
+
environment = Environment(
|
|
372
|
+
loader=FileSystemLoader(PACKAGE_FOLDER / APPNAME / "cli" / "template"),
|
|
373
|
+
trim_blocks=True,
|
|
374
|
+
lstrip_blocks=True,
|
|
375
|
+
autoescape=True,
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
extra_config = _read_extra_config_file(extra_config_file)
|
|
379
|
+
|
|
380
|
+
template = environment.get_template("algo_store_config.j2")
|
|
381
|
+
store_config = template.render(
|
|
382
|
+
port=store_port,
|
|
383
|
+
server_port=server_port,
|
|
384
|
+
host_uri=server_url,
|
|
385
|
+
user_provided_config=extra_config,
|
|
386
|
+
)
|
|
387
|
+
folders = AlgorithmStoreContext.instance_folders(
|
|
388
|
+
instance_type=InstanceType.ALGORITHM_STORE,
|
|
389
|
+
instance_name="{server_name}_store",
|
|
390
|
+
system_folders=False,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
config_dir = Path(folders["config"] / f"{server_name}_store")
|
|
394
|
+
config_dir.mkdir(parents=True, exist_ok=True)
|
|
395
|
+
full_path = folders["config"] / f"{server_name}_store.yaml"
|
|
396
|
+
if full_path.exists():
|
|
397
|
+
error(f"Algorithm store configuration file already exists: {full_path}")
|
|
398
|
+
exit(1)
|
|
399
|
+
|
|
400
|
+
try:
|
|
401
|
+
with open(full_path, "x") as f:
|
|
402
|
+
f.write(store_config)
|
|
403
|
+
info(
|
|
404
|
+
"Algorithm store configuration ready, writing to "
|
|
405
|
+
f"{Fore.GREEN}{full_path}{Style.RESET_ALL}"
|
|
406
|
+
)
|
|
407
|
+
except Exception as e:
|
|
408
|
+
error(f"Could not write algorithm store configuration file: {e}")
|
|
409
|
+
exit(1)
|
|
410
|
+
|
|
411
|
+
return full_path
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def demo_network(
|
|
415
|
+
num_nodes: int,
|
|
416
|
+
server_url: str,
|
|
417
|
+
server_port: int,
|
|
418
|
+
server_name: str,
|
|
419
|
+
extra_server_config: Path,
|
|
420
|
+
extra_node_config: Path,
|
|
421
|
+
extra_store_config: Path,
|
|
422
|
+
ui_image: str,
|
|
423
|
+
ui_port: int,
|
|
424
|
+
algorithm_store_port: int,
|
|
425
|
+
) -> tuple[list[dict], Path, Path]:
|
|
426
|
+
"""Generates the demo network.
|
|
427
|
+
|
|
428
|
+
Parameters
|
|
429
|
+
----------
|
|
430
|
+
num_nodes : int
|
|
431
|
+
Integer to determine how many configurations to create.
|
|
432
|
+
server_url : str
|
|
433
|
+
Url of the dummy server.
|
|
434
|
+
server_port : int
|
|
435
|
+
Port of the dummy server.
|
|
436
|
+
server_name : str
|
|
437
|
+
Server name.
|
|
438
|
+
extra_server_config : Path
|
|
439
|
+
Path to file with additional server configuration.
|
|
440
|
+
extra_node_config : Path
|
|
441
|
+
Path to file with additional node configuration.
|
|
442
|
+
extra_store_config : Path
|
|
443
|
+
Path to file with additional algorithm store configuration.
|
|
444
|
+
ui_image : str | None
|
|
445
|
+
UI docker image to specify in configuration files. Will be used on startup of
|
|
446
|
+
the network.
|
|
447
|
+
ui_port : int
|
|
448
|
+
Port to run the UI on.
|
|
449
|
+
algorithm_store_port : int
|
|
450
|
+
Port to run the algorithm store on.
|
|
451
|
+
|
|
452
|
+
Returns
|
|
453
|
+
-------
|
|
454
|
+
tuple[list[dict], Path, Path]
|
|
455
|
+
Tuple containing node, server import and server configurations.
|
|
456
|
+
"""
|
|
457
|
+
node_configs = generate_node_configs(
|
|
458
|
+
num_nodes, server_url, server_port, server_name, extra_node_config
|
|
459
|
+
)
|
|
460
|
+
server_import_config = create_vserver_import_config(node_configs, server_name)
|
|
461
|
+
server_config = create_vserver_config(
|
|
462
|
+
server_name,
|
|
463
|
+
server_port,
|
|
464
|
+
server_url,
|
|
465
|
+
extra_server_config,
|
|
466
|
+
ui_image,
|
|
467
|
+
ui_port,
|
|
468
|
+
algorithm_store_port,
|
|
469
|
+
)
|
|
470
|
+
store_config = create_algo_store_config(
|
|
471
|
+
server_name, server_url, server_port, algorithm_store_port, extra_store_config
|
|
472
|
+
)
|
|
473
|
+
return (node_configs, server_import_config, server_config, store_config)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
@click.command()
|
|
477
|
+
@click.option(
|
|
478
|
+
"-n", "--name", default=None, type=str, help="Name for your development setup"
|
|
479
|
+
)
|
|
480
|
+
@click.option(
|
|
481
|
+
"--num-nodes",
|
|
482
|
+
type=int,
|
|
483
|
+
default=3,
|
|
484
|
+
help="Generate this number of nodes in the development network",
|
|
485
|
+
)
|
|
486
|
+
@click.option(
|
|
487
|
+
"--server-url",
|
|
488
|
+
type=str,
|
|
489
|
+
default="http://host.docker.internal",
|
|
490
|
+
help="Server URL to point to. If you are using Docker Desktop, the default "
|
|
491
|
+
"http://host.docker.internal should not be changed. If you are using Linux without"
|
|
492
|
+
" Docker Desktop, you should set this to http://172.17.0.1",
|
|
493
|
+
)
|
|
494
|
+
@click.option(
|
|
495
|
+
"-p",
|
|
496
|
+
"--server-port",
|
|
497
|
+
type=int,
|
|
498
|
+
default=Ports.DEV_SERVER.value,
|
|
499
|
+
help=f"Port to run the server on. Default is {Ports.DEV_SERVER.value}.",
|
|
500
|
+
)
|
|
501
|
+
@click.option(
|
|
502
|
+
"--ui-port",
|
|
503
|
+
type=int,
|
|
504
|
+
default=Ports.DEV_UI.value,
|
|
505
|
+
help=f"Port to run the UI on. Default is {Ports.DEV_UI.value}.",
|
|
506
|
+
)
|
|
507
|
+
@click.option(
|
|
508
|
+
"--algorithm-store-port",
|
|
509
|
+
type=int,
|
|
510
|
+
default=Ports.DEV_ALGO_STORE.value,
|
|
511
|
+
help=(
|
|
512
|
+
f"Port to run the algorithm store on. Default is {Ports.DEV_ALGO_STORE.value}."
|
|
513
|
+
),
|
|
514
|
+
)
|
|
515
|
+
@click.option(
|
|
516
|
+
"-i",
|
|
517
|
+
"--image",
|
|
518
|
+
type=str,
|
|
519
|
+
default=None,
|
|
520
|
+
help="Server docker image to use when setting up resources for "
|
|
521
|
+
"the development server",
|
|
522
|
+
)
|
|
523
|
+
@click.option(
|
|
524
|
+
"--ui-image",
|
|
525
|
+
type=str,
|
|
526
|
+
default=None,
|
|
527
|
+
help="UI docker image to specify in configuration files. Will be used on startup of"
|
|
528
|
+
" the network",
|
|
529
|
+
)
|
|
530
|
+
@click.option(
|
|
531
|
+
"--extra-server-config",
|
|
532
|
+
type=click.Path(exists=True),
|
|
533
|
+
default=None,
|
|
534
|
+
help="YAML File with additional server "
|
|
535
|
+
"configuration. This will be appended to the server "
|
|
536
|
+
"configuration file",
|
|
537
|
+
)
|
|
538
|
+
@click.option(
|
|
539
|
+
"--extra-node-config",
|
|
540
|
+
type=click.Path("rb"),
|
|
541
|
+
default=None,
|
|
542
|
+
help="YAML File with additional node configuration. This will be"
|
|
543
|
+
" appended to each of the node configuration files",
|
|
544
|
+
)
|
|
545
|
+
@click.option(
|
|
546
|
+
"--extra-store-config",
|
|
547
|
+
type=click.Path("rb"),
|
|
548
|
+
default=None,
|
|
549
|
+
help="YAML File with additional algorithm store configuration. This will be"
|
|
550
|
+
" appended to the algorithm store configuration file",
|
|
551
|
+
)
|
|
552
|
+
@click.pass_context
|
|
553
|
+
def create_demo_network(
|
|
554
|
+
click_ctx: click.Context,
|
|
555
|
+
name: str,
|
|
556
|
+
num_nodes: int,
|
|
557
|
+
server_url: str,
|
|
558
|
+
server_port: int,
|
|
559
|
+
ui_port: int,
|
|
560
|
+
algorithm_store_port: int,
|
|
561
|
+
image: str = None,
|
|
562
|
+
ui_image: str = None,
|
|
563
|
+
extra_server_config: Path = None,
|
|
564
|
+
extra_node_config: Path = None,
|
|
565
|
+
extra_store_config: Path = None,
|
|
566
|
+
) -> dict:
|
|
567
|
+
"""Creates a demo network.
|
|
568
|
+
|
|
569
|
+
Creates server instance as well as its import configuration file. Server
|
|
570
|
+
name is set to 'dev_default_server'. Generates `n` node configurations, but
|
|
571
|
+
by default this is set to 3. Then runs a Batch import of
|
|
572
|
+
organizations/collaborations/users and tasks.
|
|
573
|
+
"""
|
|
574
|
+
server_name = prompt_config_name(name)
|
|
575
|
+
if not ServerContext.config_exists(server_name, False):
|
|
576
|
+
demo = demo_network(
|
|
577
|
+
num_nodes,
|
|
578
|
+
server_url,
|
|
579
|
+
server_port,
|
|
580
|
+
server_name,
|
|
581
|
+
extra_server_config,
|
|
582
|
+
extra_node_config,
|
|
583
|
+
extra_store_config,
|
|
584
|
+
ui_image,
|
|
585
|
+
ui_port,
|
|
586
|
+
algorithm_store_port,
|
|
587
|
+
)
|
|
588
|
+
info(
|
|
589
|
+
f"Created {Fore.GREEN}{len(demo[0])}{Style.RESET_ALL} node "
|
|
590
|
+
f"configuration(s), attaching them to {Fore.GREEN}{server_name}"
|
|
591
|
+
f"{Style.RESET_ALL}."
|
|
592
|
+
)
|
|
593
|
+
else:
|
|
594
|
+
error(f"Configuration {Fore.RED}{server_name}{Style.RESET_ALL} already exists!")
|
|
595
|
+
exit(1)
|
|
596
|
+
(node_config, server_import_config, server_config, store_config) = demo
|
|
597
|
+
ctx = get_server_context(server_name, False, ServerContext)
|
|
598
|
+
click_ctx.invoke(
|
|
599
|
+
cli_server_import,
|
|
600
|
+
ctx=ctx,
|
|
601
|
+
file=server_import_config,
|
|
602
|
+
drop_all=False,
|
|
603
|
+
image=image,
|
|
604
|
+
mount_src="",
|
|
605
|
+
keep=False,
|
|
606
|
+
wait=True,
|
|
607
|
+
)
|
|
608
|
+
info(
|
|
609
|
+
"Development network was set up successfully! You can now start the "
|
|
610
|
+
f"server and nodes with {Fore.GREEN}v6 dev start-demo-network"
|
|
611
|
+
f"{Style.RESET_ALL}"
|
|
612
|
+
)
|
|
613
|
+
# find user credentials to print. Read from server import file
|
|
614
|
+
with open(server_import_config, "r") as f:
|
|
615
|
+
server_import_config = yaml.safe_load(f)
|
|
616
|
+
|
|
617
|
+
try:
|
|
618
|
+
user = server_import_config["organizations"][0]["users"][0]
|
|
619
|
+
username = user["username"]
|
|
620
|
+
password = user["password"]
|
|
621
|
+
info(
|
|
622
|
+
"You can login with the following credentials:\n"
|
|
623
|
+
f"Username: {username}\n"
|
|
624
|
+
f"Password: {password}\n"
|
|
625
|
+
)
|
|
626
|
+
except KeyError:
|
|
627
|
+
# No user found, skip printing credentials
|
|
628
|
+
pass
|
|
629
|
+
|
|
630
|
+
return {
|
|
631
|
+
"node_configs": node_config,
|
|
632
|
+
"server_import_config": server_import_config,
|
|
633
|
+
"server_config": server_config,
|
|
634
|
+
"store_config": store_config,
|
|
635
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import itertools
|
|
3
|
+
from shutil import rmtree
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
import docker
|
|
8
|
+
from colorama import Fore, Style
|
|
9
|
+
|
|
10
|
+
from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
|
|
11
|
+
from vantage6.common import info, error
|
|
12
|
+
from vantage6.common.globals import APPNAME
|
|
13
|
+
from vantage6.cli.context.server import ServerContext
|
|
14
|
+
from vantage6.cli.context.node import NodeContext
|
|
15
|
+
from vantage6.cli.server.remove import cli_server_remove
|
|
16
|
+
from vantage6.cli.utils import remove_file
|
|
17
|
+
from vantage6.common.globals import InstanceType
|
|
18
|
+
from vantage6.cli.dev.utils import get_dev_server_context
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@click.command()
|
|
22
|
+
@click.option("-n", "--name", default=None, help="Name of the configuration.")
|
|
23
|
+
@click.option(
|
|
24
|
+
"-c",
|
|
25
|
+
"--config",
|
|
26
|
+
default=None,
|
|
27
|
+
help="Path to configuration-file; overrides --name",
|
|
28
|
+
)
|
|
29
|
+
@click.pass_context
|
|
30
|
+
def remove_demo_network(
|
|
31
|
+
click_ctx: click.Context, name: str | None, config: str | None
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Remove all related demo network files and folders.
|
|
34
|
+
|
|
35
|
+
Select a server configuration to remove that server and the nodes attached
|
|
36
|
+
to it.
|
|
37
|
+
"""
|
|
38
|
+
ctx = get_dev_server_context(config, name)
|
|
39
|
+
|
|
40
|
+
# check that the server is not running
|
|
41
|
+
client = docker.from_env()
|
|
42
|
+
running_servers = client.containers.list(
|
|
43
|
+
filters={"label": f"{APPNAME}-type={InstanceType.SERVER.value}"}
|
|
44
|
+
)
|
|
45
|
+
running_server_names = [server.name for server in running_servers]
|
|
46
|
+
container_name = f"{APPNAME}-{name}-user-{InstanceType.SERVER.value}"
|
|
47
|
+
if container_name in running_server_names:
|
|
48
|
+
error(
|
|
49
|
+
f"Server {Fore.RED}{name}{Style.RESET_ALL} is still running! First stop "
|
|
50
|
+
"the network with 'v6 dev stop-demo-network'."
|
|
51
|
+
)
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
# remove the server
|
|
55
|
+
for handler in itertools.chain(ctx.log.handlers, ctx.log.root.handlers):
|
|
56
|
+
handler.close()
|
|
57
|
+
click_ctx.invoke(cli_server_remove, ctx=ctx, force=True)
|
|
58
|
+
|
|
59
|
+
# removing the server import config
|
|
60
|
+
info("Deleting demo import config file")
|
|
61
|
+
server_configs = ServerContext.instance_folders(
|
|
62
|
+
InstanceType.SERVER, ctx.name, system_folders=False
|
|
63
|
+
)
|
|
64
|
+
import_config_to_del = Path(server_configs["dev"]) / f"{ctx.name}.yaml"
|
|
65
|
+
remove_file(import_config_to_del, "import_configuration")
|
|
66
|
+
|
|
67
|
+
# also remove the server folder
|
|
68
|
+
server_configs = ServerContext.instance_folders(
|
|
69
|
+
InstanceType.SERVER, ctx.name, system_folders=False
|
|
70
|
+
)
|
|
71
|
+
server_folder = server_configs["data"]
|
|
72
|
+
if server_folder.is_dir():
|
|
73
|
+
rmtree(server_folder)
|
|
74
|
+
|
|
75
|
+
# remove the store folder
|
|
76
|
+
store_configs = AlgorithmStoreContext.instance_folders(
|
|
77
|
+
InstanceType.ALGORITHM_STORE, f"{ctx.name}_store", system_folders=False
|
|
78
|
+
)
|
|
79
|
+
store_folder = store_configs["data"]
|
|
80
|
+
if store_folder.is_dir():
|
|
81
|
+
rmtree(store_folder)
|
|
82
|
+
|
|
83
|
+
# remove the store config file
|
|
84
|
+
subprocess.run(
|
|
85
|
+
[
|
|
86
|
+
"v6",
|
|
87
|
+
"algorithm-store",
|
|
88
|
+
"remove",
|
|
89
|
+
"-n",
|
|
90
|
+
f"{ctx.name}_store",
|
|
91
|
+
"--force",
|
|
92
|
+
"--user",
|
|
93
|
+
]
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# remove the nodes
|
|
97
|
+
configs, _ = NodeContext.available_configurations(system_folders=False)
|
|
98
|
+
node_names = [
|
|
99
|
+
config.name for config in configs if config.name.startswith(f"{ctx.name}_node_")
|
|
100
|
+
]
|
|
101
|
+
for name in node_names:
|
|
102
|
+
node_ctx = NodeContext(name, False)
|
|
103
|
+
for handler in itertools.chain(
|
|
104
|
+
node_ctx.log.handlers, node_ctx.log.root.handlers
|
|
105
|
+
):
|
|
106
|
+
handler.close()
|
|
107
|
+
subprocess.run(["v6", "node", "remove", "-n", name, "--user", "--force"])
|
|
108
|
+
|
|
109
|
+
# remove data files attached to the network
|
|
110
|
+
data_dirs_nodes = NodeContext.instance_folders("node", "", False)["dev"]
|
|
111
|
+
rmtree(Path(data_dirs_nodes / ctx.name))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: vantage6
|
|
3
|
-
Version: 5.0.
|
|
3
|
+
Version: 5.0.0a16
|
|
4
4
|
Summary: vantage6 command line interface
|
|
5
5
|
Home-page: https://github.com/vantage6/vantage6
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -16,8 +16,8 @@ Requires-Dist: questionary==1.10.0
|
|
|
16
16
|
Requires-Dist: rich==13.5.2
|
|
17
17
|
Requires-Dist: schema==0.7.5
|
|
18
18
|
Requires-Dist: sqlalchemy==2.0.37
|
|
19
|
-
Requires-Dist: vantage6-common==5.0.
|
|
20
|
-
Requires-Dist: vantage6-client==5.0.
|
|
19
|
+
Requires-Dist: vantage6-common==5.0.0a16
|
|
20
|
+
Requires-Dist: vantage6-client==5.0.0a16
|
|
21
21
|
Provides-Extra: dev
|
|
22
22
|
Requires-Dist: coverage==6.4.4; extra == "dev"
|
|
23
23
|
Requires-Dist: black; extra == "dev"
|
|
@@ -3,11 +3,11 @@ tests_cli/test_example.py,sha256=0fw_v-lgZEacshWSDwLNyLMA1_xc48bKUGM3ll-n1L0,146
|
|
|
3
3
|
tests_cli/test_node_cli.py,sha256=ajFdG1sTa9H7PjqK2dhcp3k59HGsJyO6ZoRfxgIUHcA,16842
|
|
4
4
|
tests_cli/test_server_cli.py,sha256=Yv6k0mkqElrpPgFtro_OH2Bjixz7mDXioxhvXr9VsAQ,6550
|
|
5
5
|
tests_cli/test_wizard.py,sha256=NIj59eiCBuVNJXwhofrWLmLIKAsD45gSOzqOFWLmWhY,4916
|
|
6
|
-
vantage6/cli/__build__,sha256=
|
|
6
|
+
vantage6/cli/__build__,sha256=sX720Zx6Wx7oO5B8WVUm3LHrBtuCJ9ZQ1d2gqfTOjNk,2
|
|
7
7
|
vantage6/cli/__init__.py,sha256=ZXbeQ_-g2-M4XYteWZkoO5lMFYhqjm5doQgGy1fq8i0,125
|
|
8
8
|
vantage6/cli/_version.py,sha256=iDijqhgy5jzZ0LAyzW1LlXeeuMcHWMyg9D8xbXtV7Ck,696
|
|
9
9
|
vantage6/cli/cli.py,sha256=B0F328FSBBslwplMPk8lIlr0r-taKuCgb8v9DIdyE3Q,5699
|
|
10
|
-
vantage6/cli/configuration_manager.py,sha256=
|
|
10
|
+
vantage6/cli/configuration_manager.py,sha256=CHGyYkHT8sIaEZjRrmWuiiDPfFdpFEbpl-yV5jG7OgM,3563
|
|
11
11
|
vantage6/cli/configuration_wizard.py,sha256=ifqvrVqHkxoM0ZVUVIwlYXFByzAbuVlahNjmwFGLVRU,20874
|
|
12
12
|
vantage6/cli/globals.py,sha256=8AWI55FBbumVQTuI1bJzKp5hiRWtiwsVgTTKqWgRBes,1616
|
|
13
13
|
vantage6/cli/utils.py,sha256=Jfr6IeHMQDk_wU5X7rJ1dRY118dhVVX8PwzwMYMv9Vw,2481
|
|
@@ -26,8 +26,10 @@ vantage6/cli/common/utils.py,sha256=eYlR-n2r3j7rW-EPSUnJR98Df8ye551WOl8tW912p3A,
|
|
|
26
26
|
vantage6/cli/context/__init__.py,sha256=e8rfY2tCyu6_SLQ-rbVzEHkDtmbnGCZRHFN_HH-2bnA,2683
|
|
27
27
|
vantage6/cli/context/algorithm_store.py,sha256=RimxNcoqfWeu2WQede6wsOu1rx-azzXIPVkCDqVJLWs,3944
|
|
28
28
|
vantage6/cli/context/base_server.py,sha256=paKSzNrKWD-J6eakHAtGELk2cD05A8NqoCAuQfF7c2s,2972
|
|
29
|
-
vantage6/cli/context/node.py,sha256=
|
|
29
|
+
vantage6/cli/context/node.py,sha256=vEmlWtx7V0e5yqRp9Yi-FhIzSDd4LgnXPVXKYn2n048,7370
|
|
30
30
|
vantage6/cli/context/server.py,sha256=vBGJWNsJoVcIryX5OLiWnFklNRcjOVkhqm2U5tqW5b0,3946
|
|
31
|
+
vantage6/cli/dev/create.py,sha256=6LiK0MUZjZK_W932WnlMMVeCqX1L11F87Rk1UkU6O-4,19347
|
|
32
|
+
vantage6/cli/dev/remove.py,sha256=R_OU_LXLDCnoD-2xnegg4lh0B3t8EgpqzDqueLx16io,3730
|
|
31
33
|
vantage6/cli/node/attach.py,sha256=cmouPrkbIbg21_wlAe-L-ecmrKVxiDkzGmEtRaCnKQY,276
|
|
32
34
|
vantage6/cli/node/clean.py,sha256=uCty2GNuwoTybs1nIOygQLxtbleQ-rnnS6_4ieWVmCw,1199
|
|
33
35
|
vantage6/cli/node/create_private_key.py,sha256=yciL1DtACxrBeEGxeaDi0NefDTvegG6s4rr5jA9J5TY,5207
|
|
@@ -57,14 +59,14 @@ vantage6/cli/server/stop.py,sha256=DY3r9VsUk_r3cqIm1iL-U-kstLVb9pZsiGDSZyAMrKA,4
|
|
|
57
59
|
vantage6/cli/server/version.py,sha256=aXAztHEky_F2jPbfPdHPfsAY7rdTurl0_3S6bL94_QQ,1318
|
|
58
60
|
vantage6/cli/server/common/__init__.py,sha256=htv0mFYa4GhIHdzA2xqUUgKhHcMh09UQERlIjIgrwOM,2062
|
|
59
61
|
vantage6/cli/template/algo_store_config.j2,sha256=XR-ly-47p6egH8lVh4lZZDh3YSV4kFnkZprdsfSkS2Y,552
|
|
60
|
-
vantage6/cli/template/node_config.j2,sha256
|
|
62
|
+
vantage6/cli/template/node_config.j2,sha256=NQHr_cyqNkKjYeU5IikBp5B4fDll_opXN_XPOVh8zk4,691
|
|
61
63
|
vantage6/cli/template/server_config.j2,sha256=3gEPY8YlqUMAQEgfR7a1HTU8WaCRhVzTS-IwPhsU1Gg,802
|
|
62
64
|
vantage6/cli/template/server_import_config.j2,sha256=9WT2XeG9-ADoYLb4ahXhof3i9Fcvg0oqwNPyFwLJpvc,1827
|
|
63
65
|
vantage6/cli/test/feature_tester.py,sha256=M8hvebupPwYjcBZoUB8GB3qb8G1-d3ipNzRMc_3-Z8E,2761
|
|
64
66
|
vantage6/cli/test/integration_test.py,sha256=MctR_t-WEyxzFpMdc6ByTcX1BQglZiT5-CIOQXTBBWo,4034
|
|
65
67
|
vantage6/cli/test/common/diagnostic_runner.py,sha256=F8vEaCD6HeKWDcQGVzRkPYxdvEk9owqfciOVdN3bHbw,6607
|
|
66
|
-
vantage6-5.0.
|
|
67
|
-
vantage6-5.0.
|
|
68
|
-
vantage6-5.0.
|
|
69
|
-
vantage6-5.0.
|
|
70
|
-
vantage6-5.0.
|
|
68
|
+
vantage6-5.0.0a16.dist-info/METADATA,sha256=9NR6r10h5B4hb7KTxj7jLrfNC3u4fkyQQp79tVr1nvI,10887
|
|
69
|
+
vantage6-5.0.0a16.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
70
|
+
vantage6-5.0.0a16.dist-info/entry_points.txt,sha256=YFBvwjxoeAGxYyPC-YevEgOBBYRGaXkS6jiOGGCLNy0,157
|
|
71
|
+
vantage6-5.0.0a16.dist-info/top_level.txt,sha256=CYDIBS8jEfFq5YCs_Fuit54K9-3wdosZppTrsymIoUk,19
|
|
72
|
+
vantage6-5.0.0a16.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|