cvdlink 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- FeatureCloud/__init__.py +0 -0
- FeatureCloud/api/__init__.py +0 -0
- FeatureCloud/api/cli/__init__.py +0 -0
- FeatureCloud/api/cli/__main__.py +115 -0
- FeatureCloud/api/cli/app/__init__.py +0 -0
- FeatureCloud/api/cli/app/commands.py +182 -0
- FeatureCloud/api/cli/controller/__init__.py +0 -0
- FeatureCloud/api/cli/controller/commands.py +181 -0
- FeatureCloud/api/cli/test/__init__.py +0 -0
- FeatureCloud/api/cli/test/commands.py +251 -0
- FeatureCloud/api/cli/test/workflow/__init__.py +0 -0
- FeatureCloud/api/cli/test/workflow/commands.py +32 -0
- FeatureCloud/api/imp/__init__.py +0 -0
- FeatureCloud/api/imp/app/__init__.py +0 -0
- FeatureCloud/api/imp/app/commands.py +278 -0
- FeatureCloud/api/imp/controller/__init__.py +0 -0
- FeatureCloud/api/imp/controller/commands.py +246 -0
- FeatureCloud/api/imp/exceptions.py +29 -0
- FeatureCloud/api/imp/test/__init__.py +0 -0
- FeatureCloud/api/imp/test/api/__init__.py +0 -0
- FeatureCloud/api/imp/test/api/backend/__init__.py +0 -0
- FeatureCloud/api/imp/test/api/backend/auth.py +54 -0
- FeatureCloud/api/imp/test/api/backend/project.py +84 -0
- FeatureCloud/api/imp/test/api/controller.py +97 -0
- FeatureCloud/api/imp/test/commands.py +124 -0
- FeatureCloud/api/imp/test/helper.py +40 -0
- FeatureCloud/api/imp/util.py +45 -0
- FeatureCloud/app/__init__.py +0 -0
- FeatureCloud/app/api/__init__.py +0 -0
- FeatureCloud/app/api/http_ctrl.py +48 -0
- FeatureCloud/app/api/http_web.py +16 -0
- FeatureCloud/app/engine/__init__.py +0 -0
- FeatureCloud/app/engine/app.py +1214 -0
- FeatureCloud/app/engine/library.py +46 -0
- FeatureCloud/workflow/__init__.py +0 -0
- FeatureCloud/workflow/app.py +197 -0
- FeatureCloud/workflow/controller.py +17 -0
- FeatureCloud/workflow/example_wf.py +83 -0
- FeatureCloud/workflow/workflow.py +86 -0
- cvdlink-0.1.1.dist-info/METADATA +176 -0
- cvdlink-0.1.1.dist-info/RECORD +45 -0
- cvdlink-0.1.1.dist-info/WHEEL +5 -0
- cvdlink-0.1.1.dist-info/entry_points.txt +5 -0
- cvdlink-0.1.1.dist-info/licenses/LICENSE +201 -0
- cvdlink-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This is the module-level docstring for library.py
|
|
3
|
+
"""
|
|
4
|
+
from distutils import dir_util
|
|
5
|
+
|
|
6
|
+
import yaml
|
|
7
|
+
|
|
8
|
+
from app import AppState
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class BlankState(AppState):
|
|
12
|
+
|
|
13
|
+
def __init__(self, next_state: str = 'terminal'):
|
|
14
|
+
super().__init__()
|
|
15
|
+
self.next_state = next_state
|
|
16
|
+
|
|
17
|
+
def register(self):
|
|
18
|
+
if self.next_state:
|
|
19
|
+
self.register_transition(self.next_state)
|
|
20
|
+
|
|
21
|
+
def run(self):
|
|
22
|
+
return self.next_state
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class CopyState(BlankState):
|
|
26
|
+
|
|
27
|
+
def __init__(self, next_state=None):
|
|
28
|
+
super().__init__(next_state)
|
|
29
|
+
|
|
30
|
+
def run(self):
|
|
31
|
+
dir_util.copy_tree('/mnt/input/', '/mnt/output/')
|
|
32
|
+
return super().run()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ConfigState(BlankState):
|
|
36
|
+
|
|
37
|
+
def __init__(self, next_state, section, config='config'):
|
|
38
|
+
super().__init__(next_state)
|
|
39
|
+
self.section = section
|
|
40
|
+
self.config = config
|
|
41
|
+
|
|
42
|
+
def run(self):
|
|
43
|
+
if self.section:
|
|
44
|
+
with open('/mnt/input/config.yml') as f:
|
|
45
|
+
self.store(self.config, yaml.load(f, Loader=yaml.FullLoader)[self.section])
|
|
46
|
+
return super().run()
|
|
File without changes
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
from os import listdir
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import zipfile
|
|
5
|
+
from time import sleep
|
|
6
|
+
from FeatureCloud.workflow.controller import Controller
|
|
7
|
+
from functools import partial
|
|
8
|
+
import shutil
|
|
9
|
+
from distutils.dir_util import copy_tree
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class TestApp(Controller):
|
|
13
|
+
"""
|
|
14
|
+
Attributes:
|
|
15
|
+
-----------
|
|
16
|
+
app_image: str
|
|
17
|
+
image name of the app in FeatureCloud docker repository
|
|
18
|
+
test_id: int
|
|
19
|
+
ID of the test running on a specific controller.
|
|
20
|
+
n_clients: int
|
|
21
|
+
Number of clients that app will run for
|
|
22
|
+
results_ready: bool
|
|
23
|
+
It is true once the results are extracted to the app's directory
|
|
24
|
+
app_id: int
|
|
25
|
+
ID of app in the workflow
|
|
26
|
+
generic_dir: str
|
|
27
|
+
Relative path to directory containing generic files
|
|
28
|
+
clients_path: list
|
|
29
|
+
Full path to directory containing the clients' data
|
|
30
|
+
clients_relative_path: list
|
|
31
|
+
Relative path to directory containing the clients' data
|
|
32
|
+
results_path: str
|
|
33
|
+
Full path to directory containing the app's results for clients
|
|
34
|
+
results_relative_path: str
|
|
35
|
+
Relative path to directory containing the app's results for clients
|
|
36
|
+
Methods:
|
|
37
|
+
--------
|
|
38
|
+
set_id(test_id):
|
|
39
|
+
extract_results(def_res_file):
|
|
40
|
+
wait_until_finishes():
|
|
41
|
+
clean_dirs(def_re_dir):
|
|
42
|
+
create_paths(ctrl_data_path, ctrl_test_path):
|
|
43
|
+
copy_results(ctrl_data_path, dest_generic, dest_clients, default_res_name):
|
|
44
|
+
|
|
45
|
+
"""
|
|
46
|
+
def __init__(self, app_id, ctrl_data_path, ctrl_test_path, n_clients, app_image, **kwargs):
|
|
47
|
+
super().__init__(**kwargs)
|
|
48
|
+
if app_image.strip().startswith("featurecloud.ai/"):
|
|
49
|
+
self.app_image = app_image.strip()
|
|
50
|
+
else:
|
|
51
|
+
self.app_image = f"featurecloud.ai/{app_image.strip()}"
|
|
52
|
+
self.test_id = None
|
|
53
|
+
self.n_clients = n_clients
|
|
54
|
+
self.results_ready = False
|
|
55
|
+
self.app_id = app_id
|
|
56
|
+
self.generic_dir = ""
|
|
57
|
+
self.clients_path = []
|
|
58
|
+
self.clients_relative_path = []
|
|
59
|
+
self.results_path = ""
|
|
60
|
+
self.results_relative_path = ""
|
|
61
|
+
self.create_paths(ctrl_data_path, ctrl_test_path)
|
|
62
|
+
self.start = partial(self.start,
|
|
63
|
+
client_dirs=self.clients_relative_path,
|
|
64
|
+
generic_dir=self.generic_dir,
|
|
65
|
+
app_image=app_image,
|
|
66
|
+
download_results=self.results_relative_path)
|
|
67
|
+
self.stop = partial(self.stop, self.test_id)
|
|
68
|
+
|
|
69
|
+
def set_id(self, test_id: int):
|
|
70
|
+
""" Set the app's test ID and partially define the
|
|
71
|
+
delete method based on it.
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
Parameters
|
|
75
|
+
----------
|
|
76
|
+
test_id: int
|
|
77
|
+
"""
|
|
78
|
+
self.test_id = test_id
|
|
79
|
+
self.delete = partial(self.delete, test_id=self.test_id, del_all=None)
|
|
80
|
+
|
|
81
|
+
def extract_results(self, def_res_file: str):
|
|
82
|
+
""" extract app's results zip files for all clients
|
|
83
|
+
into their corresponding directories
|
|
84
|
+
|
|
85
|
+
Parameters
|
|
86
|
+
----------
|
|
87
|
+
def_res_file: str
|
|
88
|
+
Default name for the app's result directory
|
|
89
|
+
Same name for all clients.
|
|
90
|
+
|
|
91
|
+
"""
|
|
92
|
+
zip_files = [f for f in listdir(self.results_path) if f.endswith(".zip")]
|
|
93
|
+
Path(self.results_path).mkdir(exist_ok=True, parents=True)
|
|
94
|
+
if len(zip_files) > 1:
|
|
95
|
+
print(f"Extracting the results of {self.app_image} ...")
|
|
96
|
+
for zip_file in zip_files:
|
|
97
|
+
client_n = int(zip_file.strip().split("client_")[-1].strip().split("_")[0])
|
|
98
|
+
res_dir = f"{self.clients_path[client_n]}/{def_res_file}"
|
|
99
|
+
zip_file_path = f"{self.results_path}/{zip_file}"
|
|
100
|
+
if not os.path.exists(res_dir):
|
|
101
|
+
os.makedirs(res_dir, exist_ok=True)
|
|
102
|
+
print(f"Create {res_dir} directory...")
|
|
103
|
+
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
|
|
104
|
+
zip_ref.extractall(res_dir)
|
|
105
|
+
print(f"Extract client {client_n} to {res_dir} directory...")
|
|
106
|
+
self.results_ready = True
|
|
107
|
+
else:
|
|
108
|
+
print(f"Looking into {self.results_path}\n"
|
|
109
|
+
f"There is no file yet!\n"
|
|
110
|
+
"We will check again later....")
|
|
111
|
+
sleep(5)
|
|
112
|
+
self.extract_results(def_res_file)
|
|
113
|
+
|
|
114
|
+
def wait_until_finishes(self):
|
|
115
|
+
""" Waits until the app status becomes finished.
|
|
116
|
+
|
|
117
|
+
"""
|
|
118
|
+
while not self.is_finished():
|
|
119
|
+
sleep(5)
|
|
120
|
+
|
|
121
|
+
def is_finished(self):
|
|
122
|
+
""" Check either the app status is finished.
|
|
123
|
+
|
|
124
|
+
"""
|
|
125
|
+
df = self.info(test_id=self.test_id, format='dataframe')
|
|
126
|
+
# if df is None:
|
|
127
|
+
# print(msg)
|
|
128
|
+
return df.status.values == "finished"
|
|
129
|
+
|
|
130
|
+
def clean_dirs(self, def_re_dir: str):
|
|
131
|
+
""" creates results directories.
|
|
132
|
+
And removes existing results in the directories
|
|
133
|
+
|
|
134
|
+
Parameters
|
|
135
|
+
----------
|
|
136
|
+
def_re_dir: str
|
|
137
|
+
Default name for the app's result directory
|
|
138
|
+
Same name for all clients.
|
|
139
|
+
"""
|
|
140
|
+
for c_dir in self.clients_path:
|
|
141
|
+
print(c_dir, def_re_dir)
|
|
142
|
+
if os.path.exists(f"{c_dir}/{def_re_dir}"):
|
|
143
|
+
print(f"Delete {c_dir}/{def_re_dir}")
|
|
144
|
+
shutil.rmtree(f"{c_dir}/{def_re_dir}")
|
|
145
|
+
if os.path.exists(self.results_path):
|
|
146
|
+
for zip_file in os.listdir(self.results_path):
|
|
147
|
+
if zip_file.endswith(".zip"):
|
|
148
|
+
print(f"Delete {self.results_path}/{zip_file}")
|
|
149
|
+
os.remove(f"{self.results_path}/{zip_file}")
|
|
150
|
+
else:
|
|
151
|
+
Path(self.results_path).mkdir(exist_ok=True, parents=True)
|
|
152
|
+
|
|
153
|
+
def create_paths(self, ctrl_data_path: str, ctrl_test_path: str):
|
|
154
|
+
""" Generate paths to directories containing the app's data(for each client)
|
|
155
|
+
And also for app's results.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
ctrl_data_path: str
|
|
160
|
+
path to the target controller's data folder
|
|
161
|
+
ctrl_test_path: str
|
|
162
|
+
path to the target controller's tests folder
|
|
163
|
+
|
|
164
|
+
"""
|
|
165
|
+
self.results_relative_path = f"./results/app{self.app_id}"
|
|
166
|
+
clients_relpath = [f"./app{self.app_id}/client_{c}" for c in range(self.n_clients)]
|
|
167
|
+
self.clients_relative_path = ",".join(clients_relpath)
|
|
168
|
+
self.clients_path = [f"{ctrl_data_path}{clients_relpath[c][1:]}" for c in
|
|
169
|
+
range(self.n_clients)]
|
|
170
|
+
self.results_path = f"{ctrl_test_path}{self.results_relative_path[1:]}"
|
|
171
|
+
self.generic_dir = f"./app{self.app_id}/generic"
|
|
172
|
+
|
|
173
|
+
def copy_results(self, ctrl_data_path: str, dest_generic: str, dest_clients: list, default_res_name: str):
|
|
174
|
+
""" Copy results of the app to
|
|
175
|
+
as the data to the directory of the next app.
|
|
176
|
+
|
|
177
|
+
Parameters
|
|
178
|
+
----------
|
|
179
|
+
ctrl_data_path: str
|
|
180
|
+
path to the target controller's data folder
|
|
181
|
+
dest_generic: str
|
|
182
|
+
Full path to directory containing
|
|
183
|
+
the generic data of the next app in the workflow
|
|
184
|
+
dest_clients: str
|
|
185
|
+
Full path to directory containing
|
|
186
|
+
the clients' data of the next app in the workflow
|
|
187
|
+
default_res_name: str
|
|
188
|
+
Default name for the app's result directory
|
|
189
|
+
Same name for all clients.
|
|
190
|
+
|
|
191
|
+
"""
|
|
192
|
+
for client_n, client_res in enumerate(self.clients_path):
|
|
193
|
+
res_dir = f"{client_res}/{default_res_name}"
|
|
194
|
+
print(f"Copy {res_dir} to {dest_clients[client_n]} ...")
|
|
195
|
+
copy_tree(res_dir, dest_clients[client_n])
|
|
196
|
+
copy_tree(f"{ctrl_data_path}{dest_generic[1:]}",
|
|
197
|
+
f"{ctrl_data_path}{dest_generic[1:]}")
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from FeatureCloud.api.imp.test import commands
|
|
2
|
+
from functools import partial
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Controller:
|
|
6
|
+
def __init__(self, controller_host: str, channel: str, query_interval: str):
|
|
7
|
+
self.controller_host = controller_host
|
|
8
|
+
self.channel = channel
|
|
9
|
+
self.query_interval = query_interval
|
|
10
|
+
self.start = partial(commands.start, controller_host=controller_host, channel=channel,
|
|
11
|
+
query_interval=query_interval)
|
|
12
|
+
self.stop = partial(commands.stop, controller_host=controller_host)
|
|
13
|
+
self.delete = partial(commands.delete, controller_host=controller_host)
|
|
14
|
+
self.list = partial(commands.list, controller_host=controller_host)
|
|
15
|
+
self.traffic = partial(commands.traffic, controller_host=controller_host)
|
|
16
|
+
self.logs = partial(commands.logs, controller_host=controller_host)
|
|
17
|
+
self.info = partial(commands.info, controller_host=controller_host)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from FeatureCloud.workflow.workflow import TestWorkFlow
|
|
2
|
+
from FeatureCloud.workflow.app import TestApp
|
|
3
|
+
from time import sleep
|
|
4
|
+
from functools import partial
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class WorkFlow(TestWorkFlow):
|
|
8
|
+
""" Example workflow containing following apps:
|
|
9
|
+
1- featurecloud.ai/fc_cross_validation
|
|
10
|
+
2- featurecloud.ai/basic_rf
|
|
11
|
+
3- featurecloud.ai/fc_roc
|
|
12
|
+
|
|
13
|
+
Methods:
|
|
14
|
+
--------
|
|
15
|
+
register_apps(): registering the apps
|
|
16
|
+
run(): running apps with the same order as registration
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
def __init__(self, controller_host: str, channel: str, query_interval: str):
|
|
20
|
+
super().__init__(controller_host, channel, query_interval)
|
|
21
|
+
|
|
22
|
+
self.controller_path = "/home/mohammad/PycharmProjects/FeatureCloud/data"
|
|
23
|
+
self.ctrl_data_path = f"{self.controller_path}"
|
|
24
|
+
self.ctrl_test_path = f"{self.controller_path}/tests"
|
|
25
|
+
|
|
26
|
+
self.generic_dir = {}
|
|
27
|
+
self.n_clients = 2
|
|
28
|
+
self.TestApp = partial(TestApp,
|
|
29
|
+
n_clients=self.n_clients,
|
|
30
|
+
ctrl_data_path=self.ctrl_data_path,
|
|
31
|
+
ctrl_test_path=self.ctrl_test_path,
|
|
32
|
+
controller_host=controller_host,
|
|
33
|
+
channel=channel,
|
|
34
|
+
query_interval=query_interval)
|
|
35
|
+
|
|
36
|
+
def register_apps(self):
|
|
37
|
+
""" Registering the three apps.
|
|
38
|
+
|
|
39
|
+
"""
|
|
40
|
+
app_id = 0
|
|
41
|
+
app1 = self.TestApp(app_id=app_id, app_image="featurecloud.ai/fc_cross_validation")
|
|
42
|
+
self.register(app1)
|
|
43
|
+
|
|
44
|
+
app_id += 1
|
|
45
|
+
app2 = self.TestApp(app_id=app_id, app_image="featurecloud.ai/basic_rf")
|
|
46
|
+
self.register(app2)
|
|
47
|
+
|
|
48
|
+
app_id += 1
|
|
49
|
+
app3 = self.TestApp(app_id=app_id, app_image="featurecloud.ai/fc_roc")
|
|
50
|
+
self.register(app3)
|
|
51
|
+
|
|
52
|
+
def run(self):
|
|
53
|
+
""" Running apps with the same registration order,
|
|
54
|
+
logging the app execution,
|
|
55
|
+
setting the ID
|
|
56
|
+
waiting until the app execution is finished
|
|
57
|
+
extracting the result to the app's directory
|
|
58
|
+
Waiting until the extraction is over
|
|
59
|
+
Delete the test run for the app
|
|
60
|
+
Copy the results as data for the next app
|
|
61
|
+
|
|
62
|
+
"""
|
|
63
|
+
print("Workflow execution starts ...")
|
|
64
|
+
for i, app in enumerate(self.apps):
|
|
65
|
+
app.clean_dirs(self.default_res_dir_name)
|
|
66
|
+
id = app.start()
|
|
67
|
+
app.set_id(id)
|
|
68
|
+
print(f"{app.app_image}(ID: {app.test_id}) is running ...")
|
|
69
|
+
app.wait_until_finishes()
|
|
70
|
+
print("App execution is finished!")
|
|
71
|
+
app.extract_results(self.default_res_dir_name)
|
|
72
|
+
print("extracting the data...")
|
|
73
|
+
while not app.results_ready:
|
|
74
|
+
sleep(5)
|
|
75
|
+
print("Delete the app container...")
|
|
76
|
+
app.delete()
|
|
77
|
+
if i < len(self.apps) - 1:
|
|
78
|
+
print(f"Move {app.app_image} results to the directory of the next app({self.apps[i + 1].app_image})")
|
|
79
|
+
app.copy_results(ctrl_data_path=self.ctrl_data_path,
|
|
80
|
+
dest_clients=self.apps[i + 1].clients_path,
|
|
81
|
+
dest_generic=self.apps[i + 1].generic_dir,
|
|
82
|
+
default_res_name=self.default_res_dir_name)
|
|
83
|
+
print("Workflow execution is finished!")
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from FeatureCloud.workflow.controller import Controller
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TestWorkFlow(abc.ABC):
|
|
6
|
+
""" The abstract TestWorkFlow class to cover basic functionalities
|
|
7
|
+
for FeatureCloud workflow.
|
|
8
|
+
Non-linear streams
|
|
9
|
+
Attributes:
|
|
10
|
+
-----------
|
|
11
|
+
apps: list of instances of TestApp in the workflow
|
|
12
|
+
controller: an instance of Controller class
|
|
13
|
+
default_res_dir_name: str
|
|
14
|
+
the dir-name of apps' results
|
|
15
|
+
Methods:
|
|
16
|
+
--------
|
|
17
|
+
register_apps():
|
|
18
|
+
run():
|
|
19
|
+
register(app):
|
|
20
|
+
stop(controller_ind):
|
|
21
|
+
delete(controller_ind):
|
|
22
|
+
list(controller_ind, format):
|
|
23
|
+
info(format, controller_ind):
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, controller_host: str, channel: str, query_interval: str):
|
|
27
|
+
self.apps = []
|
|
28
|
+
self.controller = Controller(controller_host, channel, query_interval)
|
|
29
|
+
self.default_res_dir_name = "AppResSults"
|
|
30
|
+
|
|
31
|
+
@abc.abstractmethod
|
|
32
|
+
def register_apps(self):
|
|
33
|
+
""" Abstract method tha should be implemented
|
|
34
|
+
by developers to register apps into the workflow.
|
|
35
|
+
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
@abc.abstractmethod
|
|
39
|
+
def run(self):
|
|
40
|
+
""" Abstract method tha should be implemented
|
|
41
|
+
by developers to run the workflow.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def register(self, app):
|
|
45
|
+
""" Adding TestApp instance to the app list
|
|
46
|
+
and logging the apps attributes.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
app: TestApp
|
|
51
|
+
app instance to be registered.
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
self.apps.append(app)
|
|
55
|
+
clients_dirs = "\n\t\t".join(app.clients_path)
|
|
56
|
+
msg = f"{app.app_image} app is registered:\n" \
|
|
57
|
+
f"\tController: {app.controller_host}\n" \
|
|
58
|
+
f"\tClient data:\n" \
|
|
59
|
+
f"\t\t{clients_dirs}\n" \
|
|
60
|
+
f"\tGeneric data: {app.generic_dir}\n" \
|
|
61
|
+
f"\tResult dir: {app.results_path}"
|
|
62
|
+
print(msg)
|
|
63
|
+
|
|
64
|
+
def stop(self):
|
|
65
|
+
""" Stop all tests in the controller.
|
|
66
|
+
|
|
67
|
+
"""
|
|
68
|
+
for test_id in self.controller.list():
|
|
69
|
+
self.controller.stop(test_id)
|
|
70
|
+
|
|
71
|
+
def delete(self):
|
|
72
|
+
""" Delete all tests in the controller.
|
|
73
|
+
|
|
74
|
+
"""
|
|
75
|
+
for test_id in self.controller.list():
|
|
76
|
+
self.controller.delete(test_id)
|
|
77
|
+
|
|
78
|
+
def info(self, format: str):
|
|
79
|
+
""" info of all tests in the specified controller or all of them.
|
|
80
|
+
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
info_list = []
|
|
84
|
+
for test_id in self.controller.list():
|
|
85
|
+
info_list.append(self.controller.info(test_id, format))
|
|
86
|
+
return info_list
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cvdlink
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Secure Federated Learning Platform
|
|
5
|
+
Home-page: https://github.com/FeatureCloud/app-template
|
|
6
|
+
Author: CoSyBio Group, University of Hamburg
|
|
7
|
+
Author-email: mohammad.bakhtiari@uni-hamburg.de
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/FeatureCloud/app-template/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.7
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: bottle
|
|
15
|
+
Requires-Dist: jsonpickle
|
|
16
|
+
Requires-Dist: joblib
|
|
17
|
+
Requires-Dist: numpy
|
|
18
|
+
Requires-Dist: pydot
|
|
19
|
+
Requires-Dist: pyyaml
|
|
20
|
+
Requires-Dist: flake8~=3.9.2
|
|
21
|
+
Requires-Dist: pycodestyle~=2.7.0
|
|
22
|
+
Requires-Dist: Click~=8.0.1
|
|
23
|
+
Requires-Dist: requests
|
|
24
|
+
Requires-Dist: urllib3~=1.26.6
|
|
25
|
+
Requires-Dist: pandas>=2.0
|
|
26
|
+
Requires-Dist: pyinstaller
|
|
27
|
+
Requires-Dist: docker==7.1.0
|
|
28
|
+
Requires-Dist: gitpython
|
|
29
|
+
Requires-Dist: tqdm
|
|
30
|
+
Dynamic: author
|
|
31
|
+
Dynamic: author-email
|
|
32
|
+
Dynamic: classifier
|
|
33
|
+
Dynamic: description
|
|
34
|
+
Dynamic: description-content-type
|
|
35
|
+
Dynamic: home-page
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
Dynamic: project-url
|
|
38
|
+
Dynamic: requires-dist
|
|
39
|
+
Dynamic: requires-python
|
|
40
|
+
Dynamic: summary
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# CVDLink (FeatureCloud Extension)
|
|
44
|
+
|
|
45
|
+
## Overview
|
|
46
|
+
|
|
47
|
+
**CVDLink** is a domain-specific extension of the **FeatureCloud** Python package, customized and deployed for the **CVDLink project** available on https://fc.cdvlink-project.eu
|
|
48
|
+
It builds on the core FeatureCloud platform to support **privacy-preserving federated learning and data analysis**, while providing **CVDLink-specific defaults, endpoints, and infrastructure integrations**.
|
|
49
|
+
|
|
50
|
+
CVDLink targets federated collaborations in cardiovascular research and related biomedical domains, enabling partners to participate in secure, distributed workflows without sharing raw data.
|
|
51
|
+
|
|
52
|
+
> **Important:**
|
|
53
|
+
> CVDLink is **not a replacement or independent fork** of FeatureCloud.
|
|
54
|
+
> It is an **extension and specialization** built on top of the FeatureCloud engine.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Relationship to FeatureCloud
|
|
59
|
+
|
|
60
|
+
CVDLink is based on the FeatureCloud platform and reuses:
|
|
61
|
+
|
|
62
|
+
- FeatureCloud controller and execution engine
|
|
63
|
+
- FeatureCloud app and state model
|
|
64
|
+
- FeatureCloud CLI command structure
|
|
65
|
+
- FeatureCloud testing and workflow system
|
|
66
|
+
|
|
67
|
+
CVDLink extends FeatureCloud by providing:
|
|
68
|
+
|
|
69
|
+
- A **CVDLink-specific CLI entry point**
|
|
70
|
+
- **Profile-based configuration** for CVDLink deployments
|
|
71
|
+
- **CVDLink-specific defaults** for:
|
|
72
|
+
- Controller images
|
|
73
|
+
- Global API endpoints
|
|
74
|
+
- Docker registries
|
|
75
|
+
- Relay servers
|
|
76
|
+
- Seamless switching between FeatureCloud and CVDLink environments using the same codebase
|
|
77
|
+
|
|
78
|
+
For general FeatureCloud concepts, architecture, and app development, please refer to:
|
|
79
|
+
- https://featurecloud.ai
|
|
80
|
+
- [FeatureCloud GitHub repositor](https://github.com/FeatureCloud/FeatureCloud/tree/cvdlink)
|
|
81
|
+
- Matschinske, J., Späth, J., Bakhtiari, M., Probul, N., Kazemi Majdabadi, M. M., Nasirigerdeh, R., ... & Baumbach, J. (2023). The FeatureCloud platform for federated learning in biomedicine: unified approach. Journal of Medical Internet Research, 25, e42621.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Installation
|
|
86
|
+
|
|
87
|
+
Install the CVDLink Python package via pip:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install cvdlink
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
## CLI Overview
|
|
95
|
+
|
|
96
|
+
CVDLink provides a **FeatureCloud-compatible command-line interface** with a dedicated entry point:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
cvdlink --help
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The CLI mirrors the FeatureCloud command structure, ensuring that existing FeatureCloud users can work with CVDLink without a learning curve.
|
|
103
|
+
|
|
104
|
+
Internally, the CLI uses a profile-based configuration, allowing the same codebase to support both FeatureCloud and CVDLink deployments.
|
|
105
|
+
|
|
106
|
+
### Controller Commands
|
|
107
|
+
|
|
108
|
+
The controller command group is used to start, manage, and inspect CVDLink controller instances.
|
|
109
|
+
|
|
110
|
+
##### Start a Controller
|
|
111
|
+
```bash
|
|
112
|
+
cvdlink controller start [NAME] [OPTIONS]
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
By default, this command:
|
|
116
|
+
|
|
117
|
+
* Uses CVDLink-specific controller Docker images
|
|
118
|
+
* Connects to CVDLink global API endpoints
|
|
119
|
+
* Uses the CVDLink relay infrastructure
|
|
120
|
+
* Applies CVDLink defaults defined in the configuration profile
|
|
121
|
+
|
|
122
|
+
#### Common Options
|
|
123
|
+
|
|
124
|
+
* --port: Port number for the controller (default: 8000)
|
|
125
|
+
* --data-dir: Directory used to store controller data
|
|
126
|
+
* --controller-image: Override the controller Docker image
|
|
127
|
+
* --global-endpoint: Override the global API endpoint
|
|
128
|
+
* --registry: Override the Docker registry
|
|
129
|
+
* --relay-address: Override the relay server address
|
|
130
|
+
* --poll-interval: Override workflow poll interval
|
|
131
|
+
* --query-interval: Override workflow query interval
|
|
132
|
+
* --config-file: Path to a custom configuration file inside the container
|
|
133
|
+
|
|
134
|
+
Example:
|
|
135
|
+
```bash
|
|
136
|
+
cvdlink controller start --data-dir ./data
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
Other Controller Commands
|
|
141
|
+
|
|
142
|
+
* cvdlink controller status – Display controller status
|
|
143
|
+
* cvdlink controller logs – Show controller logs
|
|
144
|
+
* cvdlink controller tail – Follow controller logs
|
|
145
|
+
* cvdlink controller ls – List running controllers
|
|
146
|
+
* cvdlink controller stop – Stop a controller instance
|
|
147
|
+
|
|
148
|
+
#### App Commands
|
|
149
|
+
|
|
150
|
+
The app command group is used to create, build, manage, and publish federated apps.
|
|
151
|
+
|
|
152
|
+
##### Create a New App
|
|
153
|
+
```bash
|
|
154
|
+
cvdlink app new --template-name <TEMPLATE_URL>
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Creates a new federated app based on a FeatureCloud-compatible template.
|
|
158
|
+
|
|
159
|
+
Build an App Image
|
|
160
|
+
```bash
|
|
161
|
+
cvdlink app build --path <APP_PATH> --image-name <NAME> --tag <TAG>
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Publish an App
|
|
165
|
+
```bash
|
|
166
|
+
cvdlink app publish --name <NAME> --tag <TAG>
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Pushes the app image to the configured Docker registry. The image name should include fc.cvdlink-project.eu/ as the registry prefix which by default, this uses the CVDLink registry, unless overridden.
|
|
170
|
+
|
|
171
|
+
Additional App Commands:
|
|
172
|
+
|
|
173
|
+
* download – Download an app image
|
|
174
|
+
* remove – Remove a local app image
|
|
175
|
+
* plot-states – Visualize app states and transitions
|
|
176
|
+
* All app commands follow the same semantics as FeatureCloud but operate within the CVDLink ecosystem by default.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
FeatureCloud/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
FeatureCloud/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
FeatureCloud/api/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
FeatureCloud/api/cli/__main__.py,sha256=MCVGVzgBzOZXq_6P0llQYV2UUX5ciLvwsIlAL2XpEpU,3641
|
|
5
|
+
FeatureCloud/api/cli/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
FeatureCloud/api/cli/app/commands.py,sha256=wXInRwylUVmoa4zw7Vge96zP3EAL8FZhsMW_akd6pAE,6804
|
|
7
|
+
FeatureCloud/api/cli/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
FeatureCloud/api/cli/controller/commands.py,sha256=GEgQwXdhx7TBbl_CLcU59AGHWDbJz68vCNgnvVPEnKg,6288
|
|
9
|
+
FeatureCloud/api/cli/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
FeatureCloud/api/cli/test/commands.py,sha256=M7oPMG8CuTRu-UJBwpyRIeglhXy7Vj-xGcFwQ-jkZo4,13126
|
|
11
|
+
FeatureCloud/api/cli/test/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
FeatureCloud/api/cli/test/workflow/commands.py,sha256=MKi8m6W_EYODWk4mSdPxIpVxu8OUUMBvcxGCLhHk3NA,1249
|
|
13
|
+
FeatureCloud/api/imp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
FeatureCloud/api/imp/exceptions.py,sha256=vauYTlbJEVkjfHnsFZXNx2EmITiEsnfiDj0cVvOq1ks,762
|
|
15
|
+
FeatureCloud/api/imp/util.py,sha256=fF6OoCHPot0gS_fUmBjPMvvxnUJFPurdUbkDBcr-8QA,1128
|
|
16
|
+
FeatureCloud/api/imp/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
FeatureCloud/api/imp/app/commands.py,sha256=7NlKqIVdPAzivt9j8MoZeP-zOWpHPxhhnL5zoO21X28,8648
|
|
18
|
+
FeatureCloud/api/imp/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
FeatureCloud/api/imp/controller/commands.py,sha256=bmaguJrrr26SYI7iHnnm9NY1P8vPprP3yomt1MZq4J0,7550
|
|
20
|
+
FeatureCloud/api/imp/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
FeatureCloud/api/imp/test/commands.py,sha256=YuTWmHFV0ys2Goq-8lmloSDFXouj33TPjEqBZafNRmk,4444
|
|
22
|
+
FeatureCloud/api/imp/test/helper.py,sha256=_7ZsFOxLuxotWZ3dUFifsUURbbMXrVrOH-3El-5Lz4E,1227
|
|
23
|
+
FeatureCloud/api/imp/test/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
FeatureCloud/api/imp/test/api/controller.py,sha256=LzJzZgaEOhQdkY4aNewModjMuuaWLQvfU73-_u91n7M,2999
|
|
25
|
+
FeatureCloud/api/imp/test/api/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
+
FeatureCloud/api/imp/test/api/backend/auth.py,sha256=7Ds4tXQWo-GIkSqoK0qfLv-rtbdwYn8-KO4TUdd3O6g,1521
|
|
27
|
+
FeatureCloud/api/imp/test/api/backend/project.py,sha256=QPURzxXmCMfMVZNpBbsagXAErc-MHeA3snvjh7aclQw,2128
|
|
28
|
+
FeatureCloud/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
|
+
FeatureCloud/app/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
30
|
+
FeatureCloud/app/api/http_ctrl.py,sha256=g0P0Vv3ltLj9ngzty7ZMIgmABTPtFmEWKxJgGaeKnJM,1380
|
|
31
|
+
FeatureCloud/app/api/http_web.py,sha256=TOgcB8UiW1sJ2feudckzaaHcdFSQvHVWKF7K8ND-FJE,499
|
|
32
|
+
FeatureCloud/app/engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
|
+
FeatureCloud/app/engine/app.py,sha256=kk0XY5WdsSD9KCTXS8kEcCF1HPH92HW77JGIvbL3eQM,48989
|
|
34
|
+
FeatureCloud/app/engine/library.py,sha256=WGrgHakcsfUIwQnd5_F7BgzpHM8EDXdsSeXytNLGUmE,1064
|
|
35
|
+
FeatureCloud/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
36
|
+
FeatureCloud/workflow/app.py,sha256=SSD82zX_qqQE0wr9gGsPR4Bn6kEwq6O7AI4ocT4yVLM,7707
|
|
37
|
+
FeatureCloud/workflow/controller.py,sha256=MUNotYuiTuAfQUHfgE5jnVeQ2YuSD0Od4yfFhiHdWA0,922
|
|
38
|
+
FeatureCloud/workflow/example_wf.py,sha256=saJ5R8K-APbityQk1ykFyTZgC8ijuT-jJdjumFBFMlc,3327
|
|
39
|
+
FeatureCloud/workflow/workflow.py,sha256=xYSQeG-76f08WCZGAKN1Cl_SgtaBOS2XoE6tpjGrR8w,2526
|
|
40
|
+
cvdlink-0.1.1.dist-info/licenses/LICENSE,sha256=BgX6XyNxc1g4sAz8y9To3QBry89ZYTObngyahwMcpxw,11352
|
|
41
|
+
cvdlink-0.1.1.dist-info/METADATA,sha256=Ae_o3bjHdhIOrHm_L0lKEnTVssgQRTcK8k43cLa-OfY,5798
|
|
42
|
+
cvdlink-0.1.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
43
|
+
cvdlink-0.1.1.dist-info/entry_points.txt,sha256=uew2Fofb9y5uNNWFMsNIwmuLVTHROTigmH0bC_V80o8,226
|
|
44
|
+
cvdlink-0.1.1.dist-info/top_level.txt,sha256=80W3BaTCpbvGi9n3eeAJIE8iGOPAdSns2ZGA7-OhaJs,13
|
|
45
|
+
cvdlink-0.1.1.dist-info/RECORD,,
|