vantage6-algorithm-tools 5.0.0a0__tar.gz → 5.0.0a7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vantage6-algorithm-tools-5.0.0a0/vantage6_algorithm_tools.egg-info → vantage6_algorithm_tools-5.0.0a7}/PKG-INFO +2 -2
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/setup.py +1 -1
- vantage6_algorithm_tools-5.0.0a7/vantage6/algorithm/client/__build__ +1 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/client/__init__.py +5 -21
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/action.py +1 -1
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/data.py +21 -22
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/__init__.py +1 -2
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/mock_client.py +3 -48
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/wrap.py +1 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7/vantage6_algorithm_tools.egg-info}/PKG-INFO +1 -1
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6_algorithm_tools.egg-info/requires.txt +1 -1
- vantage6-algorithm-tools-5.0.0a0/vantage6/algorithm/client/__build__ +0 -1
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/setup.cfg +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/algorithm_module.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_deserialization.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_docker_wrapper.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_serialization.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/client/_version.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/__init__.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/algorithm_client.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/metadata.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/decorator/source_database.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/exceptions.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/preprocessing/__init__.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/preprocessing/functions.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/util.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6/algorithm/tools/wrappers.py +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6_algorithm_tools.egg-info/SOURCES.txt +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6_algorithm_tools.egg-info/dependency_links.txt +0 -0
- {vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/vantage6_algorithm_tools.egg-info/top_level.txt +0 -0
|
@@ -23,7 +23,7 @@ with codecs.open(version_path) as f:
|
|
|
23
23
|
|
|
24
24
|
# setup the package
|
|
25
25
|
setup(
|
|
26
|
-
name="
|
|
26
|
+
name="vantage6_algorithm_tools",
|
|
27
27
|
version=version_ns["__version__"],
|
|
28
28
|
description="Vantage6 algorithm tools",
|
|
29
29
|
long_description=long_description,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Client for the algorithm container to communicate with the vantage6 server."""
|
|
2
2
|
|
|
3
3
|
import jwt
|
|
4
4
|
import json as json_lib
|
|
@@ -339,7 +339,7 @@ class AlgorithmClient(ClientBase):
|
|
|
339
339
|
|
|
340
340
|
def create(
|
|
341
341
|
self,
|
|
342
|
-
input_:
|
|
342
|
+
input_: dict,
|
|
343
343
|
organizations: list[int] = None,
|
|
344
344
|
name: str = "subtask",
|
|
345
345
|
description: str = None,
|
|
@@ -353,8 +353,9 @@ class AlgorithmClient(ClientBase):
|
|
|
353
353
|
|
|
354
354
|
Parameters
|
|
355
355
|
----------
|
|
356
|
-
input_ :
|
|
357
|
-
Input to the task.
|
|
356
|
+
input_ : dict
|
|
357
|
+
Input to the task. This dictionary usually contains the algorithm method
|
|
358
|
+
to call and the arguments to pass to the method.
|
|
358
359
|
organizations : list[int]
|
|
359
360
|
List of organization IDs that should execute the task.
|
|
360
361
|
name: str, optional
|
|
@@ -626,23 +627,6 @@ class AlgorithmClient(ClientBase):
|
|
|
626
627
|
"""
|
|
627
628
|
return self.parent.request(f"collaboration/{self.parent.collaboration_id}")
|
|
628
629
|
|
|
629
|
-
class Node(ClientBase.SubClient):
|
|
630
|
-
"""
|
|
631
|
-
Get information about the node.
|
|
632
|
-
"""
|
|
633
|
-
|
|
634
|
-
def get(self) -> dict:
|
|
635
|
-
"""
|
|
636
|
-
Get the node data.
|
|
637
|
-
|
|
638
|
-
Returns
|
|
639
|
-
-------
|
|
640
|
-
dict
|
|
641
|
-
Dictionary containing data on the node this algorithm is
|
|
642
|
-
running on.
|
|
643
|
-
"""
|
|
644
|
-
return self.parent.request(f"node/{self.parent.node_id}")
|
|
645
|
-
|
|
646
630
|
class Study(ClientBase.SubClient):
|
|
647
631
|
"""
|
|
648
632
|
Get information about the study or studies.
|
|
@@ -67,7 +67,7 @@ def _convert_to_parquet(data: Any) -> pa.Table:
|
|
|
67
67
|
SessionError
|
|
68
68
|
If the DataFrame cannot be converted to a Parquet Table.
|
|
69
69
|
DataTypeError
|
|
70
|
-
If the data extraction function returns an unsupported
|
|
70
|
+
If the data extraction function returns an unsupported dataframe type.
|
|
71
71
|
"""
|
|
72
72
|
info("Converting algorithm output to a Parquet Table.")
|
|
73
73
|
match type(data):
|
|
@@ -7,27 +7,26 @@ from vantage6.common.globals import ContainerEnvNames
|
|
|
7
7
|
from vantage6.algorithm.tools.util import info, error, warn
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def
|
|
10
|
+
def _get_user_dataframes() -> list[str]:
|
|
11
11
|
"""
|
|
12
|
-
Get the database
|
|
12
|
+
Get the database names that the user requested from the environment
|
|
13
13
|
|
|
14
14
|
Returns
|
|
15
15
|
-------
|
|
16
16
|
list[str]
|
|
17
|
-
List of database
|
|
17
|
+
List of database names
|
|
18
18
|
"""
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
return labels.split(",")
|
|
19
|
+
dfs = os.environ[ContainerEnvNames.USER_REQUESTED_DATAFRAMES.value]
|
|
20
|
+
return dfs.split(",")
|
|
22
21
|
|
|
23
22
|
|
|
24
|
-
def
|
|
23
|
+
def _read_df_from_disk(df_name: str) -> pd.DataFrame:
|
|
25
24
|
"""
|
|
26
|
-
Load data from a
|
|
25
|
+
Load data from a dataframe file
|
|
27
26
|
|
|
28
27
|
Parameters
|
|
29
28
|
----------
|
|
30
|
-
|
|
29
|
+
df_name : str
|
|
31
30
|
Label of the database to load
|
|
32
31
|
|
|
33
32
|
Returns
|
|
@@ -35,14 +34,14 @@ def _get_data_from_handle(label: str) -> pd.DataFrame:
|
|
|
35
34
|
pd.DataFrame
|
|
36
35
|
Data from the database
|
|
37
36
|
"""
|
|
38
|
-
# Load the dataframe by the user specified
|
|
37
|
+
# Load the dataframe by the user specified df name. The dataframes are always stored
|
|
39
38
|
# in the session folder, which is set by the vantage6 node. The label is the name of
|
|
40
39
|
# the dataframe file, which is set by the user when creating the task.
|
|
41
40
|
dataframe_folder = os.environ[ContainerEnvNames.SESSION_FOLDER.value]
|
|
42
|
-
|
|
43
|
-
info(f"Using '{
|
|
41
|
+
dataframe_file = os.path.join(dataframe_folder, f"{df_name}.parquet")
|
|
42
|
+
info(f"Using '{dataframe_file}' with dataframe name '{df_name}' as database")
|
|
44
43
|
|
|
45
|
-
return pd.read_parquet(
|
|
44
|
+
return pd.read_parquet(dataframe_file)
|
|
46
45
|
|
|
47
46
|
|
|
48
47
|
def data(number_of_databases: int = 1) -> callable:
|
|
@@ -97,30 +96,30 @@ def data(number_of_databases: int = 1) -> callable:
|
|
|
97
96
|
if mock_data is not None:
|
|
98
97
|
return func(*mock_data, *args, **kwargs)
|
|
99
98
|
|
|
100
|
-
#
|
|
101
|
-
|
|
99
|
+
# get the dataframe names that the user requested
|
|
100
|
+
dataframes = _get_user_dataframes()
|
|
102
101
|
|
|
103
102
|
# check if user provided enough databases
|
|
104
|
-
if len(
|
|
103
|
+
if len(dataframes) < number_of_databases:
|
|
105
104
|
error(
|
|
106
105
|
f"Algorithm requires {number_of_databases} databases "
|
|
107
|
-
f"but only {len(
|
|
106
|
+
f"but only {len(dataframes)} were provided. "
|
|
108
107
|
"Exiting..."
|
|
109
108
|
)
|
|
110
109
|
exit(1)
|
|
111
|
-
elif len(
|
|
110
|
+
elif len(dataframes) > number_of_databases:
|
|
112
111
|
warn(
|
|
113
112
|
f"Algorithm requires only {number_of_databases} databases"
|
|
114
|
-
f", but {len(
|
|
113
|
+
f", but {len(dataframes)} were provided. Using the "
|
|
115
114
|
f"first {number_of_databases} databases."
|
|
116
115
|
)
|
|
117
116
|
|
|
118
117
|
for i in range(number_of_databases):
|
|
119
118
|
|
|
120
|
-
|
|
119
|
+
dataframe_name = dataframes[i]
|
|
121
120
|
# read the data from the database
|
|
122
|
-
info("Reading
|
|
123
|
-
data_ =
|
|
121
|
+
info("Reading dataframe")
|
|
122
|
+
data_ = _read_df_from_disk(dataframe_name)
|
|
124
123
|
|
|
125
124
|
# add the data to the arguments
|
|
126
125
|
args = (data_, *args)
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
"""
|
|
1
|
+
"""Algorithm tools to support the development of algorithms for the vantage6 platform."""
|
|
3
2
|
|
|
4
3
|
# make sure the version is available
|
|
5
4
|
from vantage6.algorithm.client._version import __version__ # noqa: F401
|
|
@@ -104,9 +104,9 @@ class MockAlgorithmClient:
|
|
|
104
104
|
self.organizations_with_data.append(org_id)
|
|
105
105
|
org_data = []
|
|
106
106
|
for dataset in org_datasets:
|
|
107
|
-
|
|
108
|
-
if isinstance(
|
|
109
|
-
df =
|
|
107
|
+
db = dataset.get("database")
|
|
108
|
+
if isinstance(db, pd.DataFrame):
|
|
109
|
+
df = db
|
|
110
110
|
else:
|
|
111
111
|
df = load_data(
|
|
112
112
|
database_uri=dataset.get("database"),
|
|
@@ -542,51 +542,6 @@ class MockAlgorithmClient:
|
|
|
542
542
|
"organizations": f"/api/organization?collaboration_id={collab_id}",
|
|
543
543
|
}
|
|
544
544
|
|
|
545
|
-
class Node(SubClient):
|
|
546
|
-
"""
|
|
547
|
-
Node subclient for the MockAlgorithmClient
|
|
548
|
-
"""
|
|
549
|
-
|
|
550
|
-
def get(self, is_online: bool = True) -> dict:
|
|
551
|
-
"""
|
|
552
|
-
Get mocked node
|
|
553
|
-
|
|
554
|
-
Parameters
|
|
555
|
-
----------
|
|
556
|
-
is_online : bool
|
|
557
|
-
Whether the node is online or not. Default True.
|
|
558
|
-
|
|
559
|
-
Returns
|
|
560
|
-
-------
|
|
561
|
-
dict
|
|
562
|
-
A mocked node.
|
|
563
|
-
"""
|
|
564
|
-
node_id = self.parent.node_id
|
|
565
|
-
collab_id = self.parent.collaboration_id
|
|
566
|
-
return {
|
|
567
|
-
"id": node_id,
|
|
568
|
-
"name": "mock-node",
|
|
569
|
-
"status": (
|
|
570
|
-
AuthStatus.ONLINE.value if is_online else AuthStatus.OFFLINE.value
|
|
571
|
-
),
|
|
572
|
-
"ip": "1.2.3.4",
|
|
573
|
-
"config": {
|
|
574
|
-
"key": "value",
|
|
575
|
-
},
|
|
576
|
-
"collaboration": {
|
|
577
|
-
"id": collab_id,
|
|
578
|
-
"link": f"/api/collaboration/{collab_id}",
|
|
579
|
-
"methods": ["DELETE", "PATCH", "GET"],
|
|
580
|
-
},
|
|
581
|
-
"last_seen": "2021-01-01T00:00:00.000000",
|
|
582
|
-
"type": "node",
|
|
583
|
-
"organization": {
|
|
584
|
-
"id": node_id,
|
|
585
|
-
"link": f"/api/organization/{node_id}",
|
|
586
|
-
"methods": ["GET", "PATCH"],
|
|
587
|
-
},
|
|
588
|
-
}
|
|
589
|
-
|
|
590
545
|
# TODO implement the get_addresses method before using this part
|
|
591
546
|
# class VPN(SubClient):
|
|
592
547
|
# """
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0
|
|
File without changes
|
{vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/algorithm_module.py
RENAMED
|
File without changes
|
{vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_deserialization.py
RENAMED
|
File without changes
|
{vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_docker_wrapper.py
RENAMED
|
File without changes
|
{vantage6-algorithm-tools-5.0.0a0 → vantage6_algorithm_tools-5.0.0a7}/tests/test_serialization.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|