vantage6-algorithm-tools 5.0.0b1__tar.gz → 5.0.0b4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/PKG-INFO +2 -2
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/pyproject.toml +2 -2
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/algorithm/test___init__.py +3 -5
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_network.py +8 -8
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_node.py +5 -5
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_server.py +16 -18
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/client/__init__.py +35 -50
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/algorithm_client.py +3 -3
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/client.py +30 -28
- vantage6_algorithm_tools-5.0.0b1/vantage6/algorithm/mock/server.py → vantage6_algorithm_tools-5.0.0b4/vantage6/algorithm/mock/hq.py +15 -15
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/network.py +6 -6
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/node.py +4 -6
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/tools/wrap.py +3 -3
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/.gitignore +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/Makefile +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/README.md +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_deserialization.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_docker_wrapper.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_preprocessing.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_serialization.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/algorithm/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/algorithm_module.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/base/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_algorithm_client.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_user_client.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_util.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/data_extraction/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/data_extraction/mock_extract.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/action.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/data.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/metadata.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/decorator/ohdsi.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/globals.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/mock/util.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/aggregation.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/algorithm_json_data.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/column.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/datetime.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/encoding.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/preprocessing/filtering.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/tools/__init__.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/tools/error_handling.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/tools/exceptions.py +0 -0
- {vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/vantage6/algorithm/tools/util.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vantage6-algorithm-tools
|
|
3
|
-
Version: 5.0.
|
|
3
|
+
Version: 5.0.0b4
|
|
4
4
|
Summary: Vantage6 algorithm tools
|
|
5
5
|
Author: Vantage6 Team
|
|
6
6
|
Maintainer-email: Frank Martin <f.martin@iknl.nl>, Bart van Beusekom <b.vanbeusekom@iknl.nl>
|
|
@@ -14,7 +14,7 @@ Requires-Dist: pyfiglet==1.0.4
|
|
|
14
14
|
Requires-Dist: pyjwt==2.9.0
|
|
15
15
|
Requires-Dist: sparqlwrapper>=2.0.0
|
|
16
16
|
Requires-Dist: sqlalchemy>=1.4.46
|
|
17
|
-
Requires-Dist: vantage6-common==5.0.
|
|
17
|
+
Requires-Dist: vantage6-common==5.0.0b4
|
|
18
18
|
Provides-Extra: dev
|
|
19
19
|
Requires-Dist: black; extra == 'dev'
|
|
20
20
|
Requires-Dist: coverage==7.10.2; extra == 'dev'
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vantage6-algorithm-tools"
|
|
7
|
-
version = "5.0.
|
|
7
|
+
version = "5.0.0b4"
|
|
8
8
|
description = "Vantage6 algorithm tools"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -27,7 +27,7 @@ dependencies = [
|
|
|
27
27
|
"pyfiglet==1.0.4",
|
|
28
28
|
"SPARQLWrapper>=2.0.0",
|
|
29
29
|
"sqlalchemy>=1.4.46",
|
|
30
|
-
"vantage6-common==5.0.
|
|
30
|
+
"vantage6-common==5.0.0b4",
|
|
31
31
|
]
|
|
32
32
|
|
|
33
33
|
[project.optional-dependencies]
|
|
@@ -29,7 +29,7 @@ class TestAlgorithmClient(unittest.TestCase):
|
|
|
29
29
|
dummy_token = jwt.encode(payload, key="", algorithm=None)
|
|
30
30
|
self.client = AlgorithmClient(
|
|
31
31
|
token=dummy_token,
|
|
32
|
-
|
|
32
|
+
hq_url="http://dummy_host:1234",
|
|
33
33
|
)
|
|
34
34
|
self.client.parent = MagicMock()
|
|
35
35
|
self.client.parent.request = MagicMock()
|
|
@@ -89,9 +89,7 @@ class TestAlgorithmClient(unittest.TestCase):
|
|
|
89
89
|
self.assertEqual(results[0], {"foo": "bar"})
|
|
90
90
|
|
|
91
91
|
@patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
|
|
92
|
-
@patch(
|
|
93
|
-
"vantage6.common.client.client_base.ClientBase._download_run_data_from_server"
|
|
94
|
-
)
|
|
92
|
+
@patch("vantage6.common.client.client_base.ClientBase._download_run_data_from_hq")
|
|
95
93
|
def test_result_from_task_azure(
|
|
96
94
|
self, mock_download_run_data, mock_multi_page_request
|
|
97
95
|
):
|
|
@@ -100,7 +98,7 @@ class TestAlgorithmClient(unittest.TestCase):
|
|
|
100
98
|
self.client.result.parent, "_multi_page_request"
|
|
101
99
|
) as mock_multi_page_request,
|
|
102
100
|
patch.object(
|
|
103
|
-
self.client.result.parent, "
|
|
101
|
+
self.client.result.parent, "_download_run_data_from_hq"
|
|
104
102
|
) as mock_download_run_data,
|
|
105
103
|
):
|
|
106
104
|
# Simulate a result where blob_storage_used is True
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_network.py
RENAMED
|
@@ -6,8 +6,8 @@ import pandas as pd
|
|
|
6
6
|
from vantage6.algorithm.data_extraction.mock_extract import MockDatabaseType
|
|
7
7
|
from vantage6.algorithm.mock import MockNetwork
|
|
8
8
|
from vantage6.algorithm.mock.client import MockUserClient
|
|
9
|
+
from vantage6.algorithm.mock.hq import MockHQ
|
|
9
10
|
from vantage6.algorithm.mock.node import MockNode
|
|
10
|
-
from vantage6.algorithm.mock.server import MockServer
|
|
11
11
|
|
|
12
12
|
TEST_ALGORITHM_NAME = "test_algorithm"
|
|
13
13
|
DATASET_LABEL = "dataset_1"
|
|
@@ -58,8 +58,8 @@ class TestMockNetworkDataframe(TestCase):
|
|
|
58
58
|
|
|
59
59
|
def test_attributes(self):
|
|
60
60
|
"""Test if properties are properly initialized"""
|
|
61
|
-
# Check the
|
|
62
|
-
self.assertIsInstance(self.network.
|
|
61
|
+
# Check the types
|
|
62
|
+
self.assertIsInstance(self.network.hq, MockHQ)
|
|
63
63
|
self.assertIsInstance(self.network.user_client, MockUserClient)
|
|
64
64
|
|
|
65
65
|
self.assertEqual(len(self.network.nodes), 2)
|
|
@@ -79,11 +79,11 @@ class TestMockNetworkDataframe(TestCase):
|
|
|
79
79
|
self.assertIsInstance(self.network.get_node(1), MockNode)
|
|
80
80
|
self.assertIsInstance(self.network.get_node(2), MockNode)
|
|
81
81
|
|
|
82
|
-
def
|
|
83
|
-
"""Test if
|
|
84
|
-
self.assertEqual(len(self.network.
|
|
85
|
-
self.assertEqual(len(self.network.
|
|
86
|
-
self.assertEqual(len(self.network.
|
|
82
|
+
def test_hq_initialization(self):
|
|
83
|
+
"""Test if HQ is properly initialized"""
|
|
84
|
+
self.assertEqual(len(self.network.hq.tasks), 0)
|
|
85
|
+
self.assertEqual(len(self.network.hq.runs), 0)
|
|
86
|
+
self.assertEqual(len(self.network.hq.results), 0)
|
|
87
87
|
|
|
88
88
|
def test_node_initialization(self):
|
|
89
89
|
"""Test if node is properly initialized"""
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_node.py
RENAMED
|
@@ -50,9 +50,9 @@ class TestMockNodeDataframe(TestCase):
|
|
|
50
50
|
def test_simulate_task_run(self):
|
|
51
51
|
"""Test if task run simulation works properly"""
|
|
52
52
|
|
|
53
|
-
# Mock the
|
|
54
|
-
self.network.
|
|
55
|
-
self.network.
|
|
53
|
+
# Mock the HQ's get_label_for_df_id to return the correct label
|
|
54
|
+
self.network.hq = MagicMock()
|
|
55
|
+
self.network.hq.get_label_for_df_id = MagicMock(return_value=LABEL_1)
|
|
56
56
|
|
|
57
57
|
# Create a mock method function
|
|
58
58
|
mock_method = MagicMock()
|
|
@@ -186,8 +186,8 @@ class TestMockNodeDataframe(TestCase):
|
|
|
186
186
|
|
|
187
187
|
def test_task_env_vars(self):
|
|
188
188
|
"""Test if _task_env_vars returns correct environment variables"""
|
|
189
|
-
# Mock the
|
|
190
|
-
self.network.
|
|
189
|
+
# Mock the HQ tasks to simulate task count
|
|
190
|
+
self.network.hq.tasks = [1, 2, 3] # 3 existing tasks
|
|
191
191
|
|
|
192
192
|
# Call _task_env_vars with test values
|
|
193
193
|
action = AlgorithmStepType.FEDERATED_COMPUTE
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_server.py
RENAMED
|
@@ -2,10 +2,10 @@ import json
|
|
|
2
2
|
from unittest import TestCase
|
|
3
3
|
|
|
4
4
|
from vantage6.algorithm.mock import MockNetwork
|
|
5
|
-
from vantage6.algorithm.mock.
|
|
5
|
+
from vantage6.algorithm.mock.hq import MockHQ
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
class
|
|
8
|
+
class TestMockHQ(TestCase):
|
|
9
9
|
def setUp(self):
|
|
10
10
|
"""Set up test fixtures"""
|
|
11
11
|
self.network = MockNetwork(
|
|
@@ -21,23 +21,23 @@ class TestMockServer(TestCase):
|
|
|
21
21
|
],
|
|
22
22
|
collaboration_id=1,
|
|
23
23
|
)
|
|
24
|
-
self.
|
|
24
|
+
self.hq = MockHQ(self.network)
|
|
25
25
|
|
|
26
26
|
def test_initialization(self):
|
|
27
|
-
"""Test if
|
|
28
|
-
self.assertEqual(self.
|
|
29
|
-
self.assertEqual(self.
|
|
30
|
-
self.assertEqual(self.
|
|
31
|
-
self.assertEqual(len(self.
|
|
32
|
-
self.assertEqual(len(self.
|
|
33
|
-
self.assertEqual(len(self.
|
|
27
|
+
"""Test if HQ is properly initialized"""
|
|
28
|
+
self.assertEqual(self.hq.network.collaboration_id, 1)
|
|
29
|
+
self.assertEqual(self.hq.session_id, 1)
|
|
30
|
+
self.assertEqual(self.hq.study_id, 1)
|
|
31
|
+
self.assertEqual(len(self.hq.tasks), 0)
|
|
32
|
+
self.assertEqual(len(self.hq.runs), 0)
|
|
33
|
+
self.assertEqual(len(self.hq.results), 0)
|
|
34
34
|
|
|
35
35
|
def test_save_result(self):
|
|
36
36
|
"""Test if results are properly saved"""
|
|
37
37
|
test_result = {"test": "data"}
|
|
38
|
-
result = self.
|
|
38
|
+
result = self.hq.save_result(test_result, task_id=1)
|
|
39
39
|
|
|
40
|
-
self.assertEqual(len(self.
|
|
40
|
+
self.assertEqual(len(self.hq.results), 1)
|
|
41
41
|
self.assertEqual(result["id"], 1)
|
|
42
42
|
self.assertEqual(json.loads(result["result"]), test_result)
|
|
43
43
|
self.assertEqual(result["task"]["id"], 1)
|
|
@@ -45,11 +45,9 @@ class TestMockServer(TestCase):
|
|
|
45
45
|
def test_save_run(self):
|
|
46
46
|
"""Test if runs are properly saved"""
|
|
47
47
|
test_args = {"arg1": "value1"}
|
|
48
|
-
run = self.
|
|
49
|
-
arguments=test_args, task_id=1, result_id=1, org_id=1
|
|
50
|
-
)
|
|
48
|
+
run = self.hq.save_run(arguments=test_args, task_id=1, result_id=1, org_id=1)
|
|
51
49
|
|
|
52
|
-
self.assertEqual(len(self.
|
|
50
|
+
self.assertEqual(len(self.hq.runs), 1)
|
|
53
51
|
self.assertEqual(run["id"], 1)
|
|
54
52
|
self.assertEqual(json.loads(run["arguments"]), test_args)
|
|
55
53
|
self.assertEqual(run["task"]["id"], 1)
|
|
@@ -59,14 +57,14 @@ class TestMockServer(TestCase):
|
|
|
59
57
|
def test_save_task(self):
|
|
60
58
|
"""Test if tasks are properly saved"""
|
|
61
59
|
test_databases = [{"name": "test_db", "uri": "test_uri"}]
|
|
62
|
-
task = self.
|
|
60
|
+
task = self.hq.save_task(
|
|
63
61
|
name="test_task",
|
|
64
62
|
description="test description",
|
|
65
63
|
databases=test_databases,
|
|
66
64
|
init_organization_id=1,
|
|
67
65
|
)
|
|
68
66
|
|
|
69
|
-
self.assertEqual(len(self.
|
|
67
|
+
self.assertEqual(len(self.hq.tasks), 1)
|
|
70
68
|
self.assertEqual(task["id"], 1)
|
|
71
69
|
self.assertEqual(task["name"], "test_task")
|
|
72
70
|
self.assertEqual(task["description"], "test description")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Client for the algorithm container to communicate with the vantage6
|
|
1
|
+
"""Client for the algorithm container to communicate with the vantage6 HQ."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from typing import Any
|
|
@@ -16,29 +16,27 @@ from vantage6.algorithm.tools import __version__ # noqa: F401
|
|
|
16
16
|
|
|
17
17
|
class AlgorithmClient(ClientBase):
|
|
18
18
|
"""
|
|
19
|
-
Interface to communicate between the algorithm container and
|
|
20
|
-
server
|
|
21
|
-
|
|
22
|
-
An algorithm container cannot communicate directly to
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
the local proxy server - however, the interface looks identical to make
|
|
29
|
-
it easier to use.
|
|
19
|
+
Interface to communicate between the algorithm container and HQ via a local proxy
|
|
20
|
+
server.
|
|
21
|
+
|
|
22
|
+
An algorithm container cannot communicate directly to HQ as it has no internet
|
|
23
|
+
connection. The algorithm can, however, talk to a local proxy server which has
|
|
24
|
+
interface to HQ. This way we make sure that the algorithm container does not share
|
|
25
|
+
details with others, and we also can encrypt the results for a specific receiver.
|
|
26
|
+
Thus, this not a interface to HQ but to the local proxy server - however, the
|
|
27
|
+
interface looks identical to make it easier to use.
|
|
30
28
|
|
|
31
29
|
Parameters
|
|
32
30
|
----------
|
|
33
31
|
token: str
|
|
34
32
|
JWT (container) token, generated by the node the algorithm container
|
|
35
33
|
runs on
|
|
36
|
-
|
|
37
|
-
The url of
|
|
34
|
+
hq_url : str
|
|
35
|
+
The url of HQ to connect to.
|
|
38
36
|
"""
|
|
39
37
|
|
|
40
|
-
def __init__(self, token: str,
|
|
41
|
-
super().__init__(
|
|
38
|
+
def __init__(self, token: str, hq_url: str) -> None:
|
|
39
|
+
super().__init__(hq_url=hq_url)
|
|
42
40
|
|
|
43
41
|
# obtain the identity from the token
|
|
44
42
|
jwt_payload = jwt.decode(
|
|
@@ -74,7 +72,7 @@ class AlgorithmClient(ClientBase):
|
|
|
74
72
|
|
|
75
73
|
def request(self, *args, **kwargs) -> dict:
|
|
76
74
|
"""
|
|
77
|
-
Make a request to
|
|
75
|
+
Make a request to HQ. This overwrites the parent
|
|
78
76
|
function so that containers will not try to refresh their token, which
|
|
79
77
|
they would be unable to do.
|
|
80
78
|
|
|
@@ -86,7 +84,7 @@ class AlgorithmClient(ClientBase):
|
|
|
86
84
|
Returns
|
|
87
85
|
-------
|
|
88
86
|
dict
|
|
89
|
-
Response from
|
|
87
|
+
Response from HQ.
|
|
90
88
|
"""
|
|
91
89
|
return super().request(*args, **kwargs, retry=False)
|
|
92
90
|
|
|
@@ -119,15 +117,14 @@ class AlgorithmClient(ClientBase):
|
|
|
119
117
|
|
|
120
118
|
def wait_for_results(self, task_id: int, interval: float = 1) -> list:
|
|
121
119
|
"""
|
|
122
|
-
Poll
|
|
123
|
-
them
|
|
120
|
+
Poll HQ until results are available and then return them.
|
|
124
121
|
|
|
125
122
|
Parameters
|
|
126
123
|
----------
|
|
127
124
|
task_id: int
|
|
128
125
|
ID of the task for which the results should be obtained.
|
|
129
126
|
interval: float
|
|
130
|
-
Interval in seconds to wait between checking
|
|
127
|
+
Interval in seconds to wait between checking HQ for results.
|
|
131
128
|
|
|
132
129
|
Returns
|
|
133
130
|
-------
|
|
@@ -141,8 +138,7 @@ class AlgorithmClient(ClientBase):
|
|
|
141
138
|
|
|
142
139
|
def _multi_page_request(self, endpoint: str, params: dict = None) -> dict:
|
|
143
140
|
"""
|
|
144
|
-
Make multiple requests to
|
|
145
|
-
of results.
|
|
141
|
+
Make multiple requests to HQ to get all pages of a list of results.
|
|
146
142
|
|
|
147
143
|
Parameters
|
|
148
144
|
----------
|
|
@@ -154,7 +150,7 @@ class AlgorithmClient(ClientBase):
|
|
|
154
150
|
Returns
|
|
155
151
|
-------
|
|
156
152
|
dict
|
|
157
|
-
Response from
|
|
153
|
+
Response from HQ.
|
|
158
154
|
"""
|
|
159
155
|
if params is None:
|
|
160
156
|
params = {}
|
|
@@ -176,14 +172,11 @@ class AlgorithmClient(ClientBase):
|
|
|
176
172
|
class Run(ClientBase.SubClient):
|
|
177
173
|
"""
|
|
178
174
|
Algorithm Run client for the algorithm container.
|
|
179
|
-
|
|
180
|
-
This client is used to obtain algorithm runs of tasks with the same
|
|
181
|
-
job_id from the central server.
|
|
182
175
|
"""
|
|
183
176
|
|
|
184
177
|
def get(self, id_) -> dict:
|
|
185
178
|
"""
|
|
186
|
-
Obtain a specific algorithm run from
|
|
179
|
+
Obtain a specific algorithm run from HQ.
|
|
187
180
|
|
|
188
181
|
Parameters
|
|
189
182
|
----------
|
|
@@ -199,11 +192,10 @@ class AlgorithmClient(ClientBase):
|
|
|
199
192
|
|
|
200
193
|
def from_task(self, task_id: int) -> list:
|
|
201
194
|
"""
|
|
202
|
-
Obtain algorithm runs from a specific task at
|
|
195
|
+
Obtain algorithm runs from a specific task at HQ.
|
|
203
196
|
|
|
204
197
|
Containers are allowed to obtain the runs of their children
|
|
205
|
-
(having the same job_id at
|
|
206
|
-
at te central server.
|
|
198
|
+
(having the same job_id at HQ). The permissions are checked at HQ.
|
|
207
199
|
|
|
208
200
|
Note that the returned results are not decrypted. The algorithm is
|
|
209
201
|
responsible for decrypting the results.
|
|
@@ -227,13 +219,11 @@ class AlgorithmClient(ClientBase):
|
|
|
227
219
|
class Result(ClientBase.SubClient):
|
|
228
220
|
"""
|
|
229
221
|
Result client for the algorithm container.
|
|
230
|
-
|
|
231
|
-
This client is used to get results from the central server.
|
|
232
222
|
"""
|
|
233
223
|
|
|
234
224
|
def get(self, id_: int) -> Any:
|
|
235
225
|
"""
|
|
236
|
-
Obtain a specific result from
|
|
226
|
+
Obtain a specific result from HQ.
|
|
237
227
|
|
|
238
228
|
Parameters
|
|
239
229
|
----------
|
|
@@ -263,17 +253,16 @@ class AlgorithmClient(ClientBase):
|
|
|
263
253
|
|
|
264
254
|
def from_task(self, task_id: int) -> list[Any]:
|
|
265
255
|
"""
|
|
266
|
-
Obtain results from a specific task at
|
|
256
|
+
Obtain results from a specific task at HQ.
|
|
267
257
|
|
|
268
258
|
Containers are allowed to obtain the results of their children
|
|
269
|
-
(having the same job_id at
|
|
270
|
-
at te central server.
|
|
259
|
+
(having the same job_id at HQ). The permissions are checked at HQ.
|
|
271
260
|
|
|
272
261
|
Results are decrypted by the proxy server and decoded here before
|
|
273
262
|
returning them to the algorithm.
|
|
274
263
|
|
|
275
264
|
If blob storage was used to store the results, the results
|
|
276
|
-
will be streamed from blob storage, through
|
|
265
|
+
will be streamed from blob storage, through HQ and proxy,
|
|
277
266
|
and decryption will take place on a chunk-by-chunk basis.
|
|
278
267
|
|
|
279
268
|
Parameters
|
|
@@ -294,9 +283,7 @@ class AlgorithmClient(ClientBase):
|
|
|
294
283
|
return None
|
|
295
284
|
try:
|
|
296
285
|
if run.get("blob_storage_used", False):
|
|
297
|
-
run_data = self.parent.
|
|
298
|
-
result_data
|
|
299
|
-
)
|
|
286
|
+
run_data = self.parent._download_run_data_from_hq(result_data)
|
|
300
287
|
return json.loads(run_data.decode(STRING_ENCODING))
|
|
301
288
|
else:
|
|
302
289
|
return json.loads(base64s_to_bytes(result_data).decode())
|
|
@@ -322,7 +309,7 @@ class AlgorithmClient(ClientBase):
|
|
|
322
309
|
|
|
323
310
|
def get(self, task_id: int) -> dict:
|
|
324
311
|
"""
|
|
325
|
-
Retrieve a task at
|
|
312
|
+
Retrieve a task at HQ.
|
|
326
313
|
|
|
327
314
|
Parameters
|
|
328
315
|
----------
|
|
@@ -345,15 +332,13 @@ class AlgorithmClient(ClientBase):
|
|
|
345
332
|
description: str = None,
|
|
346
333
|
) -> dict:
|
|
347
334
|
"""
|
|
348
|
-
Create a new (child) task at
|
|
335
|
+
Create a new (child) task at HQ.
|
|
349
336
|
|
|
350
|
-
Containers are allowed to create child tasks (having the
|
|
351
|
-
same job_id) at the central server. The docker image must
|
|
352
|
-
be the same as the docker image of this container self.
|
|
337
|
+
Containers are allowed to create child tasks (having the same job_id) at HQ.
|
|
353
338
|
|
|
354
|
-
If blob storage is configured at
|
|
355
|
-
|
|
356
|
-
|
|
339
|
+
If blob storage is configured at HQ, the input arguments data will be
|
|
340
|
+
uploaded to blob storage and a UUID reference will be passed as a result
|
|
341
|
+
instead.
|
|
357
342
|
|
|
358
343
|
Parameters
|
|
359
344
|
----------
|
|
@@ -404,7 +389,7 @@ class AlgorithmClient(ClientBase):
|
|
|
404
389
|
"Blob store is enabled, uploading arguments data to blob "
|
|
405
390
|
"storage."
|
|
406
391
|
)
|
|
407
|
-
arguments_uuid = self.parent.
|
|
392
|
+
arguments_uuid = self.parent._upload_run_data_to_hq(
|
|
408
393
|
serialized_arguments, pub_key=pub_key
|
|
409
394
|
)
|
|
410
395
|
org_arguments = arguments_uuid
|
|
@@ -19,7 +19,7 @@ def _algorithm_client() -> Callable:
|
|
|
19
19
|
|
|
20
20
|
By adding @algorithm_client to a function, the ``algorithm_client``
|
|
21
21
|
argument will be added to the front of the argument list. This client can
|
|
22
|
-
be used to communicate with the
|
|
22
|
+
be used to communicate with the vantage6 hub.
|
|
23
23
|
|
|
24
24
|
There is one reserved argument `mock_client` in the function to be
|
|
25
25
|
decorated. If this argument is provided, the decorator will add this
|
|
@@ -71,12 +71,12 @@ def _algorithm_client() -> Callable:
|
|
|
71
71
|
)
|
|
72
72
|
exit(1)
|
|
73
73
|
|
|
74
|
-
# read
|
|
74
|
+
# read HQ address from the environment
|
|
75
75
|
host = os.environ[ContainerEnvNames.HOST.value]
|
|
76
76
|
port = os.environ[ContainerEnvNames.PORT.value]
|
|
77
77
|
api_path = os.environ[ContainerEnvNames.API_PATH.value]
|
|
78
78
|
|
|
79
|
-
client = AlgorithmClient(token=token,
|
|
79
|
+
client = AlgorithmClient(token=token, hq_url=f"{host}:{port}{api_path}")
|
|
80
80
|
return func(client, *args, **kwargs)
|
|
81
81
|
|
|
82
82
|
# set attribute that this function is wrapped in an algorithm client
|
|
@@ -34,15 +34,14 @@ class MockBaseClient:
|
|
|
34
34
|
"_access_token",
|
|
35
35
|
"_ClientBase__auth_url",
|
|
36
36
|
"_ClientBase__check_algorithm_store_valid",
|
|
37
|
-
"
|
|
37
|
+
"_ClientBase__hq_url",
|
|
38
38
|
"_decrypt_run_data",
|
|
39
39
|
"_decrypt_field",
|
|
40
|
-
"
|
|
40
|
+
"_download_run_data_from_hq",
|
|
41
41
|
"_fetch_and_decrypt_run_data",
|
|
42
42
|
"_multi_page_request",
|
|
43
43
|
"_refresh_token",
|
|
44
|
-
"
|
|
45
|
-
"_upload_run_data_to_server",
|
|
44
|
+
"_upload_run_data_to_hq",
|
|
46
45
|
"auth_url",
|
|
47
46
|
"authenticate",
|
|
48
47
|
"check_if_blob_store_enabled",
|
|
@@ -57,7 +56,7 @@ class MockBaseClient:
|
|
|
57
56
|
"node_id",
|
|
58
57
|
"obtain_new_token",
|
|
59
58
|
"request",
|
|
60
|
-
"
|
|
59
|
+
"hq_url",
|
|
61
60
|
"session_id",
|
|
62
61
|
"setup_encryption",
|
|
63
62
|
"store_id",
|
|
@@ -168,7 +167,7 @@ class MockBaseClient:
|
|
|
168
167
|
dict
|
|
169
168
|
Dictionary containing study data.
|
|
170
169
|
"""
|
|
171
|
-
return self.parent.network.
|
|
170
|
+
return self.parent.network.hq.study
|
|
172
171
|
|
|
173
172
|
class Task(SubClient):
|
|
174
173
|
"""
|
|
@@ -238,7 +237,7 @@ class MockBaseClient:
|
|
|
238
237
|
if not databases:
|
|
239
238
|
databases = []
|
|
240
239
|
|
|
241
|
-
task = self.parent.network.
|
|
240
|
+
task = self.parent.network.hq.save_task(
|
|
242
241
|
init_organization_id=self.parent.organization_id,
|
|
243
242
|
name=name,
|
|
244
243
|
description=description,
|
|
@@ -277,10 +276,8 @@ class MockBaseClient:
|
|
|
277
276
|
traceback.print_exc()
|
|
278
277
|
exit(1)
|
|
279
278
|
|
|
280
|
-
result_response = self.parent.network.
|
|
281
|
-
|
|
282
|
-
)
|
|
283
|
-
self.parent.network.server.save_run(
|
|
279
|
+
result_response = self.parent.network.hq.save_result(result, task["id"])
|
|
280
|
+
self.parent.network.hq.save_run(
|
|
284
281
|
arguments, task["id"], result_response["id"], org_id
|
|
285
282
|
)
|
|
286
283
|
|
|
@@ -311,7 +308,7 @@ class MockBaseClient:
|
|
|
311
308
|
dict
|
|
312
309
|
A mocked run.
|
|
313
310
|
"""
|
|
314
|
-
for run in self.parent.network.
|
|
311
|
+
for run in self.parent.network.hq.runs:
|
|
315
312
|
if run.get("id") == id_:
|
|
316
313
|
return run
|
|
317
314
|
return {"msg": f"Could not find run with id {id_}"}
|
|
@@ -331,7 +328,7 @@ class MockBaseClient:
|
|
|
331
328
|
A list of mocked runs.
|
|
332
329
|
"""
|
|
333
330
|
runs = []
|
|
334
|
-
for run in self.parent.network.
|
|
331
|
+
for run in self.parent.network.hq.runs:
|
|
335
332
|
if run.get("task").get("id") == task_id:
|
|
336
333
|
runs.append(run)
|
|
337
334
|
return runs
|
|
@@ -355,7 +352,7 @@ class MockBaseClient:
|
|
|
355
352
|
Any
|
|
356
353
|
A mocked result.
|
|
357
354
|
"""
|
|
358
|
-
for result in self.network.parent.
|
|
355
|
+
for result in self.network.parent.hq.results:
|
|
359
356
|
if result.get("id") == id_:
|
|
360
357
|
return json.loads(result.get("result"))
|
|
361
358
|
return {"msg": f"Could not find result with id {id_}"}
|
|
@@ -375,7 +372,7 @@ class MockBaseClient:
|
|
|
375
372
|
The results of the task.
|
|
376
373
|
"""
|
|
377
374
|
results = []
|
|
378
|
-
for result in self.parent.network.
|
|
375
|
+
for result in self.parent.network.hq.results:
|
|
379
376
|
if result.get("task").get("id") == task_id:
|
|
380
377
|
results.append(json.loads(result.get("result")))
|
|
381
378
|
return results
|
|
@@ -477,7 +474,7 @@ class MockBaseClient:
|
|
|
477
474
|
dict
|
|
478
475
|
A mocked collaboration.
|
|
479
476
|
"""
|
|
480
|
-
id_ = self.parent.network.
|
|
477
|
+
id_ = self.parent.network.hq.collaboration_id
|
|
481
478
|
return {
|
|
482
479
|
"id": id_,
|
|
483
480
|
"name": "mock-collaboration",
|
|
@@ -530,6 +527,11 @@ class MockUserClient(MockBaseClient):
|
|
|
530
527
|
"setup_collaboration",
|
|
531
528
|
"User",
|
|
532
529
|
"Util",
|
|
530
|
+
"_try_open_browser_for_device_flow",
|
|
531
|
+
"_get_device_flow_endpoints",
|
|
532
|
+
"_poll_device_token",
|
|
533
|
+
"_start_device_authorization",
|
|
534
|
+
"_finalize_successful_authentication",
|
|
533
535
|
]
|
|
534
536
|
)
|
|
535
537
|
|
|
@@ -545,7 +547,7 @@ class MockUserClient(MockBaseClient):
|
|
|
545
547
|
"""
|
|
546
548
|
Get dataframe by ID
|
|
547
549
|
"""
|
|
548
|
-
for dataframe in self.parent.network.
|
|
550
|
+
for dataframe in self.parent.network.hq.dataframes:
|
|
549
551
|
if dataframe.get("id") == id_:
|
|
550
552
|
return dataframe
|
|
551
553
|
return {"msg": f"Could not find dataframe with id {id_}"}
|
|
@@ -561,7 +563,7 @@ class MockUserClient(MockBaseClient):
|
|
|
561
563
|
if not arguments:
|
|
562
564
|
arguments = {}
|
|
563
565
|
|
|
564
|
-
task = self.parent.network.
|
|
566
|
+
task = self.parent.network.hq.save_task(
|
|
565
567
|
init_organization_id=self.parent.organization_id,
|
|
566
568
|
name=name,
|
|
567
569
|
description=f"Mock dataframe creation for {label}",
|
|
@@ -594,12 +596,12 @@ class MockUserClient(MockBaseClient):
|
|
|
594
596
|
|
|
595
597
|
# In case of a dataframe we do not store a result, as the dataframe
|
|
596
598
|
# creation on the node is the result of this action.
|
|
597
|
-
result_response = self.parent.network.
|
|
598
|
-
self.parent.network.
|
|
599
|
+
result_response = self.parent.network.hq.save_result({}, task["id"])
|
|
600
|
+
self.parent.network.hq.save_run(
|
|
599
601
|
arguments, task["id"], result_response["id"], org_id
|
|
600
602
|
)
|
|
601
603
|
|
|
602
|
-
dataframe = self.parent.network.
|
|
604
|
+
dataframe = self.parent.network.hq.save_dataframe(
|
|
603
605
|
name=name,
|
|
604
606
|
dataframes=dataframes,
|
|
605
607
|
source_db_label=label,
|
|
@@ -611,12 +613,12 @@ class MockUserClient(MockBaseClient):
|
|
|
611
613
|
self, id_: int, image: str, method: str, arguments: dict
|
|
612
614
|
) -> dict:
|
|
613
615
|
""" """
|
|
614
|
-
dataframe = self.parent.network.
|
|
616
|
+
dataframe = self.parent.network.hq.get_dataframe(id_)
|
|
615
617
|
data_frame_name = dataframe.get("name")
|
|
616
618
|
if not dataframe or not data_frame_name:
|
|
617
619
|
return {"msg": f"An error occurred while fetching dataframe {id_}"}
|
|
618
620
|
|
|
619
|
-
task = self.parent.network.
|
|
621
|
+
task = self.parent.network.hq.save_task(
|
|
620
622
|
init_organization_id=self.parent.organization_id,
|
|
621
623
|
name=f"Preprocess {data_frame_name}",
|
|
622
624
|
description=f"Preprocess {data_frame_name}",
|
|
@@ -644,18 +646,18 @@ class MockUserClient(MockBaseClient):
|
|
|
644
646
|
exit(1)
|
|
645
647
|
dataframes.append(df)
|
|
646
648
|
|
|
647
|
-
result_response = self.parent.network.
|
|
648
|
-
self.parent.network.
|
|
649
|
+
result_response = self.parent.network.hq.save_result({}, task["id"])
|
|
650
|
+
self.parent.network.hq.save_run(
|
|
649
651
|
arguments, task["id"], result_response["id"], org_id
|
|
650
652
|
)
|
|
651
653
|
|
|
652
|
-
return self.parent.network.
|
|
654
|
+
return self.parent.network.hq.update_dataframe(id_, dataframes)
|
|
653
655
|
|
|
654
656
|
def list(self) -> list[dict]:
|
|
655
657
|
"""
|
|
656
658
|
List all dataframes
|
|
657
659
|
"""
|
|
658
|
-
return self.parent.network.
|
|
660
|
+
return self.parent.network.hq.dataframes
|
|
659
661
|
|
|
660
662
|
|
|
661
663
|
class MockAlgorithmClient(MockBaseClient):
|
|
@@ -667,7 +669,7 @@ class MockAlgorithmClient(MockBaseClient):
|
|
|
667
669
|
self.image = "mock-image"
|
|
668
670
|
self.node_id = node.id_
|
|
669
671
|
self.collaboration_id = node.collaboration_id
|
|
670
|
-
self.study_id = node.network.
|
|
672
|
+
self.study_id = node.network.hq.study_id
|
|
671
673
|
self.organization_id = node.organization_id
|
|
672
674
|
self.databases = databases
|
|
673
675
|
|
|
@@ -10,23 +10,23 @@ if TYPE_CHECKING:
|
|
|
10
10
|
from vantage6.algorithm.mock.network import MockNetwork
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class
|
|
13
|
+
class MockHQ:
|
|
14
14
|
def __init__(self, network: "MockNetwork"):
|
|
15
15
|
"""
|
|
16
|
-
Create a mock
|
|
16
|
+
Create a mock HQ.
|
|
17
17
|
|
|
18
|
-
Typically, you do not need to create a mock
|
|
19
|
-
use the MockNetwork class to create a mock network.
|
|
18
|
+
Typically, you do not need to create a mock HQ manually. Instead, you should
|
|
19
|
+
use the MockNetwork class to create a mock network, which contains a mock HQ.
|
|
20
20
|
|
|
21
21
|
Parameters
|
|
22
22
|
----------
|
|
23
|
-
|
|
24
|
-
The
|
|
23
|
+
network : MockNetwork
|
|
24
|
+
The network that the HQ belongs to.
|
|
25
25
|
"""
|
|
26
26
|
self.network = network
|
|
27
27
|
|
|
28
28
|
# These contain the task, runs, results and dataframes as dictionaries that are
|
|
29
|
-
# the same format as you would get from the
|
|
29
|
+
# the same format as you would get from the HQ responses.
|
|
30
30
|
self.tasks = []
|
|
31
31
|
self.runs = []
|
|
32
32
|
self.results = []
|
|
@@ -45,18 +45,18 @@ class MockServer:
|
|
|
45
45
|
return {
|
|
46
46
|
"collaboration": {
|
|
47
47
|
"id": self.network.collaboration_id,
|
|
48
|
-
"link": f"/
|
|
48
|
+
"link": f"/hq/collaboration/{self.network.collaboration_id}",
|
|
49
49
|
"methods": ["PATCH", "GET", "DELETE"],
|
|
50
50
|
},
|
|
51
|
-
"organizations": f"/
|
|
52
|
-
"tasks": f"/
|
|
51
|
+
"organizations": f"/hq/organization?study_id={self.study_id}",
|
|
52
|
+
"tasks": f"/hq/task?study_id={self.study_id}",
|
|
53
53
|
"name": "Mock Study",
|
|
54
54
|
"id": self.study_id,
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
def save_result(self, result: Any, task_id: int):
|
|
58
58
|
"""
|
|
59
|
-
Save a result to the mock
|
|
59
|
+
Save a result to the mock HQ.
|
|
60
60
|
|
|
61
61
|
Parameters
|
|
62
62
|
----------
|
|
@@ -92,7 +92,7 @@ class MockServer:
|
|
|
92
92
|
self, arguments: dict, task_id: int, result_id: int, org_id: int
|
|
93
93
|
) -> dict:
|
|
94
94
|
"""
|
|
95
|
-
Save a run to the mock
|
|
95
|
+
Save a run to the mock HQ.
|
|
96
96
|
|
|
97
97
|
Parameters
|
|
98
98
|
----------
|
|
@@ -153,7 +153,7 @@ class MockServer:
|
|
|
153
153
|
init_organization_id: int,
|
|
154
154
|
) -> dict:
|
|
155
155
|
"""
|
|
156
|
-
Save a task to the mock
|
|
156
|
+
Save a task to the mock HQ.
|
|
157
157
|
|
|
158
158
|
Parameters
|
|
159
159
|
----------
|
|
@@ -245,8 +245,8 @@ class MockServer:
|
|
|
245
245
|
"link": f"/api/session/{self.session_id}",
|
|
246
246
|
"methods": ["GET", "PATCH", "DELETE"],
|
|
247
247
|
},
|
|
248
|
-
"tasks": {"msg": "not implemented in the
|
|
249
|
-
"last_session_task": {"msg": "not implemented in the
|
|
248
|
+
"tasks": {"msg": "not implemented in the MockHQ"},
|
|
249
|
+
"last_session_task": {"msg": "not implemented in the MockHQ"},
|
|
250
250
|
"columns": [
|
|
251
251
|
{
|
|
252
252
|
"name": column,
|
|
@@ -7,8 +7,8 @@ from vantage6.algorithm.tools.util import error
|
|
|
7
7
|
|
|
8
8
|
from vantage6.algorithm.mock.client import MockUserClient
|
|
9
9
|
from vantage6.algorithm.mock.globals import MockDatabase
|
|
10
|
+
from vantage6.algorithm.mock.hq import MockHQ
|
|
10
11
|
from vantage6.algorithm.mock.node import MockNode
|
|
11
|
-
from vantage6.algorithm.mock.server import MockServer
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class MockNetwork:
|
|
@@ -44,8 +44,8 @@ class MockNetwork:
|
|
|
44
44
|
----------
|
|
45
45
|
nodes : list[MockNode]
|
|
46
46
|
The nodes of the mock network.
|
|
47
|
-
|
|
48
|
-
The
|
|
47
|
+
hq : MockHQ
|
|
48
|
+
The HQ of the mock network.
|
|
49
49
|
user_client : MockUserClient
|
|
50
50
|
The user client of the mock network.
|
|
51
51
|
|
|
@@ -120,7 +120,7 @@ class MockNetwork:
|
|
|
120
120
|
organization_ids = list(range(1, len(datasets) + 1))
|
|
121
121
|
node_ids = list(range(1, len(datasets) + 1))
|
|
122
122
|
|
|
123
|
-
self.
|
|
123
|
+
self.hq = MockHQ(self)
|
|
124
124
|
|
|
125
125
|
databases = [
|
|
126
126
|
[
|
|
@@ -151,7 +151,7 @@ class MockNetwork:
|
|
|
151
151
|
exit(1)
|
|
152
152
|
|
|
153
153
|
# In the case the user provides a pandas DataFrame we need to create a dataframe
|
|
154
|
-
# record in
|
|
154
|
+
# record in HQ.
|
|
155
155
|
dfs_by_label = {}
|
|
156
156
|
for node_databases in databases:
|
|
157
157
|
for database in node_databases:
|
|
@@ -161,7 +161,7 @@ class MockNetwork:
|
|
|
161
161
|
|
|
162
162
|
for label, dfs in dfs_by_label.items():
|
|
163
163
|
if all(isinstance(df, pd.DataFrame) for df in dfs):
|
|
164
|
-
self.
|
|
164
|
+
self.hq.save_dataframe(
|
|
165
165
|
name=label, dataframes=dfs, source_db_label=label
|
|
166
166
|
)
|
|
167
167
|
|
|
@@ -60,7 +60,7 @@ class MockNode:
|
|
|
60
60
|
self.dataframes = {}
|
|
61
61
|
|
|
62
62
|
# In case a pandas dataframe is provided we assume the user directly wants to
|
|
63
|
-
# use it rather than running an extraction
|
|
63
|
+
# use it rather than running an extraction algorithm step first.
|
|
64
64
|
for dataset in self.datasets:
|
|
65
65
|
if isinstance(dataset.database, pd.DataFrame):
|
|
66
66
|
self.dataframes[dataset.label] = dataset.database
|
|
@@ -138,16 +138,14 @@ class MockNode:
|
|
|
138
138
|
for db_group in databases:
|
|
139
139
|
group_data = {}
|
|
140
140
|
for db in db_group:
|
|
141
|
-
label = self.network.
|
|
142
|
-
db["dataframe_id"]
|
|
143
|
-
)
|
|
141
|
+
label = self.network.hq.get_label_for_df_id(db["dataframe_id"])
|
|
144
142
|
self._validate_dataframe_exists(label)
|
|
145
143
|
group_data[label] = self.dataframes[label]
|
|
146
144
|
mock_data.append(group_data)
|
|
147
145
|
else:
|
|
148
146
|
# Single list format
|
|
149
147
|
for db in databases:
|
|
150
|
-
label = self.network.
|
|
148
|
+
label = self.network.hq.get_label_for_df_id(db["dataframe_id"])
|
|
151
149
|
self._validate_dataframe_exists(label)
|
|
152
150
|
mock_data.append(self.dataframes[label])
|
|
153
151
|
|
|
@@ -303,7 +301,7 @@ class MockNode:
|
|
|
303
301
|
"""
|
|
304
302
|
Get the task environment variables.
|
|
305
303
|
"""
|
|
306
|
-
task_id = len(self.network.
|
|
304
|
+
task_id = len(self.network.hq.tasks)
|
|
307
305
|
return {
|
|
308
306
|
**self.env,
|
|
309
307
|
ContainerEnvNames.FUNCTION_ACTION.value: action.value,
|
|
@@ -70,7 +70,7 @@ def wrap_algorithm(log_traceback: bool = True) -> None:
|
|
|
70
70
|
)
|
|
71
71
|
|
|
72
72
|
# write output from the method to mounted output file. Which will be
|
|
73
|
-
# transferred back to
|
|
73
|
+
# transferred back to HQ by the node-instance.
|
|
74
74
|
output_file = os.environ[ContainerEnvNames.OUTPUT_FILE.value]
|
|
75
75
|
info(f"Writing output to {output_file}")
|
|
76
76
|
|
|
@@ -182,9 +182,9 @@ def _write_output(output: Any, output_file: str) -> None:
|
|
|
182
182
|
"""
|
|
183
183
|
Write output to output file.
|
|
184
184
|
|
|
185
|
-
In case the result needs to be sent to
|
|
185
|
+
In case the result needs to be sent to HQ, the output file should contain
|
|
186
186
|
valid JSON data. This is because the node will read the output file and send the
|
|
187
|
-
data to
|
|
187
|
+
data to HQ.
|
|
188
188
|
|
|
189
189
|
In the case we are building a session, the output of the algorithm is expected to
|
|
190
190
|
be a parquet table. In this case, the output file should contain the parquet data.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_deserialization.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_docker_wrapper.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_preprocessing.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/_test_serialization.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/algorithm/__init__.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/algorithm_module.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/base/__init__.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_user_client.py
RENAMED
|
File without changes
|
{vantage6_algorithm_tools-5.0.0b1 → vantage6_algorithm_tools-5.0.0b4}/tests/mock/test_util.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|