vantage6-algorithm-tools 4.13.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. vantage6_algorithm_tools-4.13.4/PKG-INFO +232 -0
  2. vantage6_algorithm_tools-4.13.4/setup.cfg +4 -0
  3. vantage6_algorithm_tools-4.13.4/setup.py +58 -0
  4. vantage6_algorithm_tools-4.13.4/tests/__init__.py +0 -0
  5. vantage6_algorithm_tools-4.13.4/tests/algorithm/__init__.py +0 -0
  6. vantage6_algorithm_tools-4.13.4/tests/algorithm/test___init__.py +128 -0
  7. vantage6_algorithm_tools-4.13.4/tests/algorithm_module.py +5 -0
  8. vantage6_algorithm_tools-4.13.4/tests/base/__init__.py +0 -0
  9. vantage6_algorithm_tools-4.13.4/tests/test_deserialization.py +15 -0
  10. vantage6_algorithm_tools-4.13.4/tests/test_docker_wrapper.py +127 -0
  11. vantage6_algorithm_tools-4.13.4/tests/test_serialization.py +25 -0
  12. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/__build__ +1 -0
  13. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/__init__.py +698 -0
  14. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/_version.py +23 -0
  15. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/__init__.py +4 -0
  16. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/decorators.py +530 -0
  17. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/exceptions.py +162 -0
  18. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/mock_client.py +618 -0
  19. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/preprocessing/__init__.py +65 -0
  20. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/preprocessing/functions.py +73 -0
  21. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/util.py +177 -0
  22. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/wrap.py +194 -0
  23. vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/wrappers.py +318 -0
  24. vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/PKG-INFO +232 -0
  25. vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/SOURCES.txt +26 -0
  26. vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/dependency_links.txt +1 -0
  27. vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/requires.txt +12 -0
  28. vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/top_level.txt +2 -0
@@ -0,0 +1,232 @@
1
+ Metadata-Version: 2.4
2
+ Name: vantage6_algorithm_tools
3
+ Version: 4.13.4
4
+ Summary: Vantage6 algorithm tools
5
+ Home-page: https://github.com/vantage6/vantage6
6
+ Requires-Python: >=3.6
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: openpyxl>=3.0.0
9
+ Requires-Dist: pandas>=1.5.3
10
+ Requires-Dist: pyjwt==2.9.0
11
+ Requires-Dist: pyfiglet==1.0.4
12
+ Requires-Dist: psycopg2-binary==2.9.10
13
+ Requires-Dist: SPARQLWrapper>=2.0.0
14
+ Requires-Dist: sqlalchemy==1.4.46
15
+ Requires-Dist: vantage6-common==4.13.4
16
+ Provides-Extra: dev
17
+ Requires-Dist: black; extra == "dev"
18
+ Requires-Dist: pre-commit; extra == "dev"
19
+ Dynamic: description
20
+ Dynamic: description-content-type
21
+ Dynamic: home-page
22
+ Dynamic: provides-extra
23
+ Dynamic: requires-dist
24
+ Dynamic: requires-python
25
+ Dynamic: summary
26
+
27
+ <h1 align="center">
28
+ <br>
29
+ <a href="https://vantage6.ai"><img src="https://github.com/IKNL/guidelines/blob/master/resources/logos/vantage6.png?raw=true" alt="vantage6" width="350"></a>
30
+ </h1>
31
+
32
+ <h3 align=center> A Privacy Enhancing Technology (PET) Operations platform</h3>
33
+ <h3 align="center">
34
+
35
+ <!-- Badges go here-->
36
+
37
+ [![Release](https://github.com/vantage6/vantage6/actions/workflows/release.yml/badge.svg)](https://github.com/vantage6/vantage6/actions/workflows/release.yml)
38
+ [![PyPI vantage6](https://badge.fury.io/py/vantage6.svg)](https://badge.fury.io/py/vantage6)
39
+ [![Unittests](https://github.com/vantage6/vantage6/actions/workflows/unit_tests.yml/badge.svg)](https://github.com/vantage6/vantage6/actions/workflows/unit_tests.yml)
40
+ [![Coverage Status](https://coveralls.io/repos/github/vantage6/vantage6/badge.svg?branch=main)](https://coveralls.io/github/vantage6/vantage6?branch=main)
41
+ [![Codacy Badge](https://app.codacy.com/project/badge/Grade/2e60ac3b3f284620805f7399cba317be)](https://app.codacy.com/gh/vantage6/vantage6/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
42
+ [![DOI](https://zenodo.org/badge/492818831.svg)](https://zenodo.org/badge/latestdoi/492818831)
43
+ [![Discord](https://img.shields.io/discord/643526403207331841)](https://discord.gg/yAyFf6Y)
44
+ [![Research software directory](https://img.shields.io/badge/rsd-vantage6-deepskyblue)](https://research-software-directory.org/software/vantage6)
45
+
46
+
47
+ </h3>
48
+
49
+ <p align="center">
50
+ <a href="#books-quickstart">Quickstart</a> •
51
+ <a href="#project-structure">Project structure</a> •
52
+ <a href="#gift_heart-join-the-community">Join the community</a> •
53
+ <a href="#scroll-license">License</a> •
54
+ <a href="#black_nib-code-of-conduct">Code of conduct</a> •
55
+ <a href="#black_nib-references">References</a>
56
+ </p>
57
+
58
+ ---
59
+
60
+ This repository is contains all the **vantage6** infrastructure source code. The **vantage6** technology enables to manage and deploy privacy enhancing technologies like Federated Learning (FL) and Multi-Party Computation (MPC). Please visit our [website](https://vantage6.ai) to learn more!
61
+
62
+ You can find more (user) documentation at [readthedocs](https://docs.vantage6.ai). If you have any questions, suggestions or just want to chat about federated learning: join our [Discord)](https://discord.gg/yAyFf6Y) channel.
63
+
64
+ ## Infrastructure overview
65
+
66
+ ![Vantage6 architecture overview](docs/images/overview-infrastructure.png)
67
+
68
+ _A High level overview of the vantage6 infrastructure. Vantage6 has both a
69
+ client-server and peer-to-peer architecture. The client is used by the researcher to
70
+ create (PET) computation requests. It is also used to manage users, organizations and
71
+ collaborations. The server contains users, organizations, collaborations, tasks and
72
+ their results. It provides a central access point for both the clients and nodes. The
73
+ nodes have access to privacy sensitive data and handle computation requests retrieved
74
+ from the server. Computation request are executed as separate containers on the node.
75
+ These containers are connected to containers at other nodes by a VPN network._
76
+
77
+ ## :books: Quickstart
78
+
79
+ ### Requirements
80
+
81
+ The **vantage6** infrastructure is delivered in Docker images. To run these images, you
82
+ need to have [Docker](https://docs.docker.com/get-docker/) installed. To install the
83
+ latest version of the vantage6 CLI, you need to have
84
+ [Python](https://www.python.org/downloads/), we recommend using an environment manager
85
+ like [mini-conda](https://docs.conda.io/en/latest/miniconda.html).
86
+
87
+ Install the latest version of the vantage6 CLI by using:
88
+
89
+ ```bash
90
+ pip install vantage6
91
+ ```
92
+
93
+ This install the `v6` commands, which allows you to manage your nodes and servers. To view all available options, run:
94
+
95
+ ```bash
96
+ v6 --help
97
+ ```
98
+
99
+ For example you can create a local test setup by using:
100
+
101
+ ```bash
102
+ v6 dev create-demo-network
103
+ ```
104
+
105
+ This creates a local network with a server and two nodes. You can start the network by running:
106
+
107
+ ```bash
108
+ v6 dev start-demo-network
109
+ ```
110
+
111
+ This will start the server and nodes in the background. You can view the logs by running:
112
+
113
+ ```bash
114
+ # View node logs
115
+ v6 node attach
116
+
117
+ # View server logs
118
+ v6 server attach
119
+ ```
120
+
121
+ From here you can use the [vantage6-client](https://pypi.org/project/vantage6-client)
122
+ to interact with the server. The demo network has a pre-configured organization with
123
+ the following credentials:
124
+
125
+ - Username: `dev_admin`
126
+ - Password: `password`
127
+
128
+ For example, you can create a new organization by running:
129
+
130
+ ```python
131
+ from vantage6.client import Client
132
+
133
+ client = Client('http://127.0.0.1', 7601, '/api', log_level='debug')
134
+ client.authenticate('dev_admin', 'password')
135
+ client.setup_encryption(None)
136
+
137
+ client.organization.create(
138
+ name='My organization',
139
+ address1='My address',
140
+ address2='My address',
141
+ zipcode='1234AB',
142
+ country='The Netherlands',
143
+ domain='my-organization.com'
144
+ )
145
+ ```
146
+
147
+ You can find more (user) documentation at [readthedocs](https://docs.vantage6.ai)
148
+
149
+ ## Project structure
150
+
151
+ ### PYPI packages
152
+
153
+ This repository is home to 6 PyPi packages:
154
+
155
+ - [vantage6](https://pypi.org/project/vantage6) -> _CLI for managing node and server instances_
156
+ - [vantage6-client](https://pypi.org/project/vantage6-client) -> _Python client for interacting with the vantage6-server_
157
+ - [vantage6-algorithm-tools](https://pypi.org/project/vantage6-algorithm-tools) -> _Python tools to facilitate algorithm development_
158
+ - [vantage6-node](https://pypi.org/project/vantage6-node) -> _Node application package_
159
+ - [vantage6-server](https://pypi.org/project/vantage6-server) -> _Server application package_
160
+ - [vantage6-algorithm-store](https://pypi.org/project/vantage6-algorithm-store) -> _Algorithm store application package_
161
+ - [vantage6-common](https://pypi.org/project/vantage6-common) -> _Package with common vantage6 functions_
162
+ - [vantage6-backend-common](https://pypi.org/project/vantage6-backend-common) -> _Package with functions common to central server and algorithm store_
163
+
164
+ **Note that when using vantage6 you do not install the _server_ and _node_ packages. These are delivered to you in Docker images.**
165
+
166
+ This repository also hosts the code for the vantage6 user interface (UI). The UI
167
+ is an Angular web application that can be used to interact with the vantage6 server
168
+ easily.
169
+
170
+ ### Docker images
171
+
172
+ The vantage6 infrastructure is delivered in Docker images. All Docker images are stored
173
+ in our private [Harbor](https://goharbor.io/) registry. The most important images are:
174
+
175
+ - `harbor2.vantage6.ai/infrastructure/node:VERSION` -> _Node application Docker image_
176
+ - `harbor2.vantage6.ai/infrastructure/server:VERSION` -> _Server application Docker image_
177
+ - `harbor2.vantage6.ai/infrastructure/ui:VERSION` -> _User interface Docker image_
178
+ - `harbor2.vantage6.ai/infrastructure/algorithm-store:VERSION` -> _Algorithm store Docker image_
179
+
180
+ with `VERSION` being the full semantic version of the vantage6 infrastructure, e.g.
181
+ `4.0.0` or `4.1.0rc0`.
182
+
183
+ Several other images are used to support the infrastructure:
184
+
185
+ - `harbor2.vantage6.ai/infrastructure/infrastructure-base:VERSION` -> _Base image for the infrastructure_
186
+ - `harbor2.vantage6.ai/infrastructure/squid:VERSION` -> _Squid proxy image used for the whitelisting service_
187
+ - `harbor2.vantage6.ai/infrastructure/alpine` -> _Alpine image used for vpn traffic forwarding_
188
+ - `harbor2.vantage6.ai/infrastructure/vpn-client` -> _VPN image used to connect to the VPN_
189
+ - `harbor2.vantage6.ai/infrastructure/vpn-configurator` -> _VPN image used for initialization_
190
+ - `harbor2.vantage6.ai/infrastructure/ssh-tunnel` -> _SSH tunnel image used for connecting algorithms to external services_
191
+
192
+ And finally there are some images released for algorithm development:
193
+
194
+ - `harbor2.vantage6.ai/infrastructure/algorithm-base:MAJOR.MINOR` -> _Base image for algorithm development_
195
+ - `harbor2.vantage6.ai/infrastructure/algorithm-ohdsi-base:MAJOR.MINOR` -> _Extended algorithm base image for OHDSI algorithm development_
196
+
197
+ ## :gift_heart: Join the community!
198
+
199
+ We hope to continue developing, improving, and supporting **vantage6** with the help of
200
+ the federated learning community. If you are interested in contributing, first of all,
201
+ thank you! Second, please take a look at our
202
+ [contributing guidelines](https://docs.vantage6.ai/en/main/devops/contribute.html)
203
+ and our [code of conduct](CODE_OF_CONDUCT.md).
204
+
205
+ <a href="https://github.com/vantage6/vantage6/graphs/contributors">
206
+ <img src="https://contrib.rocks/image?repo=vantage6/vantage6" />
207
+ </a>
208
+
209
+ ## :scroll: License
210
+
211
+ This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
212
+
213
+ ## :black_nib: Code of Conduct
214
+
215
+ Please note that this project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). **By participating in any way in this project you agree to abide by its terms.**
216
+
217
+ ## :black_nib: References
218
+
219
+ If you are using **vantage6**, please cite this repository as well as the accompanying papers as follows:
220
+
221
+ > - F. Martin, M. Sieswerda, H. Alradhi, et al. vantage6. Available at https://doi.org/10.5281/zenodo.7221216. Accessed on MONTH, 20XX.
222
+ > - A. Moncada-Torres, F. Martin, M. Sieswerda, J. van Soest, G. Gelijnse. VANTAGE6: an open source priVAcy preserviNg federaTed leArninG infrastructurE for Secure Insight eXchange. AMIA Annual Symposium Proceedings, 2020, p. 870-877. [[BibTeX](https://arturomoncadatorres.com/bibtex/moncada-torres2020vantage6.txt), [PDF](https://vantage6.ai/vantage6/)]
223
+ > - D. Smits\*, B. van Beusekom\*, F. Martin, L. Veen, G. Geleijnse, A. Moncada-Torres, An Improved Infrastructure for Privacy-Preserving Analysis of Patient Data, Proceedings of the International Conference of Informatics, Management, and Technology in Healthcare (ICIMTH), vol. 25, 2022, p. 144-147. [[BibTeX](https://arturomoncadatorres.com/bibtex/smits2022improved.txt), [PDF](https://ebooks.iospress.nl/volumearticle/60190)]
224
+
225
+ ---
226
+
227
+ <p align="center">
228
+ <a href="https://vantage6.ai">vantage6.ai</a> •
229
+ <a href="https://discord.gg/yAyFf6Y">Discord</a> •
230
+ <a href="https://vantage6.discourse.group/">Discourse</a> •
231
+ <a href="https://docs.vantage6.ai">User documentation</a>
232
+ </p>
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,58 @@
1
+ import codecs
2
+ import os
3
+
4
+ from os import path
5
+ from setuptools import setup, find_namespace_packages
6
+ from pathlib import Path
7
+
8
+ # get current directory
9
+ here = Path(path.abspath(path.dirname(__file__)))
10
+ parent_dir = here.parent.absolute()
11
+
12
+ # get the long description from the README file
13
+ with codecs.open(path.join(parent_dir, "README.md"), encoding="utf-8") as f:
14
+ long_description = f.read()
15
+
16
+ # Read the API version from disk. This file should be located in the package
17
+ # folder, since it's also used to set the pkg.__version__ variable.
18
+ version_path = os.path.join(here, "vantage6", "algorithm", "client", "_version.py")
19
+ version_ns = {"__file__": version_path}
20
+ with codecs.open(version_path) as f:
21
+ exec(f.read(), {}, version_ns)
22
+
23
+
24
+ # setup the package
25
+ setup(
26
+ name="vantage6_algorithm_tools",
27
+ version=version_ns["__version__"],
28
+ description="Vantage6 algorithm tools",
29
+ long_description=long_description,
30
+ long_description_content_type="text/markdown",
31
+ url="https://github.com/vantage6/vantage6",
32
+ packages=find_namespace_packages(),
33
+ python_requires=">=3.6",
34
+ install_requires=[
35
+ "openpyxl>=3.0.0",
36
+ "pandas>=1.5.3",
37
+ "pyjwt==2.9.0",
38
+ "pyfiglet==1.0.4",
39
+ # psycopg2-binary is not used directly in algorithm tools, but is necessary when
40
+ # running SQL queries on a postgres database.
41
+ "psycopg2-binary==2.9.10",
42
+ "SPARQLWrapper>=2.0.0",
43
+ "sqlalchemy==1.4.46",
44
+ f'vantage6-common=={version_ns["__version__"]}',
45
+ ],
46
+ extras_require={
47
+ "dev": [
48
+ "black",
49
+ "pre-commit",
50
+ ]
51
+ },
52
+ tests_require=["pytest"],
53
+ package_data={
54
+ "vantage6.algorithm.client": [
55
+ "__build__",
56
+ ],
57
+ },
58
+ )
File without changes
@@ -0,0 +1,128 @@
1
+ import base64
2
+ import unittest
3
+ import json
4
+ import uuid
5
+ import jwt
6
+
7
+ from unittest.mock import patch, MagicMock
8
+ from vantage6.algorithm.client import AlgorithmClient
9
+ from vantage6.common.globals import STRING_ENCODING
10
+
11
+
12
+ def encode_result(result_dict: dict) -> str:
13
+ """Encode the result dictionary as a base64 string."""
14
+ return base64.urlsafe_b64encode(json.dumps(result_dict).encode()).decode()
15
+
16
+
17
+ class TestAlgorithmClient(unittest.TestCase):
18
+ def setUp(self):
19
+ payload = {
20
+ "sub": {
21
+ "image": "dummy",
22
+ "databases": [],
23
+ "node_id": 1,
24
+ "collaboration_id": 1,
25
+ }
26
+ }
27
+ dummy_token = jwt.encode(payload, key="", algorithm=None)
28
+ self.client = AlgorithmClient(
29
+ token=dummy_token,
30
+ host="http://dummy_host",
31
+ port=1234,
32
+ )
33
+ self.client.parent = MagicMock()
34
+ self.client.parent.request = MagicMock()
35
+ self.client.parent.image = "mock_image"
36
+ self.client.parent.collaboration_id = 1
37
+ self.client.parent.databases = []
38
+ self.client.parent.study_id = None
39
+ self.client.parent.store_id = None
40
+ self.client.generate_path_to = MagicMock(
41
+ return_value="http://mock_blobstream_url"
42
+ )
43
+ self.client.parent.request.side_effect = [
44
+ {"public_key": "mock_public_key"}, # For organization public key
45
+ {"uuid": "mock_uuid"}, # For blobstream response
46
+ {"task_id": 123}, # For task creation response
47
+ ]
48
+ self.client.log = MagicMock()
49
+
50
+ @patch("requests.get")
51
+ @patch("requests.post")
52
+ @patch("vantage6.algorithm.client.serialize", return_value=b"serialized_input")
53
+ def test_create_task(self, mock_serialize, mock_requests_post, mock_requests_get):
54
+ mock_response = MagicMock()
55
+ mock_response.status_code = 200
56
+ mock_response.json.return_value = {
57
+ "uuid": "mock_uuid",
58
+ "public_key": "mock_public_key",
59
+ "task_id": 123,
60
+ }
61
+ mock_requests_post.return_value = mock_response
62
+
63
+ mock_requests_get.return_value = mock_response
64
+
65
+ input_data = {"method": "mock_method", "args": [1, 2, 3]}
66
+ organizations = [1]
67
+ result = self.client.task.create(input_=input_data, organizations=organizations)
68
+
69
+ self.assertEqual(
70
+ result,
71
+ {"uuid": "mock_uuid", "public_key": "mock_public_key", "task_id": 123},
72
+ )
73
+ mock_serialize.assert_called_once_with(input_data)
74
+
75
+ @patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
76
+ def test_result_from_task(self, mock_multi_page_request):
77
+ mock_multi_page_request.return_value = [
78
+ {
79
+ "result": encode_result({"foo": "bar"}), # base64 for '{"foo": "bar"}'
80
+ "blob_storage_used": None,
81
+ }
82
+ ]
83
+
84
+ results = self.client.result.from_task(task_id=1)
85
+ self.assertEqual(results[0], {"foo": "bar"})
86
+
87
+ @patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
88
+ @patch(
89
+ "vantage6.common.client.client_base.ClientBase._download_run_data_from_server"
90
+ )
91
+ def test_result_from_task_azure(
92
+ self, mock_download_run_data, mock_multi_page_request
93
+ ):
94
+ with patch.object(
95
+ self.client.result.parent, "_multi_page_request"
96
+ ) as mock_multi_page_request, patch.object(
97
+ self.client.result.parent, "_download_run_data_from_server"
98
+ ) as mock_download_run_data:
99
+ # Simulate a result where blob_storage_used is True
100
+ mock_multi_page_request.return_value = [
101
+ {
102
+ "result": str(uuid.uuid4()),
103
+ "blob_storage_used": True,
104
+ }
105
+ ]
106
+ expected_value = {"foo": "bar"}
107
+ mock_download_run_data.return_value = json.dumps(expected_value).encode(
108
+ STRING_ENCODING
109
+ )
110
+ results = self.client.result.from_task(task_id=1)
111
+ self.assertEqual(results[0], expected_value)
112
+
113
+ @patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
114
+ def test_result_from_task_relational(self, mock_multi_page_request):
115
+ mock_multi_page_request.return_value = [
116
+ {
117
+ "result": encode_result({"foo": "bar"}),
118
+ "blob_storage_used": False,
119
+ },
120
+ ]
121
+
122
+ results = self.client.result.from_task(task_id=1)
123
+
124
+ self.assertEqual(results[0], {"foo": "bar"})
125
+
126
+
127
+ if __name__ == "__main__":
128
+ unittest.main()
@@ -0,0 +1,5 @@
1
+ import pandas as pd
2
+
3
+
4
+ def hello_world(data: pd.DataFrame):
5
+ return data
File without changes
@@ -0,0 +1,15 @@
1
+ from pathlib import Path
2
+ from vantage6.common.client import deserialization
3
+
4
+ SIMPLE_TARGET_DATA = {"key": "value"}
5
+
6
+
7
+ def test_deserialize_json(tmp_path: Path):
8
+ data = '{"key": "value"}'
9
+ json_path = tmp_path / "jsonfile.json"
10
+ json_path.write_text(data)
11
+
12
+ with json_path.open("r") as f:
13
+ result = deserialization.deserialize(f)
14
+
15
+ assert SIMPLE_TARGET_DATA == result
@@ -0,0 +1,127 @@
1
+ # import json
2
+ # from pathlib import Path
3
+ # from unittest.mock import patch, MagicMock
4
+
5
+ # import pandas as pd
6
+
7
+ # from vantage6.algorithm.tools import wrapper
8
+
9
+ # MODULE_NAME = "algorithm_module"
10
+ # DATA = "column1,column2\n1,2"
11
+ # TOKEN = "This is a fake token"
12
+ # INPUT_PARAMETERS = {"method": "hello_world"}
13
+ # SEPARATOR = "."
14
+ # SAMPLE_DB = pd.DataFrame([[1, 2]], columns=["column1", "column2"])
15
+
16
+ # MOCK_SPARQL_ENDPOINT = "sparql://some_triplestore"
17
+
18
+
19
+ # # def test_json_input_without_format_raises_deserializationexception(tmp_path):
20
+ # # """
21
+ # # It should only be possible to provide json input if it is preceded by the
22
+ # # string "json." in unicode. Otherwise a `DeserializationException` should
23
+ # # be thrown.
24
+ # # """
25
+ # # input_file = tmp_path / 'input.json'
26
+
27
+ # # with input_file.open('wb') as f:
28
+ # # f.write(json.dumps(INPUT_PARAMETERS).encode())
29
+
30
+ # # with raises(DeserializationException):
31
+ # # run_docker_wrapper_with_echo_db(input_file, tmp_path)
32
+
33
+
34
+ # # def test_json_input_with_format_succeeds(tmp_path):
35
+ # # input_file = tmp_path / 'input.txt'
36
+
37
+ # # with input_file.open('wb') as f:
38
+ # # f.write(json.dumps(INPUT_PARAMETERS).encode())
39
+
40
+ # # output_file = run_docker_wrapper_with_echo_db(input_file, tmp_path)
41
+ # # assert file_echoes_db(output_file)
42
+
43
+
44
+ # # def test_wrapper_serializes_json_output(tmp_path):
45
+ # # input_parameters = {'method': 'hello_world', 'output_format': JSON_FORMAT}
46
+ # # input_file = create_pickle_input(tmp_path, input_parameters)
47
+
48
+ # # output_file = run_docker_wrapper_with_echo_db(input_file, tmp_path)
49
+
50
+ # # with output_file.open('rb') as f:
51
+ # # # Check whether the data is preceded by json format string
52
+ # # assert f.read(len(JSON_FORMAT) + 1).decode() == f'{JSON_FORMAT}.'
53
+
54
+ # # # Since the echo_db algorithm was triggered, output will be table that
55
+ # # # can be read by pandas.
56
+ # # result = pd.read_json(f.read())
57
+ # # pd.testing.assert_frame_equal(SAMPLE_DB, result)
58
+
59
+
60
+ # def run_docker_wrapper_with_echo_db(input_file, tmp_path):
61
+ # """
62
+ # Run the `echo_db` testing algorithm through the wrapper code. The wrapper
63
+ # communicates through files whose locations are stored in the `INPUT_FILE`,
64
+ # `TOKEN_FILE` and `DATABASE_URI` environment variables. The output of the
65
+ # algorithm is stored in the location specified in environment variable
66
+ # `OUTPUT_FILE`.
67
+
68
+ # :param input_file: input arguments to the wrapper and algorithm
69
+ # :param tmp_path: temporary path to store additional files.
70
+ # :return:
71
+ # """
72
+ # db_file = tmp_path / "db_file.csv"
73
+ # token_file = tmp_path / "token.txt"
74
+ # output_file = tmp_path / "output_file.pkl"
75
+ # db_file.write_text(DATA)
76
+ # token_file.write_text(TOKEN)
77
+ # with patch("vantage6.algorithm.tools.docker_wrapper.os") as mock_os:
78
+ # mock_os.environ = {
79
+ # "INPUT_FILE": input_file,
80
+ # "TOKEN_FILE": token_file,
81
+ # "OUTPUT_FILE": output_file,
82
+ # "DATABASE_URI": db_file,
83
+ # }
84
+
85
+ # wrapper.docker_wrapper(MODULE_NAME)
86
+ # return output_file
87
+
88
+
89
+ # @patch("vantage6.algorithm.tools.wrap._run_algorithm_method")
90
+ # @patch("vantage6.algorithm.tools.docker_wrapper.os")
91
+ # @patch("vantage6.algorithm.tools.docker_wrapper.SPARQLWrapper")
92
+ # def test_sparql_docker_wrapper_passes_dataframe(
93
+ # SPARQLWrapper: MagicMock,
94
+ # os: MagicMock,
95
+ # _run_algorithm_method: MagicMock,
96
+ # tmp_path: Path,
97
+ # ):
98
+ # input_file = tmp_path / "input_file.pkl"
99
+ # token_file = tmp_path / "token.txt"
100
+ # output_file = tmp_path / "output.pkl"
101
+
102
+ # environ = {
103
+ # "INPUT_FILE": str(input_file),
104
+ # "TOKEN_FILE": str(token_file),
105
+ # "DATABASE_URI": MOCK_SPARQL_ENDPOINT,
106
+ # "OUTPUT_FILE": str(output_file),
107
+ # }
108
+
109
+ # os.environ = environ
110
+
111
+ # input_args = {"query": "select *"}
112
+
113
+ # with input_file.open("wb") as f:
114
+ # json.dumps(input_args, f)
115
+
116
+ # with token_file.open("w") as f:
117
+ # f.write(TOKEN)
118
+
119
+ # _run_algorithm_method.return_value = pd.DataFrame()
120
+ # SPARQLWrapper.return_value.query.return_value.convert.return_value = DATA.encode()
121
+
122
+ # wrapper.sparql_wrapper(MODULE_NAME)
123
+
124
+ # _run_algorithm_method.assert_called_once()
125
+
126
+ # target_df = pd.DataFrame([[1, 2]], columns=["column1", "column2"])
127
+ # pd.testing.assert_frame_equal(target_df, _run_algorithm_method.call_args[0][0])
@@ -0,0 +1,25 @@
1
+ from pytest import mark
2
+
3
+ from vantage6.common import serialization
4
+ import pandas as pd
5
+
6
+
7
+ @mark.parametrize(
8
+ "data,target",
9
+ [
10
+ # Default serialization
11
+ ([1, 2, 3], "[1, 2, 3]"),
12
+ ("hello", '"hello"'),
13
+ ({"hello": "goodbye"}, '{"hello": "goodbye"}'),
14
+ # Pandas serialization
15
+ (
16
+ pd.DataFrame([[1, 2, 3]], columns=["one", "two", "three"]),
17
+ '{"one":{"0":1},"two":{"0":2},"three":{"0":3}}',
18
+ ),
19
+ (pd.Series([1, 2, 3]), '{"0":1,"1":2,"2":3}'),
20
+ ],
21
+ )
22
+ def test_json_serialization(data, target):
23
+ result = serialization.serialize(data)
24
+
25
+ assert target == result.decode()