vantage6-algorithm-tools 4.13.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vantage6_algorithm_tools-4.13.4/PKG-INFO +232 -0
- vantage6_algorithm_tools-4.13.4/setup.cfg +4 -0
- vantage6_algorithm_tools-4.13.4/setup.py +58 -0
- vantage6_algorithm_tools-4.13.4/tests/__init__.py +0 -0
- vantage6_algorithm_tools-4.13.4/tests/algorithm/__init__.py +0 -0
- vantage6_algorithm_tools-4.13.4/tests/algorithm/test___init__.py +128 -0
- vantage6_algorithm_tools-4.13.4/tests/algorithm_module.py +5 -0
- vantage6_algorithm_tools-4.13.4/tests/base/__init__.py +0 -0
- vantage6_algorithm_tools-4.13.4/tests/test_deserialization.py +15 -0
- vantage6_algorithm_tools-4.13.4/tests/test_docker_wrapper.py +127 -0
- vantage6_algorithm_tools-4.13.4/tests/test_serialization.py +25 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/__build__ +1 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/__init__.py +698 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/client/_version.py +23 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/__init__.py +4 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/decorators.py +530 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/exceptions.py +162 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/mock_client.py +618 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/preprocessing/__init__.py +65 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/preprocessing/functions.py +73 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/util.py +177 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/wrap.py +194 -0
- vantage6_algorithm_tools-4.13.4/vantage6/algorithm/tools/wrappers.py +318 -0
- vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/PKG-INFO +232 -0
- vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/SOURCES.txt +26 -0
- vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/dependency_links.txt +1 -0
- vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/requires.txt +12 -0
- vantage6_algorithm_tools-4.13.4/vantage6_algorithm_tools.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vantage6_algorithm_tools
|
|
3
|
+
Version: 4.13.4
|
|
4
|
+
Summary: Vantage6 algorithm tools
|
|
5
|
+
Home-page: https://github.com/vantage6/vantage6
|
|
6
|
+
Requires-Python: >=3.6
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: openpyxl>=3.0.0
|
|
9
|
+
Requires-Dist: pandas>=1.5.3
|
|
10
|
+
Requires-Dist: pyjwt==2.9.0
|
|
11
|
+
Requires-Dist: pyfiglet==1.0.4
|
|
12
|
+
Requires-Dist: psycopg2-binary==2.9.10
|
|
13
|
+
Requires-Dist: SPARQLWrapper>=2.0.0
|
|
14
|
+
Requires-Dist: sqlalchemy==1.4.46
|
|
15
|
+
Requires-Dist: vantage6-common==4.13.4
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: black; extra == "dev"
|
|
18
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
19
|
+
Dynamic: description
|
|
20
|
+
Dynamic: description-content-type
|
|
21
|
+
Dynamic: home-page
|
|
22
|
+
Dynamic: provides-extra
|
|
23
|
+
Dynamic: requires-dist
|
|
24
|
+
Dynamic: requires-python
|
|
25
|
+
Dynamic: summary
|
|
26
|
+
|
|
27
|
+
<h1 align="center">
|
|
28
|
+
<br>
|
|
29
|
+
<a href="https://vantage6.ai"><img src="https://github.com/IKNL/guidelines/blob/master/resources/logos/vantage6.png?raw=true" alt="vantage6" width="350"></a>
|
|
30
|
+
</h1>
|
|
31
|
+
|
|
32
|
+
<h3 align=center> A Privacy Enhancing Technology (PET) Operations platform</h3>
|
|
33
|
+
<h3 align="center">
|
|
34
|
+
|
|
35
|
+
<!-- Badges go here-->
|
|
36
|
+
|
|
37
|
+
[](https://github.com/vantage6/vantage6/actions/workflows/release.yml)
|
|
38
|
+
[](https://badge.fury.io/py/vantage6)
|
|
39
|
+
[](https://github.com/vantage6/vantage6/actions/workflows/unit_tests.yml)
|
|
40
|
+
[](https://coveralls.io/github/vantage6/vantage6?branch=main)
|
|
41
|
+
[](https://app.codacy.com/gh/vantage6/vantage6/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
|
|
42
|
+
[](https://zenodo.org/badge/latestdoi/492818831)
|
|
43
|
+
[](https://discord.gg/yAyFf6Y)
|
|
44
|
+
[](https://research-software-directory.org/software/vantage6)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
</h3>
|
|
48
|
+
|
|
49
|
+
<p align="center">
|
|
50
|
+
<a href="#books-quickstart">Quickstart</a> •
|
|
51
|
+
<a href="#project-structure">Project structure</a> •
|
|
52
|
+
<a href="#gift_heart-join-the-community">Join the community</a> •
|
|
53
|
+
<a href="#scroll-license">License</a> •
|
|
54
|
+
<a href="#black_nib-code-of-conduct">Code of conduct</a> •
|
|
55
|
+
<a href="#black_nib-references">References</a>
|
|
56
|
+
</p>
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
This repository is contains all the **vantage6** infrastructure source code. The **vantage6** technology enables to manage and deploy privacy enhancing technologies like Federated Learning (FL) and Multi-Party Computation (MPC). Please visit our [website](https://vantage6.ai) to learn more!
|
|
61
|
+
|
|
62
|
+
You can find more (user) documentation at [readthedocs](https://docs.vantage6.ai). If you have any questions, suggestions or just want to chat about federated learning: join our [Discord)](https://discord.gg/yAyFf6Y) channel.
|
|
63
|
+
|
|
64
|
+
## Infrastructure overview
|
|
65
|
+
|
|
66
|
+

|
|
67
|
+
|
|
68
|
+
_A High level overview of the vantage6 infrastructure. Vantage6 has both a
|
|
69
|
+
client-server and peer-to-peer architecture. The client is used by the researcher to
|
|
70
|
+
create (PET) computation requests. It is also used to manage users, organizations and
|
|
71
|
+
collaborations. The server contains users, organizations, collaborations, tasks and
|
|
72
|
+
their results. It provides a central access point for both the clients and nodes. The
|
|
73
|
+
nodes have access to privacy sensitive data and handle computation requests retrieved
|
|
74
|
+
from the server. Computation request are executed as separate containers on the node.
|
|
75
|
+
These containers are connected to containers at other nodes by a VPN network._
|
|
76
|
+
|
|
77
|
+
## :books: Quickstart
|
|
78
|
+
|
|
79
|
+
### Requirements
|
|
80
|
+
|
|
81
|
+
The **vantage6** infrastructure is delivered in Docker images. To run these images, you
|
|
82
|
+
need to have [Docker](https://docs.docker.com/get-docker/) installed. To install the
|
|
83
|
+
latest version of the vantage6 CLI, you need to have
|
|
84
|
+
[Python](https://www.python.org/downloads/), we recommend using an environment manager
|
|
85
|
+
like [mini-conda](https://docs.conda.io/en/latest/miniconda.html).
|
|
86
|
+
|
|
87
|
+
Install the latest version of the vantage6 CLI by using:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install vantage6
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This install the `v6` commands, which allows you to manage your nodes and servers. To view all available options, run:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
v6 --help
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
For example you can create a local test setup by using:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
v6 dev create-demo-network
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
This creates a local network with a server and two nodes. You can start the network by running:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
v6 dev start-demo-network
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
This will start the server and nodes in the background. You can view the logs by running:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# View node logs
|
|
115
|
+
v6 node attach
|
|
116
|
+
|
|
117
|
+
# View server logs
|
|
118
|
+
v6 server attach
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
From here you can use the [vantage6-client](https://pypi.org/project/vantage6-client)
|
|
122
|
+
to interact with the server. The demo network has a pre-configured organization with
|
|
123
|
+
the following credentials:
|
|
124
|
+
|
|
125
|
+
- Username: `dev_admin`
|
|
126
|
+
- Password: `password`
|
|
127
|
+
|
|
128
|
+
For example, you can create a new organization by running:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from vantage6.client import Client
|
|
132
|
+
|
|
133
|
+
client = Client('http://127.0.0.1', 7601, '/api', log_level='debug')
|
|
134
|
+
client.authenticate('dev_admin', 'password')
|
|
135
|
+
client.setup_encryption(None)
|
|
136
|
+
|
|
137
|
+
client.organization.create(
|
|
138
|
+
name='My organization',
|
|
139
|
+
address1='My address',
|
|
140
|
+
address2='My address',
|
|
141
|
+
zipcode='1234AB',
|
|
142
|
+
country='The Netherlands',
|
|
143
|
+
domain='my-organization.com'
|
|
144
|
+
)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
You can find more (user) documentation at [readthedocs](https://docs.vantage6.ai)
|
|
148
|
+
|
|
149
|
+
## Project structure
|
|
150
|
+
|
|
151
|
+
### PYPI packages
|
|
152
|
+
|
|
153
|
+
This repository is home to 6 PyPi packages:
|
|
154
|
+
|
|
155
|
+
- [vantage6](https://pypi.org/project/vantage6) -> _CLI for managing node and server instances_
|
|
156
|
+
- [vantage6-client](https://pypi.org/project/vantage6-client) -> _Python client for interacting with the vantage6-server_
|
|
157
|
+
- [vantage6-algorithm-tools](https://pypi.org/project/vantage6-algorithm-tools) -> _Python tools to facilitate algorithm development_
|
|
158
|
+
- [vantage6-node](https://pypi.org/project/vantage6-node) -> _Node application package_
|
|
159
|
+
- [vantage6-server](https://pypi.org/project/vantage6-server) -> _Server application package_
|
|
160
|
+
- [vantage6-algorithm-store](https://pypi.org/project/vantage6-algorithm-store) -> _Algorithm store application package_
|
|
161
|
+
- [vantage6-common](https://pypi.org/project/vantage6-common) -> _Package with common vantage6 functions_
|
|
162
|
+
- [vantage6-backend-common](https://pypi.org/project/vantage6-backend-common) -> _Package with functions common to central server and algorithm store_
|
|
163
|
+
|
|
164
|
+
**Note that when using vantage6 you do not install the _server_ and _node_ packages. These are delivered to you in Docker images.**
|
|
165
|
+
|
|
166
|
+
This repository also hosts the code for the vantage6 user interface (UI). The UI
|
|
167
|
+
is an Angular web application that can be used to interact with the vantage6 server
|
|
168
|
+
easily.
|
|
169
|
+
|
|
170
|
+
### Docker images
|
|
171
|
+
|
|
172
|
+
The vantage6 infrastructure is delivered in Docker images. All Docker images are stored
|
|
173
|
+
in our private [Harbor](https://goharbor.io/) registry. The most important images are:
|
|
174
|
+
|
|
175
|
+
- `harbor2.vantage6.ai/infrastructure/node:VERSION` -> _Node application Docker image_
|
|
176
|
+
- `harbor2.vantage6.ai/infrastructure/server:VERSION` -> _Server application Docker image_
|
|
177
|
+
- `harbor2.vantage6.ai/infrastructure/ui:VERSION` -> _User interface Docker image_
|
|
178
|
+
- `harbor2.vantage6.ai/infrastructure/algorithm-store:VERSION` -> _Algorithm store Docker image_
|
|
179
|
+
|
|
180
|
+
with `VERSION` being the full semantic version of the vantage6 infrastructure, e.g.
|
|
181
|
+
`4.0.0` or `4.1.0rc0`.
|
|
182
|
+
|
|
183
|
+
Several other images are used to support the infrastructure:
|
|
184
|
+
|
|
185
|
+
- `harbor2.vantage6.ai/infrastructure/infrastructure-base:VERSION` -> _Base image for the infrastructure_
|
|
186
|
+
- `harbor2.vantage6.ai/infrastructure/squid:VERSION` -> _Squid proxy image used for the whitelisting service_
|
|
187
|
+
- `harbor2.vantage6.ai/infrastructure/alpine` -> _Alpine image used for vpn traffic forwarding_
|
|
188
|
+
- `harbor2.vantage6.ai/infrastructure/vpn-client` -> _VPN image used to connect to the VPN_
|
|
189
|
+
- `harbor2.vantage6.ai/infrastructure/vpn-configurator` -> _VPN image used for initialization_
|
|
190
|
+
- `harbor2.vantage6.ai/infrastructure/ssh-tunnel` -> _SSH tunnel image used for connecting algorithms to external services_
|
|
191
|
+
|
|
192
|
+
And finally there are some images released for algorithm development:
|
|
193
|
+
|
|
194
|
+
- `harbor2.vantage6.ai/infrastructure/algorithm-base:MAJOR.MINOR` -> _Base image for algorithm development_
|
|
195
|
+
- `harbor2.vantage6.ai/infrastructure/algorithm-ohdsi-base:MAJOR.MINOR` -> _Extended algorithm base image for OHDSI algorithm development_
|
|
196
|
+
|
|
197
|
+
## :gift_heart: Join the community!
|
|
198
|
+
|
|
199
|
+
We hope to continue developing, improving, and supporting **vantage6** with the help of
|
|
200
|
+
the federated learning community. If you are interested in contributing, first of all,
|
|
201
|
+
thank you! Second, please take a look at our
|
|
202
|
+
[contributing guidelines](https://docs.vantage6.ai/en/main/devops/contribute.html)
|
|
203
|
+
and our [code of conduct](CODE_OF_CONDUCT.md).
|
|
204
|
+
|
|
205
|
+
<a href="https://github.com/vantage6/vantage6/graphs/contributors">
|
|
206
|
+
<img src="https://contrib.rocks/image?repo=vantage6/vantage6" />
|
|
207
|
+
</a>
|
|
208
|
+
|
|
209
|
+
## :scroll: License
|
|
210
|
+
|
|
211
|
+
This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
|
|
212
|
+
|
|
213
|
+
## :black_nib: Code of Conduct
|
|
214
|
+
|
|
215
|
+
Please note that this project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). **By participating in any way in this project you agree to abide by its terms.**
|
|
216
|
+
|
|
217
|
+
## :black_nib: References
|
|
218
|
+
|
|
219
|
+
If you are using **vantage6**, please cite this repository as well as the accompanying papers as follows:
|
|
220
|
+
|
|
221
|
+
> - F. Martin, M. Sieswerda, H. Alradhi, et al. vantage6. Available at https://doi.org/10.5281/zenodo.7221216. Accessed on MONTH, 20XX.
|
|
222
|
+
> - A. Moncada-Torres, F. Martin, M. Sieswerda, J. van Soest, G. Gelijnse. VANTAGE6: an open source priVAcy preserviNg federaTed leArninG infrastructurE for Secure Insight eXchange. AMIA Annual Symposium Proceedings, 2020, p. 870-877. [[BibTeX](https://arturomoncadatorres.com/bibtex/moncada-torres2020vantage6.txt), [PDF](https://vantage6.ai/vantage6/)]
|
|
223
|
+
> - D. Smits\*, B. van Beusekom\*, F. Martin, L. Veen, G. Geleijnse, A. Moncada-Torres, An Improved Infrastructure for Privacy-Preserving Analysis of Patient Data, Proceedings of the International Conference of Informatics, Management, and Technology in Healthcare (ICIMTH), vol. 25, 2022, p. 144-147. [[BibTeX](https://arturomoncadatorres.com/bibtex/smits2022improved.txt), [PDF](https://ebooks.iospress.nl/volumearticle/60190)]
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
<p align="center">
|
|
228
|
+
<a href="https://vantage6.ai">vantage6.ai</a> •
|
|
229
|
+
<a href="https://discord.gg/yAyFf6Y">Discord</a> •
|
|
230
|
+
<a href="https://vantage6.discourse.group/">Discourse</a> •
|
|
231
|
+
<a href="https://docs.vantage6.ai">User documentation</a>
|
|
232
|
+
</p>
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import codecs
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from os import path
|
|
5
|
+
from setuptools import setup, find_namespace_packages
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
# get current directory
|
|
9
|
+
here = Path(path.abspath(path.dirname(__file__)))
|
|
10
|
+
parent_dir = here.parent.absolute()
|
|
11
|
+
|
|
12
|
+
# get the long description from the README file
|
|
13
|
+
with codecs.open(path.join(parent_dir, "README.md"), encoding="utf-8") as f:
|
|
14
|
+
long_description = f.read()
|
|
15
|
+
|
|
16
|
+
# Read the API version from disk. This file should be located in the package
|
|
17
|
+
# folder, since it's also used to set the pkg.__version__ variable.
|
|
18
|
+
version_path = os.path.join(here, "vantage6", "algorithm", "client", "_version.py")
|
|
19
|
+
version_ns = {"__file__": version_path}
|
|
20
|
+
with codecs.open(version_path) as f:
|
|
21
|
+
exec(f.read(), {}, version_ns)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# setup the package
|
|
25
|
+
setup(
|
|
26
|
+
name="vantage6_algorithm_tools",
|
|
27
|
+
version=version_ns["__version__"],
|
|
28
|
+
description="Vantage6 algorithm tools",
|
|
29
|
+
long_description=long_description,
|
|
30
|
+
long_description_content_type="text/markdown",
|
|
31
|
+
url="https://github.com/vantage6/vantage6",
|
|
32
|
+
packages=find_namespace_packages(),
|
|
33
|
+
python_requires=">=3.6",
|
|
34
|
+
install_requires=[
|
|
35
|
+
"openpyxl>=3.0.0",
|
|
36
|
+
"pandas>=1.5.3",
|
|
37
|
+
"pyjwt==2.9.0",
|
|
38
|
+
"pyfiglet==1.0.4",
|
|
39
|
+
# psycopg2-binary is not used directly in algorithm tools, but is necessary when
|
|
40
|
+
# running SQL queries on a postgres database.
|
|
41
|
+
"psycopg2-binary==2.9.10",
|
|
42
|
+
"SPARQLWrapper>=2.0.0",
|
|
43
|
+
"sqlalchemy==1.4.46",
|
|
44
|
+
f'vantage6-common=={version_ns["__version__"]}',
|
|
45
|
+
],
|
|
46
|
+
extras_require={
|
|
47
|
+
"dev": [
|
|
48
|
+
"black",
|
|
49
|
+
"pre-commit",
|
|
50
|
+
]
|
|
51
|
+
},
|
|
52
|
+
tests_require=["pytest"],
|
|
53
|
+
package_data={
|
|
54
|
+
"vantage6.algorithm.client": [
|
|
55
|
+
"__build__",
|
|
56
|
+
],
|
|
57
|
+
},
|
|
58
|
+
)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import unittest
|
|
3
|
+
import json
|
|
4
|
+
import uuid
|
|
5
|
+
import jwt
|
|
6
|
+
|
|
7
|
+
from unittest.mock import patch, MagicMock
|
|
8
|
+
from vantage6.algorithm.client import AlgorithmClient
|
|
9
|
+
from vantage6.common.globals import STRING_ENCODING
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def encode_result(result_dict: dict) -> str:
|
|
13
|
+
"""Encode the result dictionary as a base64 string."""
|
|
14
|
+
return base64.urlsafe_b64encode(json.dumps(result_dict).encode()).decode()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TestAlgorithmClient(unittest.TestCase):
|
|
18
|
+
def setUp(self):
|
|
19
|
+
payload = {
|
|
20
|
+
"sub": {
|
|
21
|
+
"image": "dummy",
|
|
22
|
+
"databases": [],
|
|
23
|
+
"node_id": 1,
|
|
24
|
+
"collaboration_id": 1,
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
dummy_token = jwt.encode(payload, key="", algorithm=None)
|
|
28
|
+
self.client = AlgorithmClient(
|
|
29
|
+
token=dummy_token,
|
|
30
|
+
host="http://dummy_host",
|
|
31
|
+
port=1234,
|
|
32
|
+
)
|
|
33
|
+
self.client.parent = MagicMock()
|
|
34
|
+
self.client.parent.request = MagicMock()
|
|
35
|
+
self.client.parent.image = "mock_image"
|
|
36
|
+
self.client.parent.collaboration_id = 1
|
|
37
|
+
self.client.parent.databases = []
|
|
38
|
+
self.client.parent.study_id = None
|
|
39
|
+
self.client.parent.store_id = None
|
|
40
|
+
self.client.generate_path_to = MagicMock(
|
|
41
|
+
return_value="http://mock_blobstream_url"
|
|
42
|
+
)
|
|
43
|
+
self.client.parent.request.side_effect = [
|
|
44
|
+
{"public_key": "mock_public_key"}, # For organization public key
|
|
45
|
+
{"uuid": "mock_uuid"}, # For blobstream response
|
|
46
|
+
{"task_id": 123}, # For task creation response
|
|
47
|
+
]
|
|
48
|
+
self.client.log = MagicMock()
|
|
49
|
+
|
|
50
|
+
@patch("requests.get")
|
|
51
|
+
@patch("requests.post")
|
|
52
|
+
@patch("vantage6.algorithm.client.serialize", return_value=b"serialized_input")
|
|
53
|
+
def test_create_task(self, mock_serialize, mock_requests_post, mock_requests_get):
|
|
54
|
+
mock_response = MagicMock()
|
|
55
|
+
mock_response.status_code = 200
|
|
56
|
+
mock_response.json.return_value = {
|
|
57
|
+
"uuid": "mock_uuid",
|
|
58
|
+
"public_key": "mock_public_key",
|
|
59
|
+
"task_id": 123,
|
|
60
|
+
}
|
|
61
|
+
mock_requests_post.return_value = mock_response
|
|
62
|
+
|
|
63
|
+
mock_requests_get.return_value = mock_response
|
|
64
|
+
|
|
65
|
+
input_data = {"method": "mock_method", "args": [1, 2, 3]}
|
|
66
|
+
organizations = [1]
|
|
67
|
+
result = self.client.task.create(input_=input_data, organizations=organizations)
|
|
68
|
+
|
|
69
|
+
self.assertEqual(
|
|
70
|
+
result,
|
|
71
|
+
{"uuid": "mock_uuid", "public_key": "mock_public_key", "task_id": 123},
|
|
72
|
+
)
|
|
73
|
+
mock_serialize.assert_called_once_with(input_data)
|
|
74
|
+
|
|
75
|
+
@patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
|
|
76
|
+
def test_result_from_task(self, mock_multi_page_request):
|
|
77
|
+
mock_multi_page_request.return_value = [
|
|
78
|
+
{
|
|
79
|
+
"result": encode_result({"foo": "bar"}), # base64 for '{"foo": "bar"}'
|
|
80
|
+
"blob_storage_used": None,
|
|
81
|
+
}
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
results = self.client.result.from_task(task_id=1)
|
|
85
|
+
self.assertEqual(results[0], {"foo": "bar"})
|
|
86
|
+
|
|
87
|
+
@patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
|
|
88
|
+
@patch(
|
|
89
|
+
"vantage6.common.client.client_base.ClientBase._download_run_data_from_server"
|
|
90
|
+
)
|
|
91
|
+
def test_result_from_task_azure(
|
|
92
|
+
self, mock_download_run_data, mock_multi_page_request
|
|
93
|
+
):
|
|
94
|
+
with patch.object(
|
|
95
|
+
self.client.result.parent, "_multi_page_request"
|
|
96
|
+
) as mock_multi_page_request, patch.object(
|
|
97
|
+
self.client.result.parent, "_download_run_data_from_server"
|
|
98
|
+
) as mock_download_run_data:
|
|
99
|
+
# Simulate a result where blob_storage_used is True
|
|
100
|
+
mock_multi_page_request.return_value = [
|
|
101
|
+
{
|
|
102
|
+
"result": str(uuid.uuid4()),
|
|
103
|
+
"blob_storage_used": True,
|
|
104
|
+
}
|
|
105
|
+
]
|
|
106
|
+
expected_value = {"foo": "bar"}
|
|
107
|
+
mock_download_run_data.return_value = json.dumps(expected_value).encode(
|
|
108
|
+
STRING_ENCODING
|
|
109
|
+
)
|
|
110
|
+
results = self.client.result.from_task(task_id=1)
|
|
111
|
+
self.assertEqual(results[0], expected_value)
|
|
112
|
+
|
|
113
|
+
@patch("vantage6.algorithm.client.AlgorithmClient._multi_page_request")
|
|
114
|
+
def test_result_from_task_relational(self, mock_multi_page_request):
|
|
115
|
+
mock_multi_page_request.return_value = [
|
|
116
|
+
{
|
|
117
|
+
"result": encode_result({"foo": "bar"}),
|
|
118
|
+
"blob_storage_used": False,
|
|
119
|
+
},
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
results = self.client.result.from_task(task_id=1)
|
|
123
|
+
|
|
124
|
+
self.assertEqual(results[0], {"foo": "bar"})
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
if __name__ == "__main__":
|
|
128
|
+
unittest.main()
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from vantage6.common.client import deserialization
|
|
3
|
+
|
|
4
|
+
SIMPLE_TARGET_DATA = {"key": "value"}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_deserialize_json(tmp_path: Path):
|
|
8
|
+
data = '{"key": "value"}'
|
|
9
|
+
json_path = tmp_path / "jsonfile.json"
|
|
10
|
+
json_path.write_text(data)
|
|
11
|
+
|
|
12
|
+
with json_path.open("r") as f:
|
|
13
|
+
result = deserialization.deserialize(f)
|
|
14
|
+
|
|
15
|
+
assert SIMPLE_TARGET_DATA == result
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# import json
|
|
2
|
+
# from pathlib import Path
|
|
3
|
+
# from unittest.mock import patch, MagicMock
|
|
4
|
+
|
|
5
|
+
# import pandas as pd
|
|
6
|
+
|
|
7
|
+
# from vantage6.algorithm.tools import wrapper
|
|
8
|
+
|
|
9
|
+
# MODULE_NAME = "algorithm_module"
|
|
10
|
+
# DATA = "column1,column2\n1,2"
|
|
11
|
+
# TOKEN = "This is a fake token"
|
|
12
|
+
# INPUT_PARAMETERS = {"method": "hello_world"}
|
|
13
|
+
# SEPARATOR = "."
|
|
14
|
+
# SAMPLE_DB = pd.DataFrame([[1, 2]], columns=["column1", "column2"])
|
|
15
|
+
|
|
16
|
+
# MOCK_SPARQL_ENDPOINT = "sparql://some_triplestore"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# # def test_json_input_without_format_raises_deserializationexception(tmp_path):
|
|
20
|
+
# # """
|
|
21
|
+
# # It should only be possible to provide json input if it is preceded by the
|
|
22
|
+
# # string "json." in unicode. Otherwise a `DeserializationException` should
|
|
23
|
+
# # be thrown.
|
|
24
|
+
# # """
|
|
25
|
+
# # input_file = tmp_path / 'input.json'
|
|
26
|
+
|
|
27
|
+
# # with input_file.open('wb') as f:
|
|
28
|
+
# # f.write(json.dumps(INPUT_PARAMETERS).encode())
|
|
29
|
+
|
|
30
|
+
# # with raises(DeserializationException):
|
|
31
|
+
# # run_docker_wrapper_with_echo_db(input_file, tmp_path)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# # def test_json_input_with_format_succeeds(tmp_path):
|
|
35
|
+
# # input_file = tmp_path / 'input.txt'
|
|
36
|
+
|
|
37
|
+
# # with input_file.open('wb') as f:
|
|
38
|
+
# # f.write(json.dumps(INPUT_PARAMETERS).encode())
|
|
39
|
+
|
|
40
|
+
# # output_file = run_docker_wrapper_with_echo_db(input_file, tmp_path)
|
|
41
|
+
# # assert file_echoes_db(output_file)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# # def test_wrapper_serializes_json_output(tmp_path):
|
|
45
|
+
# # input_parameters = {'method': 'hello_world', 'output_format': JSON_FORMAT}
|
|
46
|
+
# # input_file = create_pickle_input(tmp_path, input_parameters)
|
|
47
|
+
|
|
48
|
+
# # output_file = run_docker_wrapper_with_echo_db(input_file, tmp_path)
|
|
49
|
+
|
|
50
|
+
# # with output_file.open('rb') as f:
|
|
51
|
+
# # # Check whether the data is preceded by json format string
|
|
52
|
+
# # assert f.read(len(JSON_FORMAT) + 1).decode() == f'{JSON_FORMAT}.'
|
|
53
|
+
|
|
54
|
+
# # # Since the echo_db algorithm was triggered, output will be table that
|
|
55
|
+
# # # can be read by pandas.
|
|
56
|
+
# # result = pd.read_json(f.read())
|
|
57
|
+
# # pd.testing.assert_frame_equal(SAMPLE_DB, result)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# def run_docker_wrapper_with_echo_db(input_file, tmp_path):
|
|
61
|
+
# """
|
|
62
|
+
# Run the `echo_db` testing algorithm through the wrapper code. The wrapper
|
|
63
|
+
# communicates through files whose locations are stored in the `INPUT_FILE`,
|
|
64
|
+
# `TOKEN_FILE` and `DATABASE_URI` environment variables. The output of the
|
|
65
|
+
# algorithm is stored in the location specified in environment variable
|
|
66
|
+
# `OUTPUT_FILE`.
|
|
67
|
+
|
|
68
|
+
# :param input_file: input arguments to the wrapper and algorithm
|
|
69
|
+
# :param tmp_path: temporary path to store additional files.
|
|
70
|
+
# :return:
|
|
71
|
+
# """
|
|
72
|
+
# db_file = tmp_path / "db_file.csv"
|
|
73
|
+
# token_file = tmp_path / "token.txt"
|
|
74
|
+
# output_file = tmp_path / "output_file.pkl"
|
|
75
|
+
# db_file.write_text(DATA)
|
|
76
|
+
# token_file.write_text(TOKEN)
|
|
77
|
+
# with patch("vantage6.algorithm.tools.docker_wrapper.os") as mock_os:
|
|
78
|
+
# mock_os.environ = {
|
|
79
|
+
# "INPUT_FILE": input_file,
|
|
80
|
+
# "TOKEN_FILE": token_file,
|
|
81
|
+
# "OUTPUT_FILE": output_file,
|
|
82
|
+
# "DATABASE_URI": db_file,
|
|
83
|
+
# }
|
|
84
|
+
|
|
85
|
+
# wrapper.docker_wrapper(MODULE_NAME)
|
|
86
|
+
# return output_file
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# @patch("vantage6.algorithm.tools.wrap._run_algorithm_method")
|
|
90
|
+
# @patch("vantage6.algorithm.tools.docker_wrapper.os")
|
|
91
|
+
# @patch("vantage6.algorithm.tools.docker_wrapper.SPARQLWrapper")
|
|
92
|
+
# def test_sparql_docker_wrapper_passes_dataframe(
|
|
93
|
+
# SPARQLWrapper: MagicMock,
|
|
94
|
+
# os: MagicMock,
|
|
95
|
+
# _run_algorithm_method: MagicMock,
|
|
96
|
+
# tmp_path: Path,
|
|
97
|
+
# ):
|
|
98
|
+
# input_file = tmp_path / "input_file.pkl"
|
|
99
|
+
# token_file = tmp_path / "token.txt"
|
|
100
|
+
# output_file = tmp_path / "output.pkl"
|
|
101
|
+
|
|
102
|
+
# environ = {
|
|
103
|
+
# "INPUT_FILE": str(input_file),
|
|
104
|
+
# "TOKEN_FILE": str(token_file),
|
|
105
|
+
# "DATABASE_URI": MOCK_SPARQL_ENDPOINT,
|
|
106
|
+
# "OUTPUT_FILE": str(output_file),
|
|
107
|
+
# }
|
|
108
|
+
|
|
109
|
+
# os.environ = environ
|
|
110
|
+
|
|
111
|
+
# input_args = {"query": "select *"}
|
|
112
|
+
|
|
113
|
+
# with input_file.open("wb") as f:
|
|
114
|
+
# json.dumps(input_args, f)
|
|
115
|
+
|
|
116
|
+
# with token_file.open("w") as f:
|
|
117
|
+
# f.write(TOKEN)
|
|
118
|
+
|
|
119
|
+
# _run_algorithm_method.return_value = pd.DataFrame()
|
|
120
|
+
# SPARQLWrapper.return_value.query.return_value.convert.return_value = DATA.encode()
|
|
121
|
+
|
|
122
|
+
# wrapper.sparql_wrapper(MODULE_NAME)
|
|
123
|
+
|
|
124
|
+
# _run_algorithm_method.assert_called_once()
|
|
125
|
+
|
|
126
|
+
# target_df = pd.DataFrame([[1, 2]], columns=["column1", "column2"])
|
|
127
|
+
# pd.testing.assert_frame_equal(target_df, _run_algorithm_method.call_args[0][0])
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from pytest import mark
|
|
2
|
+
|
|
3
|
+
from vantage6.common import serialization
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@mark.parametrize(
|
|
8
|
+
"data,target",
|
|
9
|
+
[
|
|
10
|
+
# Default serialization
|
|
11
|
+
([1, 2, 3], "[1, 2, 3]"),
|
|
12
|
+
("hello", '"hello"'),
|
|
13
|
+
({"hello": "goodbye"}, '{"hello": "goodbye"}'),
|
|
14
|
+
# Pandas serialization
|
|
15
|
+
(
|
|
16
|
+
pd.DataFrame([[1, 2, 3]], columns=["one", "two", "three"]),
|
|
17
|
+
'{"one":{"0":1},"two":{"0":2},"three":{"0":3}}',
|
|
18
|
+
),
|
|
19
|
+
(pd.Series([1, 2, 3]), '{"0":1,"1":2,"2":3}'),
|
|
20
|
+
],
|
|
21
|
+
)
|
|
22
|
+
def test_json_serialization(data, target):
|
|
23
|
+
result = serialization.serialize(data)
|
|
24
|
+
|
|
25
|
+
assert target == result.decode()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0
|