vantage6-algorithm-store 4.3.0b3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vantage6-algorithm-store-4.3.0b3/PKG-INFO +166 -0
- vantage6-algorithm-store-4.3.0b3/setup.cfg +4 -0
- vantage6-algorithm-store-4.3.0b3/setup.py +71 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/__init__.py +290 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/_version.py +23 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/db.py +71 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/globals.py +18 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/__init__.py +6 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/algorithm.py +44 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/argument.py +36 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/base.py +424 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/common/enums.py +29 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/database.py +33 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/function.py +43 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/model/vantage6_server.py +38 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/__init__.py +88 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/algorithm.py +340 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/schema/input_schema.py +96 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/schema/output_schema.py +56 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/vantage6_server.py +242 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/resource/version.py +61 -0
- vantage6-algorithm-store-4.3.0b3/vantage6/algorithm/store/wsgi.py +16 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/PKG-INFO +166 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/SOURCES.txt +26 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/dependency_links.txt +1 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/entry_points.txt +2 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/requires.txt +28 -0
- vantage6-algorithm-store-4.3.0b3/vantage6_algorithm_store.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: vantage6-algorithm-store
|
|
3
|
+
Version: 4.3.0b3
|
|
4
|
+
Summary: Vantage6 algorithm store
|
|
5
|
+
Home-page: https://github.com/vantage6/vantage6
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
|
|
10
|
+
<h1 align="center">
|
|
11
|
+
<br>
|
|
12
|
+
<a href="https://vantage6.ai"><img src="https://github.com/IKNL/guidelines/blob/master/resources/logos/vantage6.png?raw=true" alt="vantage6" width="350"></a>
|
|
13
|
+
</h1>
|
|
14
|
+
|
|
15
|
+
<h3 align=center> A Privacy Enhancing Technology (PET) Operations platform</h3>
|
|
16
|
+
<h3 align="center">
|
|
17
|
+
|
|
18
|
+
<!-- Badges go here-->
|
|
19
|
+
[](https://github.com/vantage6/vantage6/actions/workflows/release.yml)
|
|
20
|
+
[](https://badge.fury.io/py/vantage6)
|
|
21
|
+
[](https://github.com/vantage6/vantage6/actions/workflows/unit_tests.yml)
|
|
22
|
+
[](https://coveralls.io/github/vantage6/vantage6?branch=main)
|
|
23
|
+
[](https://www.codacy.com/gh/vantage6/vantage6/dashboard?utm_source=github.com&utm_medium=referral&utm_content=vantage6/vantage6&utm_campaign=Badge_Grade)
|
|
24
|
+
[](https://doi.org/10.5281/zenodo.7382602)
|
|
25
|
+
</h3>
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<a href="#books-quickstart">Quickstart</a> •
|
|
29
|
+
<a href="#project-structure">Project structure</a> •
|
|
30
|
+
<a href="#gift_heart-join-the-community">Join the community</a> •
|
|
31
|
+
<a href="#black_nib-references">References</a>
|
|
32
|
+
</p>
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
-----------------------------------------------------------------------------------------------------
|
|
36
|
+
This repository is contains all the **vantage6** infrastructure source code. The **vantage6** technology enables to manage and deploy privacy enhancing technologies like Federated Learning (FL) and Multi-Party Computation (MPC). Please visit our [website (vantage6.ai)](https://vantage6.ai) to learn more!
|
|
37
|
+
|
|
38
|
+
You can find more (user) documentation at [readthedocs (docs.vantage6.ai)](https://docs.vantage6.ai). If you have any questions, suggestions or just want to chat about federated learning: join our [Discord (https://discord.gg/yAyFf6Y)](https://discord.gg/yAyFf6Y) channel.
|
|
39
|
+
|
|
40
|
+
## Infrastructure overview
|
|
41
|
+
|
|
42
|
+

|
|
43
|
+
|
|
44
|
+
*A High level overview of the vantage6 infrastructure. Vantage6 has both a client-server and peer-to-peer architecture. The client is used by the researcher to create (PET) computation requests. It is also used to manage users, organizations and collaborations. The server contains users, organizations, collaborations, tasks and their results. It provides a central access point for both the clients and nodes. The nodes have access to privacy sensitive data and handle computation requests retrieved from the server. Computation request are executed as separate containers on the node. These containers are connected to containers at other nodes by a VPN network.*
|
|
45
|
+
|
|
46
|
+
## :books: Quickstart
|
|
47
|
+
|
|
48
|
+
### Requirements
|
|
49
|
+
The **vantage6** infrastructure is delivered in Docker images. To run these images, you need to have [Docker](https://docs.docker.com/get-docker/) installed. To install the latest version of the vantage6 CLI, you need to have [Python](https://www.python.org/downloads/), we recommend using an environment manager like [mini-conda](https://docs.conda.io/en/latest/miniconda.html).
|
|
50
|
+
|
|
51
|
+
Install the latest version of the vantage6 CLI by using:
|
|
52
|
+
```bash
|
|
53
|
+
pip install vantage6
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This install the `v6` commands, which allows you to manage your nodes and servers. To view all available options, run:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
v6 --help
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For example you can create a local test setup by using:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
v6 dev create-demo-network
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
This creates a local network with a server and two nodes. You can start the network by running:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
v6 dev start-demo-network
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
This will start the server and nodes in the background. You can view the logs by running:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
# View node logs
|
|
78
|
+
v6 node attach
|
|
79
|
+
|
|
80
|
+
# View server logs
|
|
81
|
+
v6 server attach
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
From here you can use the [vantage6-client](https://pypi.org/project/vantage6-client) to interact with the server. The demo network has a pre-configured organization with the following credentials:
|
|
85
|
+
|
|
86
|
+
* Username: `org_1-admin`
|
|
87
|
+
* Password: `password`
|
|
88
|
+
|
|
89
|
+
For example, you can create a new organization by running:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from vantage6.client import Client
|
|
93
|
+
|
|
94
|
+
client = Client('http://127.0.0.1', 5000, '/api', log_level='debug')
|
|
95
|
+
client.authenticate('org_1-admin', 'password')
|
|
96
|
+
client.setup_encryption(None)
|
|
97
|
+
|
|
98
|
+
client.organization.create(
|
|
99
|
+
name='My organization',
|
|
100
|
+
address1='My address',
|
|
101
|
+
address2='My address',
|
|
102
|
+
zipcode='1234AB',
|
|
103
|
+
country='The Netherlands',
|
|
104
|
+
domain='my-organization.com'
|
|
105
|
+
)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
You can find more (user) documentation at [readthedocs (docs.vantage6.ai)](https://docs.vantage6.ai)
|
|
109
|
+
|
|
110
|
+
## Project structure
|
|
111
|
+
|
|
112
|
+
### PYPI packages
|
|
113
|
+
This repository is home to 6 PyPi packages:
|
|
114
|
+
|
|
115
|
+
* [vantage6](https://pypi.org/project/vantage6) -> _CLI for managing node and server instances_
|
|
116
|
+
* [vantage6-client](https://pypi.org/project/vantage6-client) -> _Python client for interacting with the vantage6-server_
|
|
117
|
+
* [vantage6-algorithm-tools](https://pypi.org/project/vantage6-algorithm-tools) -> _Python tools to facilitate algorithm development_
|
|
118
|
+
* [vantage6-node](https://pypi.org/project/vantage6-node) -> _Node application package_
|
|
119
|
+
* [vantage6-server](https://pypi.org/project/vantage6-server) -> _Server application package_
|
|
120
|
+
* [vantage6-common](https://pypi.org/project/vantage6-common) -> _Server application package_
|
|
121
|
+
|
|
122
|
+
**Note that when using vantage6 you do not install the _server_ and _node_ packages. These are delivered to you in Docker images.**
|
|
123
|
+
|
|
124
|
+
### Docker images
|
|
125
|
+
The vantage6 infrastructure is delivered in Docker images. All Docker images are stored in our private [Harbor](https://goharbor.io/) registry. The most important images contain the server and node:
|
|
126
|
+
|
|
127
|
+
* `harbor2.vantage6.ai/infrastructure/node:VERSION` -> _Node application Docker image_
|
|
128
|
+
* `harbor2.vantage6.ai/infrastructure/server:VERSION` -> _Server application Docker image_
|
|
129
|
+
|
|
130
|
+
with `VERSION` being the full semantic version of the vantage6 infrastructure, e.g. `4.0.0` or `4.1.0rc0`.
|
|
131
|
+
|
|
132
|
+
Several other images are used to support the infrastructure:
|
|
133
|
+
|
|
134
|
+
* `harbor2.vantage6.ai/infrastructure/infrastructure-base:VERSION` -> _Base image for the infrastructure_
|
|
135
|
+
* `harbor2.vantage6.ai/infrastructure/squid:VERSION` -> _Squid proxy image used for the whitelisting service_
|
|
136
|
+
* `harbor2.vantage6.ai/infrastructure/alpine` -> _Alpine image used for vpn traffic forwarding_
|
|
137
|
+
* `harbor2.vantage6.ai/infrastructure/vpn-client` -> _VPN image used to connect to the VPN_
|
|
138
|
+
* `harbor2.vantage6.ai/infrastructure/vpn-configurator` -> _VPN image used for initialization_
|
|
139
|
+
* `harbor2.vantage6.ai/infrastructure/ssh-tunnel` -> _SSH tunnel image used for connecting algorithms to external services_
|
|
140
|
+
|
|
141
|
+
And finally there are some images released for algorithm development:
|
|
142
|
+
|
|
143
|
+
* `harbor2.vantage6.ai/infrastructure/algorithm-base:MAJOR.MINOR` -> _Base image for algorithm development_
|
|
144
|
+
* `harbor2.vantage6.ai/infrastructure/algorithm-ohdsi-base:MAJOR.MINOR` -> _Extended algorithm base image for OHDSI algorithm development_
|
|
145
|
+
|
|
146
|
+
## :gift_heart: Join the community!
|
|
147
|
+
We hope to continue developing, improving, and supporting **vantage6** with the help of the federated learning community. If you are interested in contributing, first of all, thank you! Second, please take a look at our [contributing guidelines](https://docs.vantage6.ai/en/main/devops/contribute.html)
|
|
148
|
+
|
|
149
|
+
<a href="https://github.com/vantage6/vantage6/graphs/contributors">
|
|
150
|
+
<img src="https://contrib.rocks/image?repo=vantage6/vantage6" />
|
|
151
|
+
</a>
|
|
152
|
+
|
|
153
|
+
## :black_nib: References
|
|
154
|
+
If you are using **vantage6**, please cite this repository as well as the accompanying papers as follows:
|
|
155
|
+
|
|
156
|
+
> * F. Martin, M. Sieswerda, H. Alradhi, et al. vantage6. Available at https://doi.org/10.5281/zenodo.7221216. Accessed on MONTH, 20XX.
|
|
157
|
+
> * A. Moncada-Torres, F. Martin, M. Sieswerda, J. van Soest, G. Gelijnse. VANTAGE6: an open source priVAcy preserviNg federaTed leArninG infrastructurE for Secure Insight eXchange. AMIA Annual Symposium Proceedings, 2020, p. 870-877. [[BibTeX](https://arturomoncadatorres.com/bibtex/moncada-torres2020vantage6.txt), [PDF](https://vantage6.ai/vantage6/)]
|
|
158
|
+
> * D. Smits\*, B. van Beusekom\*, F. Martin, L. Veen, G. Geleijnse, A. Moncada-Torres, An Improved Infrastructure for Privacy-Preserving Analysis of Patient Data, Proceedings of the International Conference of Informatics, Management, and Technology in Healthcare (ICIMTH), vol. 25, 2022, p. 144-147. [[BibTeX](https://arturomoncadatorres.com/bibtex/smits2022improved.txt), [PDF](https://ebooks.iospress.nl/volumearticle/60190)]
|
|
159
|
+
|
|
160
|
+
-----------------------------------------------------------------------------------------------------
|
|
161
|
+
<p align="center">
|
|
162
|
+
<a href="https://vantage6.ai">vantage6.ai</a> •
|
|
163
|
+
<a href="https://discord.gg/yAyFf6Y">Discord</a> •
|
|
164
|
+
<a href="https://vantage6.discourse.group/">Discourse</a> •
|
|
165
|
+
<a href="https://docs.vantage6.ai">User documentation</a> •
|
|
166
|
+
</p>
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import codecs
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from os import path
|
|
5
|
+
from setuptools import setup, find_namespace_packages
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
# get current directory
|
|
9
|
+
here = Path(path.abspath(path.dirname(__file__)))
|
|
10
|
+
parent_dir = here.parent.absolute()
|
|
11
|
+
|
|
12
|
+
# get the long description from the README file
|
|
13
|
+
with codecs.open(path.join(parent_dir, "README.md"), encoding="utf-8") as f:
|
|
14
|
+
long_description = f.read()
|
|
15
|
+
|
|
16
|
+
# Read the API version from disk. This file should be located in the package
|
|
17
|
+
# folder, since it's also used to set the pkg.__version__ variable.
|
|
18
|
+
version_path = os.path.join(here, "vantage6", "algorithm", "store", "_version.py")
|
|
19
|
+
version_ns = {"__file__": version_path}
|
|
20
|
+
with codecs.open(version_path) as f:
|
|
21
|
+
exec(f.read(), {}, version_ns)
|
|
22
|
+
|
|
23
|
+
# setup the package
|
|
24
|
+
setup(
|
|
25
|
+
name="vantage6-algorithm-store",
|
|
26
|
+
version=version_ns["__version__"],
|
|
27
|
+
description="Vantage6 algorithm store",
|
|
28
|
+
long_description=long_description,
|
|
29
|
+
long_description_content_type="text/markdown",
|
|
30
|
+
url="https://github.com/vantage6/vantage6",
|
|
31
|
+
packages=find_namespace_packages(),
|
|
32
|
+
python_requires=">=3.10",
|
|
33
|
+
install_requires=[
|
|
34
|
+
"bcrypt==4.0.1",
|
|
35
|
+
"flasgger==0.9.5",
|
|
36
|
+
"flask==2.2.5",
|
|
37
|
+
"Flask-Cors==3.0.10",
|
|
38
|
+
"Flask-JWT-Extended==4.4.4",
|
|
39
|
+
"Flask-Mail==0.9.1",
|
|
40
|
+
"Flask-Principal==0.4.0",
|
|
41
|
+
"Flask-RESTful==0.3.10",
|
|
42
|
+
"flask-marshmallow==0.15.0",
|
|
43
|
+
"Flask-SocketIO==5.3.6",
|
|
44
|
+
"gevent==23.9.1",
|
|
45
|
+
"ipython==8.10.0",
|
|
46
|
+
"kombu==5.2.4",
|
|
47
|
+
"marshmallow==3.19.0",
|
|
48
|
+
"marshmallow-sqlalchemy==0.29.0",
|
|
49
|
+
"PyJWT==2.6.0",
|
|
50
|
+
"pyotp==2.8.0",
|
|
51
|
+
"questionary==1.10.0",
|
|
52
|
+
"requests==2.31.0",
|
|
53
|
+
"requests-oauthlib==1.3.1",
|
|
54
|
+
"schema==0.7.5",
|
|
55
|
+
"SQLAlchemy==1.4.46",
|
|
56
|
+
"werkzeug==3.0.1",
|
|
57
|
+
f'vantage6 == {version_ns["__version__"]}',
|
|
58
|
+
f'vantage6-common == {version_ns["__version__"]}',
|
|
59
|
+
],
|
|
60
|
+
extras_require={"dev": ["coverage==6.4.4"]},
|
|
61
|
+
package_data={
|
|
62
|
+
"vantage6.server": [
|
|
63
|
+
"__build__",
|
|
64
|
+
"_data/**/*.yaml",
|
|
65
|
+
"server_data/*.yaml",
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
entry_points={
|
|
69
|
+
"console_scripts": ["vserver-local=vantage6.server.cli.server:cli_server"]
|
|
70
|
+
},
|
|
71
|
+
)
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The algorithm store holds the algorithms that are available to the vantage6
|
|
3
|
+
nodes. It is a repository of algorithms that can be coupled to a vantage6
|
|
4
|
+
server. The algorithms are stored in a database and can be managed through
|
|
5
|
+
the API. Note that is possible to couple multiple algorithm stores to a
|
|
6
|
+
vantage6 server. This allows both coupling a community store and a private
|
|
7
|
+
store to a vantage6 server.
|
|
8
|
+
"""
|
|
9
|
+
import os
|
|
10
|
+
from gevent import monkey
|
|
11
|
+
|
|
12
|
+
# This is a workaround for readthedocs
|
|
13
|
+
if not os.environ.get("READTHEDOCS"):
|
|
14
|
+
# flake8: noqa: E402 (ignore import error)
|
|
15
|
+
monkey.patch_all()
|
|
16
|
+
|
|
17
|
+
import importlib
|
|
18
|
+
import logging
|
|
19
|
+
import json
|
|
20
|
+
import traceback
|
|
21
|
+
|
|
22
|
+
from http import HTTPStatus
|
|
23
|
+
from werkzeug.exceptions import HTTPException
|
|
24
|
+
from flask import Flask, make_response, request, send_from_directory, Request, Response
|
|
25
|
+
from flask_cors import CORS
|
|
26
|
+
from flask_marshmallow import Marshmallow
|
|
27
|
+
from flask_restful import Api
|
|
28
|
+
from flask_principal import Principal
|
|
29
|
+
from flasgger import Swagger
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
from vantage6.common import logger_name
|
|
33
|
+
from vantage6.common.globals import APPNAME
|
|
34
|
+
from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
|
|
35
|
+
from vantage6.algorithm.store._version import __version__
|
|
36
|
+
from vantage6.algorithm.store.globals import API_PATH
|
|
37
|
+
|
|
38
|
+
# TODO the following are simply copies of the same files in the server - refactor
|
|
39
|
+
from vantage6.algorithm.store.model.base import Base, DatabaseSessionManager, Database
|
|
40
|
+
from vantage6.algorithm.store import db
|
|
41
|
+
|
|
42
|
+
# TODO move server imports to common / refactor
|
|
43
|
+
from vantage6.server.resource.common.output_schema import HATEOASModelSchema
|
|
44
|
+
from vantage6.server.permission import PermissionManager
|
|
45
|
+
from vantage6.algorithm.store.globals import RESOURCES, SERVER_MODULE_NAME
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
module_name = logger_name(__name__)
|
|
49
|
+
log = logging.getLogger(module_name)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class AlgorithmStoreApp:
|
|
53
|
+
"""
|
|
54
|
+
Vantage6 server instance.
|
|
55
|
+
|
|
56
|
+
Attributes
|
|
57
|
+
----------
|
|
58
|
+
ctx : AlgorithmStoreContext
|
|
59
|
+
Context object that contains the configuration of the algorithm store.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, ctx: AlgorithmStoreContext) -> None:
|
|
63
|
+
"""Create a vantage6-server application."""
|
|
64
|
+
|
|
65
|
+
self.ctx = ctx
|
|
66
|
+
|
|
67
|
+
# initialize, configure Flask
|
|
68
|
+
self.app = Flask(
|
|
69
|
+
SERVER_MODULE_NAME,
|
|
70
|
+
root_path=Path(__file__),
|
|
71
|
+
template_folder=Path(__file__).parent / "templates",
|
|
72
|
+
static_folder=Path(__file__).parent / "static",
|
|
73
|
+
)
|
|
74
|
+
self.debug: dict = self.ctx.config.get("debug", {})
|
|
75
|
+
self.configure_flask()
|
|
76
|
+
|
|
77
|
+
# Setup SQLAlchemy and Marshmallow for marshalling/serializing
|
|
78
|
+
self.ma = Marshmallow(self.app)
|
|
79
|
+
|
|
80
|
+
# Setup Principal, granular API access manegement
|
|
81
|
+
self.principal = Principal(self.app, use_sessions=False)
|
|
82
|
+
|
|
83
|
+
# Enable cross-origin resource sharing
|
|
84
|
+
self.cors = CORS(self.app)
|
|
85
|
+
|
|
86
|
+
# SWAGGER documentation
|
|
87
|
+
self.swagger = Swagger(self.app, template={})
|
|
88
|
+
|
|
89
|
+
# setup the permission manager for the API endpoints
|
|
90
|
+
# self.permissions = PermissionManager()
|
|
91
|
+
|
|
92
|
+
# Api - REST JSON-rpc
|
|
93
|
+
self.api = Api(self.app)
|
|
94
|
+
self.configure_api()
|
|
95
|
+
self.load_resources()
|
|
96
|
+
|
|
97
|
+
# set the server version
|
|
98
|
+
self.__version__ = __version__
|
|
99
|
+
|
|
100
|
+
log.info("Initialization done")
|
|
101
|
+
|
|
102
|
+
def configure_flask(self) -> None:
|
|
103
|
+
"""Configure the Flask settings of the vantage6 server."""
|
|
104
|
+
|
|
105
|
+
# let us handle exceptions
|
|
106
|
+
self.app.config["PROPAGATE_EXCEPTIONS"] = True
|
|
107
|
+
|
|
108
|
+
# Open Api Specification (f.k.a. swagger)
|
|
109
|
+
self.app.config["SWAGGER"] = {
|
|
110
|
+
"title": f"{APPNAME} algorithm store",
|
|
111
|
+
"uiversion": "3",
|
|
112
|
+
"openapi": "3.0.0",
|
|
113
|
+
"version": __version__,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
debug_mode = self.debug.get("flask", False)
|
|
117
|
+
if debug_mode:
|
|
118
|
+
log.debug("Flask debug mode enabled")
|
|
119
|
+
self.app.debug = debug_mode
|
|
120
|
+
|
|
121
|
+
def _get_request_path(request: Request) -> str:
|
|
122
|
+
"""
|
|
123
|
+
Return request extension of request URL, e.g.
|
|
124
|
+
http://localhost:5000/api/task/1 -> api/task/1
|
|
125
|
+
|
|
126
|
+
Parameters
|
|
127
|
+
----------
|
|
128
|
+
request: Request
|
|
129
|
+
Flask request object
|
|
130
|
+
|
|
131
|
+
Returns
|
|
132
|
+
-------
|
|
133
|
+
string:
|
|
134
|
+
The endpoint path of the request
|
|
135
|
+
"""
|
|
136
|
+
return request.url.replace(request.url_root, "")
|
|
137
|
+
|
|
138
|
+
# before request
|
|
139
|
+
@self.app.before_request
|
|
140
|
+
def do_before_request():
|
|
141
|
+
"""Before every flask request method."""
|
|
142
|
+
# Add log message before each request
|
|
143
|
+
log.debug(
|
|
144
|
+
"Received request: %s %s", request.method, _get_request_path(request)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# This will obtain a (scoped) db session from the session factory
|
|
148
|
+
# that is linked to the flask request global `g`. In every endpoint
|
|
149
|
+
# we then can access the database by using this session. We ensure
|
|
150
|
+
# that the session is removed (and uncommited changes are rolled
|
|
151
|
+
# back) at the end of every request.
|
|
152
|
+
DatabaseSessionManager.new_session()
|
|
153
|
+
|
|
154
|
+
@self.app.after_request
|
|
155
|
+
def remove_db_session(response):
|
|
156
|
+
"""After every flask request.
|
|
157
|
+
|
|
158
|
+
This will close the database session created by the
|
|
159
|
+
`before_request`.
|
|
160
|
+
"""
|
|
161
|
+
DatabaseSessionManager.clear_session()
|
|
162
|
+
return response
|
|
163
|
+
|
|
164
|
+
@self.app.errorhandler(HTTPException)
|
|
165
|
+
def error_remove_db_session(error: HTTPException):
|
|
166
|
+
"""In case an HTTP-exception occurs during the request.
|
|
167
|
+
|
|
168
|
+
It is important to close the db session to avoid having dangling
|
|
169
|
+
sessions.
|
|
170
|
+
"""
|
|
171
|
+
if error.code == 404:
|
|
172
|
+
log.debug("404 error for route '%s'", _get_request_path(request))
|
|
173
|
+
else:
|
|
174
|
+
log.warning("HTTP Exception occured during request")
|
|
175
|
+
log.debug("%s", traceback.format_exc())
|
|
176
|
+
DatabaseSessionManager.clear_session()
|
|
177
|
+
return error.get_response()
|
|
178
|
+
|
|
179
|
+
@self.app.errorhandler(Exception)
|
|
180
|
+
def error2_remove_db_session(error):
|
|
181
|
+
"""In case an exception occurs during the request.
|
|
182
|
+
|
|
183
|
+
It is important to close the db session to avoid having dangling
|
|
184
|
+
sessions.
|
|
185
|
+
"""
|
|
186
|
+
log.exception("Exception occured during request")
|
|
187
|
+
DatabaseSessionManager.clear_session()
|
|
188
|
+
return {
|
|
189
|
+
"msg": "An unexpected error occurred on the server!"
|
|
190
|
+
}, HTTPStatus.INTERNAL_SERVER_ERROR
|
|
191
|
+
|
|
192
|
+
@self.app.route("/robots.txt")
|
|
193
|
+
def static_from_root():
|
|
194
|
+
return send_from_directory(self.app.static_folder, request.path[1:])
|
|
195
|
+
|
|
196
|
+
def configure_api(self) -> None:
|
|
197
|
+
"""Define global API output and its structure."""
|
|
198
|
+
|
|
199
|
+
# helper to create HATEOAS schemas
|
|
200
|
+
HATEOASModelSchema.api = self.api
|
|
201
|
+
|
|
202
|
+
# whatever you get try to json it
|
|
203
|
+
@self.api.representation("application/json")
|
|
204
|
+
# pylint: disable=unused-argument
|
|
205
|
+
def output_json(
|
|
206
|
+
data: Base | list[Base], code: HTTPStatus, headers: dict = None
|
|
207
|
+
) -> Response:
|
|
208
|
+
"""
|
|
209
|
+
Return jsonified data for request responses.
|
|
210
|
+
|
|
211
|
+
Parameters
|
|
212
|
+
----------
|
|
213
|
+
data: Base | list[Base]
|
|
214
|
+
The data to be jsonified
|
|
215
|
+
code: HTTPStatus
|
|
216
|
+
The HTTP status code of the response
|
|
217
|
+
headers: dict
|
|
218
|
+
Additional headers to be added to the response
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
if isinstance(data, Base):
|
|
222
|
+
data = db.jsonable(data)
|
|
223
|
+
elif isinstance(data, list) and len(data) and isinstance(data[0], Base):
|
|
224
|
+
data = db.jsonable(data)
|
|
225
|
+
|
|
226
|
+
resp = make_response(json.dumps(data), code)
|
|
227
|
+
resp.headers.extend(headers or {})
|
|
228
|
+
return resp
|
|
229
|
+
|
|
230
|
+
def load_resources(self) -> None:
|
|
231
|
+
"""Import the modules containing API resources."""
|
|
232
|
+
|
|
233
|
+
# make services available to the endpoints, this way each endpoint can
|
|
234
|
+
# make use of 'em.
|
|
235
|
+
services = {
|
|
236
|
+
"api": self.api,
|
|
237
|
+
# "permissions": self.permissions,
|
|
238
|
+
"config": self.ctx.config,
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
for res in RESOURCES:
|
|
242
|
+
module = importlib.import_module("vantage6.algorithm.store.resource." + res)
|
|
243
|
+
module.setup(self.api, API_PATH, services)
|
|
244
|
+
|
|
245
|
+
def start(self) -> None:
|
|
246
|
+
"""
|
|
247
|
+
Start the server.
|
|
248
|
+
|
|
249
|
+
Before server is really started, some database settings are checked and
|
|
250
|
+
(re)set where appropriate.
|
|
251
|
+
"""
|
|
252
|
+
return self
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def run_server(config: str, system_folders: bool = True) -> AlgorithmStoreApp:
|
|
256
|
+
"""
|
|
257
|
+
Run a vantage6 server.
|
|
258
|
+
|
|
259
|
+
Parameters
|
|
260
|
+
----------
|
|
261
|
+
config: str
|
|
262
|
+
Configuration file path
|
|
263
|
+
system_folders: bool
|
|
264
|
+
Whether to use system or user folders. Default is True.
|
|
265
|
+
|
|
266
|
+
Returns
|
|
267
|
+
-------
|
|
268
|
+
AlgorithmStoreApp
|
|
269
|
+
A running instance of the vantage6 server
|
|
270
|
+
"""
|
|
271
|
+
ctx = AlgorithmStoreContext.from_external_config_file(config, system_folders)
|
|
272
|
+
allow_drop_all = ctx.config["allow_drop_all"]
|
|
273
|
+
Database().connect(uri=ctx.get_database_uri(), allow_drop_all=allow_drop_all)
|
|
274
|
+
return AlgorithmStoreApp(ctx).start()
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def run_dev_server(server_app: AlgorithmStoreApp, *args, **kwargs) -> None:
|
|
278
|
+
"""
|
|
279
|
+
Run a vantage6 development server (outside of a Docker container).
|
|
280
|
+
|
|
281
|
+
Parameters
|
|
282
|
+
----------
|
|
283
|
+
server_app: AlgorithmStoreApp
|
|
284
|
+
Instance of a vantage6 server
|
|
285
|
+
"""
|
|
286
|
+
log.warning("*" * 80)
|
|
287
|
+
log.warning(" DEVELOPMENT SERVER ".center(80, "*"))
|
|
288
|
+
log.warning("*" * 80)
|
|
289
|
+
kwargs.setdefault("log_output", False)
|
|
290
|
+
server_app.socketio.run(server_app.app, *args, **kwargs)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
here = os.path.abspath(os.path.dirname(__file__))
|
|
5
|
+
|
|
6
|
+
with open(os.path.join(here, "__build__")) as fp:
|
|
7
|
+
__build__ = json.load(fp)
|
|
8
|
+
|
|
9
|
+
# Module version
|
|
10
|
+
version_info = (4, 3, 0, "beta", __build__, 0)
|
|
11
|
+
|
|
12
|
+
# Module version stage suffix map
|
|
13
|
+
_specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}
|
|
14
|
+
version = f"{version_info[0]}.{version_info[1]}.{version_info[2]}"
|
|
15
|
+
pre_release = (
|
|
16
|
+
""
|
|
17
|
+
if version_info[3] == "final"
|
|
18
|
+
else _specifier_[version_info[3]] + str(version_info[4])
|
|
19
|
+
)
|
|
20
|
+
post_release = "" if not version_info[5] else f".post{version_info[5]}"
|
|
21
|
+
|
|
22
|
+
# Module version accessible using thomas.__version__
|
|
23
|
+
__version__ = f"{version}{pre_release}{post_release}"
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# TODO this is almost a copy of the same file in the server package. Refactor
|
|
2
|
+
# TODO this file is awkward...
|
|
3
|
+
import logging
|
|
4
|
+
import datetime
|
|
5
|
+
|
|
6
|
+
import enum
|
|
7
|
+
import sqlalchemy as sql
|
|
8
|
+
|
|
9
|
+
# Note: by importing these classes, the classes are registered in the Base's
|
|
10
|
+
# SQLAlchemy metadata. This is required for SQLAlchemy to be able to map the
|
|
11
|
+
# classes to the database tables, and e.g. initialize the database tables on
|
|
12
|
+
# startup.
|
|
13
|
+
from vantage6.algorithm.store.model import Base, Algorithm, Argument, Database, Function
|
|
14
|
+
from vantage6.common import logger_name
|
|
15
|
+
from vantage6.common.globals import STRING_ENCODING
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
module_name = logger_name(__name__)
|
|
19
|
+
log = logging.getLogger(module_name)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def jsonable(value: list[Base] | Base) -> list | dict:
|
|
23
|
+
"""
|
|
24
|
+
Convert a (list of) SQLAlchemy instance(s) to native Python objects.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
value : list[Base] | Base
|
|
29
|
+
A single SQLAlchemy instance or a list of SQLAlchemy instances
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
list | dict
|
|
34
|
+
A single Python object or a list of Python objects
|
|
35
|
+
|
|
36
|
+
Raises
|
|
37
|
+
------
|
|
38
|
+
Exception
|
|
39
|
+
If the value is not an instance of db.Base or a list of db.Base
|
|
40
|
+
"""
|
|
41
|
+
if isinstance(value, list):
|
|
42
|
+
return [jsonable(i) for i in value]
|
|
43
|
+
|
|
44
|
+
elif isinstance(value, Base):
|
|
45
|
+
log.debug(f"preparing={value}")
|
|
46
|
+
retval = dict()
|
|
47
|
+
mapper = sql.inspect(value.__class__)
|
|
48
|
+
|
|
49
|
+
columns = [
|
|
50
|
+
c.key for c in mapper.columns if c.key not in value._hidden_attributes
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
for column in columns:
|
|
54
|
+
# log.debug(f"processing column={column}")
|
|
55
|
+
column_value = getattr(value, column)
|
|
56
|
+
|
|
57
|
+
if isinstance(column_value, enum.Enum):
|
|
58
|
+
column_value = column_value.value
|
|
59
|
+
elif isinstance(column_value, datetime.datetime):
|
|
60
|
+
column_value = column_value.isoformat()
|
|
61
|
+
elif isinstance(column_value, bytes):
|
|
62
|
+
log.debug("decoding bytes!")
|
|
63
|
+
column_value = column_value.decode(STRING_ENCODING)
|
|
64
|
+
|
|
65
|
+
retval[column] = column_value
|
|
66
|
+
|
|
67
|
+
return retval
|
|
68
|
+
|
|
69
|
+
# FIXME: does it make sense to raise an exception or should base types
|
|
70
|
+
# (or other JSON-serializable types) just be returned as-is?
|
|
71
|
+
raise Exception("value should be instance of db.Base or list!")
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from vantage6.common.globals import APPNAME
|
|
4
|
+
|
|
5
|
+
# TODO cleanup this file
|
|
6
|
+
#
|
|
7
|
+
# INSTALLATION SETTINGS
|
|
8
|
+
#
|
|
9
|
+
# PACKAGE_FOLDER = Path(__file__).parent.parent.parent
|
|
10
|
+
|
|
11
|
+
SERVER_MODULE_NAME = APPNAME + "-algorithm-store"
|
|
12
|
+
|
|
13
|
+
# URL extension for the API endpoints
|
|
14
|
+
API_PATH = "/api"
|
|
15
|
+
|
|
16
|
+
# Which resources should be initialized. These names correspond to the
|
|
17
|
+
# file-names in the resource directory
|
|
18
|
+
RESOURCES = ["version", "algorithm", "vantage6_server"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from vantage6.algorithm.store.model.base import Base
|
|
2
|
+
from vantage6.algorithm.store.model.algorithm import Algorithm
|
|
3
|
+
from vantage6.algorithm.store.model.argument import Argument
|
|
4
|
+
from vantage6.algorithm.store.model.database import Database
|
|
5
|
+
from vantage6.algorithm.store.model.function import Function
|
|
6
|
+
from vantage6.algorithm.store.model.vantage6_server import Vantage6Server
|