pybiolib 1.1.1629__py3-none-any.whl → 1.1.1881__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/__init__.py +11 -3
- biolib/_internal/data_record/__init__.py +1 -0
- biolib/_internal/data_record/data_record.py +153 -0
- biolib/_internal/data_record/remote_storage_endpoint.py +27 -0
- biolib/_internal/http_client.py +45 -15
- biolib/_internal/push_application.py +22 -37
- biolib/_internal/runtime.py +73 -0
- biolib/_internal/utils/__init__.py +18 -0
- biolib/api/client.py +12 -6
- biolib/app/app.py +6 -1
- biolib/app/search_apps.py +8 -12
- biolib/biolib_api_client/api_client.py +14 -9
- biolib/biolib_api_client/app_types.py +1 -0
- biolib/biolib_api_client/auth.py +0 -12
- biolib/biolib_api_client/biolib_app_api.py +53 -27
- biolib/biolib_api_client/biolib_job_api.py +11 -40
- biolib/biolib_binary_format/utils.py +19 -2
- biolib/cli/__init__.py +9 -3
- biolib/cli/auth.py +58 -0
- biolib/cli/data_record.py +43 -0
- biolib/cli/download_container.py +3 -1
- biolib/cli/init.py +1 -0
- biolib/cli/lfs.py +39 -9
- biolib/cli/push.py +1 -1
- biolib/cli/run.py +3 -2
- biolib/cli/start.py +1 -0
- biolib/compute_node/cloud_utils/cloud_utils.py +38 -65
- biolib/compute_node/job_worker/cache_state.py +1 -1
- biolib/compute_node/job_worker/executors/docker_executor.py +10 -8
- biolib/compute_node/job_worker/job_storage.py +9 -13
- biolib/compute_node/job_worker/job_worker.py +10 -4
- biolib/compute_node/remote_host_proxy.py +48 -11
- biolib/compute_node/webserver/worker_thread.py +2 -2
- biolib/jobs/job.py +33 -32
- biolib/lfs/__init__.py +0 -2
- biolib/lfs/utils.py +23 -115
- biolib/runtime/__init__.py +13 -1
- biolib/sdk/__init__.py +17 -4
- biolib/user/sign_in.py +8 -12
- biolib/utils/__init__.py +17 -45
- biolib/utils/app_uri.py +11 -4
- biolib/utils/cache_state.py +2 -2
- biolib/utils/multipart_uploader.py +42 -68
- biolib/utils/seq_util.py +47 -9
- biolib/utils/zip/remote_zip.py +9 -17
- {pybiolib-1.1.1629.dist-info → pybiolib-1.1.1881.dist-info}/METADATA +1 -2
- {pybiolib-1.1.1629.dist-info → pybiolib-1.1.1881.dist-info}/RECORD +50 -46
- {pybiolib-1.1.1629.dist-info → pybiolib-1.1.1881.dist-info}/WHEEL +1 -1
- biolib/biolib_api_client/biolib_account_api.py +0 -21
- biolib/biolib_api_client/biolib_large_file_system_api.py +0 -53
- biolib/runtime/results.py +0 -20
- {pybiolib-1.1.1629.dist-info → pybiolib-1.1.1881.dist-info}/LICENSE +0 -0
- {pybiolib-1.1.1629.dist-info → pybiolib-1.1.1881.dist-info}/entry_points.txt +0 -0
biolib/__init__.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# Imports to hide
|
2
2
|
import os
|
3
|
+
from urllib.parse import urlparse as _urlparse
|
3
4
|
|
4
5
|
from biolib import typing_utils as _typing_utils
|
5
6
|
from biolib.app import BioLibApp as _BioLibApp
|
@@ -12,10 +13,12 @@ from biolib.biolib_api_client import BiolibApiClient as _BioLibApiClient, App
|
|
12
13
|
from biolib.jobs import Job as _Job
|
13
14
|
from biolib import user as _user
|
14
15
|
from biolib.typing_utils import List, Optional
|
16
|
+
from biolib._internal.data_record import DataRecord as _DataRecord
|
15
17
|
|
16
18
|
import biolib.api
|
17
19
|
import biolib.app
|
18
20
|
import biolib.cli
|
21
|
+
import biolib.sdk
|
19
22
|
import biolib.utils
|
20
23
|
|
21
24
|
|
@@ -32,8 +35,8 @@ def load(uri: str) -> _BioLibApp:
|
|
32
35
|
def search(
|
33
36
|
search_query: Optional[str] = None,
|
34
37
|
team: Optional[str] = None,
|
35
|
-
count: int = 100
|
36
|
-
|
38
|
+
count: int = 100,
|
39
|
+
) -> List[str]:
|
37
40
|
apps: List[str] = search_apps(search_query, team, count)
|
38
41
|
return apps
|
39
42
|
|
@@ -46,6 +49,10 @@ def fetch_jobs(count: int = 25) -> List[_Job]:
|
|
46
49
|
return _Job.fetch_jobs(count)
|
47
50
|
|
48
51
|
|
52
|
+
def fetch_data_records(uri: Optional[str] = None, count: Optional[int] = None) -> List[_DataRecord]:
|
53
|
+
return _DataRecord.fetch(uri, count)
|
54
|
+
|
55
|
+
|
49
56
|
def get_experiment(name: str) -> Experiment:
|
50
57
|
return Experiment(name)
|
51
58
|
|
@@ -77,6 +84,7 @@ def logout() -> None:
|
|
77
84
|
def set_api_base_url(api_base_url: str) -> None:
|
78
85
|
_BioLibApiClient.initialize(base_url=api_base_url)
|
79
86
|
biolib.utils.BIOLIB_BASE_URL = api_base_url
|
87
|
+
biolib.utils.BIOLIB_SITE_HOSTNAME = _urlparse(api_base_url).hostname
|
80
88
|
biolib.utils.BASE_URL_IS_PUBLIC_BIOLIB = api_base_url.endswith('biolib.com') or (
|
81
89
|
os.environ.get('BIOLIB_ENVIRONMENT_IS_PUBLIC_BIOLIB', '').upper() == 'TRUE'
|
82
90
|
)
|
@@ -127,4 +135,4 @@ _logger.configure(default_log_level=_DEFAULT_LOG_LEVEL)
|
|
127
135
|
_logger_no_user_data.configure(default_log_level=_DEFAULT_LOG_LEVEL)
|
128
136
|
_configure_requests_certificates()
|
129
137
|
|
130
|
-
set_api_base_url(biolib.utils.
|
138
|
+
set_api_base_url(biolib.utils.load_base_url_from_env())
|
@@ -0,0 +1 @@
|
|
1
|
+
from .data_record import DataRecord
|
@@ -0,0 +1,153 @@
|
|
1
|
+
import os
|
2
|
+
from collections import namedtuple
|
3
|
+
from datetime import datetime
|
4
|
+
from fnmatch import fnmatch
|
5
|
+
from struct import Struct
|
6
|
+
from typing import Callable, Dict, List, Optional, Union, cast
|
7
|
+
|
8
|
+
from biolib import lfs
|
9
|
+
from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
|
10
|
+
from biolib._internal.http_client import HttpClient
|
11
|
+
from biolib.api import client as api_client
|
12
|
+
from biolib.biolib_api_client import AppGetResponse
|
13
|
+
from biolib.biolib_binary_format import LazyLoadedFile
|
14
|
+
from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
|
15
|
+
from biolib.biolib_logging import logger
|
16
|
+
from biolib.utils.app_uri import parse_app_uri
|
17
|
+
from biolib.utils.zip.remote_zip import RemoteZip # type: ignore
|
18
|
+
|
19
|
+
PathFilter = Union[str, Callable[[str], bool]]
|
20
|
+
|
21
|
+
|
22
|
+
class DataRecord:
|
23
|
+
def __init__(self, uri: str):
|
24
|
+
self._uri = uri
|
25
|
+
uri_parsed = parse_app_uri(uri, use_account_as_name_default=False)
|
26
|
+
if not uri_parsed['app_name']:
|
27
|
+
raise ValueError('Expected parameter "uri" to contain resource name')
|
28
|
+
|
29
|
+
self._name = uri_parsed['app_name']
|
30
|
+
|
31
|
+
@property
|
32
|
+
def uri(self) -> str:
|
33
|
+
return self._uri
|
34
|
+
|
35
|
+
@property
|
36
|
+
def name(self) -> str:
|
37
|
+
return self._name
|
38
|
+
|
39
|
+
def list_files(self, path_filter: Optional[PathFilter] = None) -> List[LazyLoadedFile]:
|
40
|
+
app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': self._uri}).json()
|
41
|
+
remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
|
42
|
+
resource_version_uuid=app_response['app_version']['public_id'],
|
43
|
+
)
|
44
|
+
files: List[LazyLoadedFile] = []
|
45
|
+
with RemoteZip(url=remote_storage_endpoint.get_remote_url()) as remote_zip:
|
46
|
+
central_directory = remote_zip.get_central_directory()
|
47
|
+
for file_info in central_directory.values():
|
48
|
+
files.append(self._get_file(remote_storage_endpoint, file_info))
|
49
|
+
|
50
|
+
return self._get_filtered_files(files=files, path_filter=path_filter) if path_filter else files
|
51
|
+
|
52
|
+
def download_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
|
53
|
+
filtered_files = self.list_files(path_filter=path_filter)
|
54
|
+
|
55
|
+
if len(filtered_files) == 0:
|
56
|
+
logger.debug('No files to save')
|
57
|
+
return
|
58
|
+
|
59
|
+
for file in filtered_files:
|
60
|
+
file_path = os.path.join(output_dir, file.path)
|
61
|
+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
62
|
+
with open(file_path, mode='wb') as file_handle:
|
63
|
+
file_handle.write(file.get_data())
|
64
|
+
|
65
|
+
def save_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
|
66
|
+
self.download_files(output_dir=output_dir, path_filter=path_filter)
|
67
|
+
|
68
|
+
@staticmethod
|
69
|
+
def create(destination: str, data_path: str, name: Optional[str] = None) -> 'DataRecord':
|
70
|
+
assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
|
71
|
+
record_name = name if name else 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
|
72
|
+
record_uri = lfs.create_large_file_system(lfs_uri=f'{destination}/{record_name}')
|
73
|
+
record_version_uri = lfs.push_large_file_system(lfs_uri=record_uri, input_dir=data_path)
|
74
|
+
return DataRecord(uri=record_version_uri)
|
75
|
+
|
76
|
+
@staticmethod
|
77
|
+
def fetch(uri: Optional[str] = None, count: Optional[int] = None) -> List['DataRecord']:
|
78
|
+
max_page_size = 1_000
|
79
|
+
params: Dict[str, Union[str, int]] = {
|
80
|
+
'page_size': str(count or max_page_size),
|
81
|
+
'resource_type': 'data-record',
|
82
|
+
}
|
83
|
+
if uri:
|
84
|
+
uri_parsed = parse_app_uri(uri, use_account_as_name_default=False)
|
85
|
+
params['account_handle'] = uri_parsed['account_handle_normalized']
|
86
|
+
|
87
|
+
results = api_client.get(path='/apps/', params=params).json()['results']
|
88
|
+
if count is None and len(results) == max_page_size:
|
89
|
+
logger.warning(
|
90
|
+
f'Fetch results exceeded maximum count of {max_page_size}. Some data records might not be fetched.'
|
91
|
+
)
|
92
|
+
|
93
|
+
return [DataRecord(result['resource_uri']) for result in results]
|
94
|
+
|
95
|
+
@staticmethod
|
96
|
+
def _get_file(remote_storage_endpoint: DataRecordRemoteStorageEndpoint, file_info: Dict) -> LazyLoadedFile:
|
97
|
+
local_file_header_signature_bytes = b'\x50\x4b\x03\x04'
|
98
|
+
local_file_header_struct = Struct('<H2sHHHIIIHH')
|
99
|
+
LocalFileHeader = namedtuple(
|
100
|
+
'LocalFileHeader',
|
101
|
+
(
|
102
|
+
'version',
|
103
|
+
'flags',
|
104
|
+
'compression_raw',
|
105
|
+
'mod_time',
|
106
|
+
'mod_date',
|
107
|
+
'crc_32_expected',
|
108
|
+
'compressed_size_raw',
|
109
|
+
'uncompressed_size_raw',
|
110
|
+
'file_name_len',
|
111
|
+
'extra_field_len',
|
112
|
+
),
|
113
|
+
)
|
114
|
+
|
115
|
+
local_file_header_start = file_info['header_offset'] + len(local_file_header_signature_bytes)
|
116
|
+
local_file_header_end = local_file_header_start + local_file_header_struct.size
|
117
|
+
|
118
|
+
def file_start_func() -> int:
|
119
|
+
local_file_header_response = HttpClient.request(
|
120
|
+
url=remote_storage_endpoint.get_remote_url(),
|
121
|
+
headers={'range': f'bytes={local_file_header_start}-{local_file_header_end - 1}'},
|
122
|
+
timeout_in_seconds=300,
|
123
|
+
)
|
124
|
+
local_file_header = LocalFileHeader._make(
|
125
|
+
local_file_header_struct.unpack(local_file_header_response.content)
|
126
|
+
)
|
127
|
+
file_start: int = (
|
128
|
+
local_file_header_end + local_file_header.file_name_len + local_file_header.extra_field_len
|
129
|
+
)
|
130
|
+
return file_start
|
131
|
+
|
132
|
+
return LazyLoadedFile(
|
133
|
+
buffer=RemoteIndexableBuffer(endpoint=remote_storage_endpoint),
|
134
|
+
length=file_info['file_size'],
|
135
|
+
path=file_info['filename'],
|
136
|
+
start=None,
|
137
|
+
start_func=file_start_func,
|
138
|
+
)
|
139
|
+
|
140
|
+
@staticmethod
|
141
|
+
def _get_filtered_files(files: List[LazyLoadedFile], path_filter: PathFilter) -> List[LazyLoadedFile]:
|
142
|
+
if not (isinstance(path_filter, str) or callable(path_filter)):
|
143
|
+
raise Exception('Expected path_filter to be a string or a function')
|
144
|
+
|
145
|
+
if callable(path_filter):
|
146
|
+
return list(filter(lambda x: path_filter(x.path), files)) # type: ignore
|
147
|
+
|
148
|
+
glob_filter = cast(str, path_filter)
|
149
|
+
|
150
|
+
def _filter_function(file: LazyLoadedFile) -> bool:
|
151
|
+
return fnmatch(file.path, glob_filter)
|
152
|
+
|
153
|
+
return list(filter(_filter_function, files))
|
@@ -0,0 +1,27 @@
|
|
1
|
+
from datetime import datetime, timedelta
|
2
|
+
|
3
|
+
from biolib.api import client as api_client
|
4
|
+
from biolib.biolib_api_client.lfs_types import LargeFileSystemVersion
|
5
|
+
from biolib.biolib_binary_format.utils import RemoteEndpoint
|
6
|
+
from biolib.biolib_logging import logger
|
7
|
+
|
8
|
+
|
9
|
+
class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
|
10
|
+
def __init__(self, resource_version_uuid: str):
|
11
|
+
self._resource_version_uuid: str = resource_version_uuid
|
12
|
+
self._expires_at = None
|
13
|
+
self._presigned_url = None
|
14
|
+
|
15
|
+
def get_remote_url(self):
|
16
|
+
if not self._presigned_url or datetime.utcnow() > self._expires_at:
|
17
|
+
lfs_version: LargeFileSystemVersion = api_client.get(
|
18
|
+
path=f'/lfs/versions/{self._resource_version_uuid}/',
|
19
|
+
).json()
|
20
|
+
self._presigned_url = lfs_version['presigned_download_url']
|
21
|
+
self._expires_at = datetime.utcnow() + timedelta(minutes=8)
|
22
|
+
logger.debug(
|
23
|
+
f'DataRecord "{self._resource_version_uuid}" fetched presigned URL '
|
24
|
+
f'with expiry at {self._expires_at.isoformat()}'
|
25
|
+
)
|
26
|
+
|
27
|
+
return self._presigned_url
|
biolib/_internal/http_client.py
CHANGED
@@ -1,12 +1,28 @@
|
|
1
1
|
import json
|
2
|
-
import
|
2
|
+
import platform
|
3
3
|
import socket
|
4
|
-
import
|
4
|
+
import ssl
|
5
|
+
import subprocess
|
6
|
+
import time
|
5
7
|
import urllib.error
|
6
8
|
import urllib.parse
|
9
|
+
import urllib.request
|
7
10
|
|
8
11
|
from biolib.biolib_logging import logger_no_user_data
|
9
|
-
from biolib.typing_utils import Dict, Optional, Union,
|
12
|
+
from biolib.typing_utils import Dict, Literal, Optional, Union, cast
|
13
|
+
|
14
|
+
_HttpMethod = Literal['GET', 'POST', 'PATCH', 'PUT']
|
15
|
+
|
16
|
+
|
17
|
+
def _create_ssl_context():
|
18
|
+
context = ssl.create_default_context()
|
19
|
+
try:
|
20
|
+
if platform.system() == 'Darwin':
|
21
|
+
certificates = subprocess.check_output('security find-certificate -a -p', shell=True).decode('utf-8')
|
22
|
+
context.load_verify_locations(cadata=certificates)
|
23
|
+
except BaseException:
|
24
|
+
pass
|
25
|
+
return context
|
10
26
|
|
11
27
|
|
12
28
|
class HttpError(urllib.error.HTTPError):
|
@@ -16,7 +32,7 @@ class HttpError(urllib.error.HTTPError):
|
|
16
32
|
code=http_error.code,
|
17
33
|
msg=http_error.msg, # type: ignore
|
18
34
|
hdrs=http_error.hdrs, # type: ignore
|
19
|
-
fp=http_error.fp
|
35
|
+
fp=http_error.fp,
|
20
36
|
)
|
21
37
|
|
22
38
|
def __str__(self):
|
@@ -25,10 +41,11 @@ class HttpError(urllib.error.HTTPError):
|
|
25
41
|
|
26
42
|
|
27
43
|
class HttpResponse:
|
28
|
-
def __init__(self, response):
|
29
|
-
self.
|
30
|
-
self.
|
31
|
-
self.
|
44
|
+
def __init__(self, response) -> None:
|
45
|
+
self.headers: Dict[str, str] = dict(response.headers)
|
46
|
+
self.status_code: int = int(response.status)
|
47
|
+
self.content: bytes = response.read()
|
48
|
+
self.url: str = response.geturl()
|
32
49
|
|
33
50
|
@property
|
34
51
|
def text(self) -> str:
|
@@ -39,14 +56,19 @@ class HttpResponse:
|
|
39
56
|
|
40
57
|
|
41
58
|
class HttpClient:
|
59
|
+
ssl_context = None
|
60
|
+
|
42
61
|
@staticmethod
|
43
62
|
def request(
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
63
|
+
url: str,
|
64
|
+
method: Optional[_HttpMethod] = None,
|
65
|
+
data: Optional[Union[Dict, bytes]] = None,
|
66
|
+
headers: Optional[Dict[str, str]] = None,
|
67
|
+
retries: int = 5,
|
68
|
+
timeout_in_seconds: Optional[int] = None,
|
49
69
|
) -> HttpResponse:
|
70
|
+
if not HttpClient.ssl_context:
|
71
|
+
HttpClient.ssl_context = _create_ssl_context()
|
50
72
|
headers_to_send = headers or {}
|
51
73
|
if isinstance(data, dict):
|
52
74
|
headers_to_send['Accept'] = 'application/json'
|
@@ -58,7 +80,8 @@ class HttpClient:
|
|
58
80
|
headers=headers_to_send,
|
59
81
|
method=method or 'GET',
|
60
82
|
)
|
61
|
-
|
83
|
+
if timeout_in_seconds is None:
|
84
|
+
timeout_in_seconds = 60 if isinstance(data, dict) else 180 # TODO: Calculate timeout based on data size
|
62
85
|
|
63
86
|
last_error: Optional[urllib.error.URLError] = None
|
64
87
|
for retry_count in range(retries + 1):
|
@@ -66,13 +89,20 @@ class HttpClient:
|
|
66
89
|
time.sleep(5 * retry_count)
|
67
90
|
logger_no_user_data.debug(f'Retrying HTTP {method} request...')
|
68
91
|
try:
|
69
|
-
with urllib.request.urlopen(
|
92
|
+
with urllib.request.urlopen(
|
93
|
+
request,
|
94
|
+
context=HttpClient.ssl_context,
|
95
|
+
timeout=timeout_in_seconds,
|
96
|
+
) as response:
|
70
97
|
return HttpResponse(response)
|
71
98
|
|
72
99
|
except urllib.error.HTTPError as error:
|
73
100
|
if error.code == 502:
|
74
101
|
logger_no_user_data.debug(f'HTTP {method} request failed with status 502 for "{url}"')
|
75
102
|
last_error = error
|
103
|
+
elif error.code == 503:
|
104
|
+
logger_no_user_data.debug(f'HTTP {method} request failed with status 503 for "{url}"')
|
105
|
+
last_error = error
|
76
106
|
else:
|
77
107
|
raise HttpError(error) from None
|
78
108
|
|
@@ -1,17 +1,18 @@
|
|
1
1
|
import os
|
2
2
|
import re
|
3
3
|
from pathlib import Path
|
4
|
-
|
4
|
+
|
5
5
|
import rich.progress
|
6
|
+
import yaml
|
6
7
|
|
7
|
-
from biolib
|
8
|
-
from biolib.typing_utils import Optional, Set, TypedDict, Iterable
|
8
|
+
from biolib import api, utils
|
9
9
|
from biolib.biolib_api_client import BiolibApiClient
|
10
|
-
from biolib.biolib_docker_client import BiolibDockerClient
|
11
10
|
from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
|
11
|
+
from biolib.biolib_docker_client import BiolibDockerClient
|
12
12
|
from biolib.biolib_errors import BioLibError
|
13
13
|
from biolib.biolib_logging import logger
|
14
|
-
from biolib import
|
14
|
+
from biolib.lfs.utils import get_files_and_size_of_directory, get_iterable_zip_stream
|
15
|
+
from biolib.typing_utils import Iterable, Optional, Set, TypedDict
|
15
16
|
|
16
17
|
REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')
|
17
18
|
|
@@ -38,9 +39,7 @@ def process_docker_status_updates(status_updates: Iterable[DockerStatusUpdate],
|
|
38
39
|
progress_detail = update['progressDetail']
|
39
40
|
|
40
41
|
if layer_id not in layer_id_to_task_id:
|
41
|
-
layer_id_to_task_id[layer_id] = progress.add_task(
|
42
|
-
description=f'[cyan]{action} layer {layer_id}'
|
43
|
-
)
|
42
|
+
layer_id_to_task_id[layer_id] = progress.add_task(description=f'[cyan]{action} layer {layer_id}')
|
44
43
|
|
45
44
|
if progress_detail and 'current' in progress_detail and 'total' in progress_detail:
|
46
45
|
progress.update(
|
@@ -60,7 +59,7 @@ def process_docker_status_updates(status_updates: Iterable[DockerStatusUpdate],
|
|
60
59
|
|
61
60
|
|
62
61
|
def set_app_version_as_active(
|
63
|
-
|
62
|
+
app_version_uuid: str,
|
64
63
|
):
|
65
64
|
logger.debug(f'Setting app version {app_version_uuid} as active.')
|
66
65
|
api.client.patch(
|
@@ -70,10 +69,10 @@ def set_app_version_as_active(
|
|
70
69
|
|
71
70
|
|
72
71
|
def push_application(
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
72
|
+
app_uri: str,
|
73
|
+
app_path: str,
|
74
|
+
app_version_to_copy_images_from: Optional[str],
|
75
|
+
is_dev_version: Optional[bool],
|
77
76
|
):
|
78
77
|
app_path_absolute = Path(app_path).resolve()
|
79
78
|
|
@@ -96,7 +95,7 @@ def push_application(
|
|
96
95
|
|
97
96
|
input_files_maps_to_root = False
|
98
97
|
try:
|
99
|
-
with open(config_yml_path
|
98
|
+
with open(config_yml_path) as config_yml_file:
|
100
99
|
config = yaml.safe_load(config_yml_file.read())
|
101
100
|
|
102
101
|
license_file_relative_path = config.get('license_file', 'LICENSE')
|
@@ -109,7 +108,7 @@ def push_application(
|
|
109
108
|
raise BioLibError(f'Could not find {description_file_relative_path}')
|
110
109
|
|
111
110
|
zip_filters.add(description_file_relative_path)
|
112
|
-
with open(description_file_absolute_path
|
111
|
+
with open(description_file_absolute_path) as description_file:
|
113
112
|
description_file_content = description_file.read()
|
114
113
|
|
115
114
|
for _, img_src_path in re.findall(REGEX_MARKDOWN_INLINE_IMAGE, description_file_content):
|
@@ -171,8 +170,9 @@ def push_application(
|
|
171
170
|
author=app['account_handle'],
|
172
171
|
set_as_active=False,
|
173
172
|
zip_binary=source_files_zip_bytes,
|
174
|
-
app_version_id_to_copy_images_from=app_response['app_version']['public_id']
|
175
|
-
|
173
|
+
app_version_id_to_copy_images_from=app_response['app_version']['public_id']
|
174
|
+
if app_version_to_copy_images_from
|
175
|
+
else None,
|
176
176
|
)
|
177
177
|
|
178
178
|
# Don't push docker images if copying from another app version
|
@@ -180,18 +180,6 @@ def push_application(
|
|
180
180
|
if not app_version_to_copy_images_from and docker_tags:
|
181
181
|
logger.info('Found docker images to push.')
|
182
182
|
|
183
|
-
try:
|
184
|
-
yaml_file = open(f'{app_path}/.biolib/config.yml', 'r', encoding='utf-8')
|
185
|
-
|
186
|
-
except Exception as error: # pylint: disable=broad-except
|
187
|
-
raise BioLibError('Could not open the config file .biolib/config.yml') from error
|
188
|
-
|
189
|
-
try:
|
190
|
-
config_data = yaml.safe_load(yaml_file)
|
191
|
-
|
192
|
-
except Exception as error: # pylint: disable=broad-except
|
193
|
-
raise BioLibError('Could not parse .biolib/config.yml. Please make sure it is valid YAML') from error
|
194
|
-
|
195
183
|
# Auth to be sent to proxy
|
196
184
|
# The tokens are sent as "{access_token},{job_id}". We leave job_id blank on push.
|
197
185
|
tokens = f'{BiolibApiClient.get().access_token},'
|
@@ -200,14 +188,12 @@ def push_application(
|
|
200
188
|
docker_client = BiolibDockerClient.get_docker_client()
|
201
189
|
|
202
190
|
for module_name, repo_and_tag in docker_tags.items():
|
203
|
-
docker_image_definition =
|
191
|
+
docker_image_definition = config['modules'][module_name]['image']
|
204
192
|
repo, tag = repo_and_tag.split(':')
|
205
193
|
|
206
194
|
if docker_image_definition.startswith('dockerhub://'):
|
207
195
|
docker_image_name = docker_image_definition.replace('dockerhub://', 'docker.io/', 1)
|
208
|
-
logger.info(
|
209
|
-
f'Pulling image {docker_image_name} defined on module {module_name} from Dockerhub.'
|
210
|
-
)
|
196
|
+
logger.info(f'Pulling image {docker_image_name} defined on module {module_name} from Dockerhub.')
|
211
197
|
dockerhub_repo, dockerhub_tag = docker_image_name.split(':')
|
212
198
|
pull_status_updates: Iterable[DockerStatusUpdate] = docker_client.api.pull(
|
213
199
|
decode=True,
|
@@ -238,7 +224,7 @@ def push_application(
|
|
238
224
|
|
239
225
|
process_docker_status_updates(push_status_updates, action='Pushing')
|
240
226
|
|
241
|
-
except Exception as exception:
|
227
|
+
except Exception as exception:
|
242
228
|
raise BioLibError(f'Failed to tag and push image {docker_image_name}.') from exception
|
243
229
|
|
244
230
|
logger.info(f'Successfully pushed {docker_image_name}')
|
@@ -249,10 +235,9 @@ def push_application(
|
|
249
235
|
data={'set_as_active': not is_dev_version},
|
250
236
|
)
|
251
237
|
|
252
|
-
sematic_version =
|
253
|
-
f"{new_app_version_json['major']}.{new_app_version_json['minor']}.{new_app_version_json['patch']}"
|
238
|
+
sematic_version = f"{new_app_version_json['major']}.{new_app_version_json['minor']}.{new_app_version_json['patch']}"
|
254
239
|
logger.info(
|
255
240
|
f"Successfully pushed new {'development ' if is_dev_version else ''}version {sematic_version} of {app_uri}."
|
256
241
|
)
|
257
242
|
|
258
|
-
return {
|
243
|
+
return {'app_uri': app_uri, 'sematic_version': sematic_version}
|
@@ -0,0 +1,73 @@
|
|
1
|
+
import json
|
2
|
+
|
3
|
+
from biolib import api
|
4
|
+
from biolib.typing_utils import Optional, TypedDict, cast
|
5
|
+
|
6
|
+
|
7
|
+
class RuntimeJobDataDict(TypedDict):
|
8
|
+
version: str
|
9
|
+
job_requested_machine: str
|
10
|
+
job_uuid: str
|
11
|
+
job_auth_token: str
|
12
|
+
|
13
|
+
|
14
|
+
class BioLibRuntimeError(Exception):
|
15
|
+
pass
|
16
|
+
|
17
|
+
|
18
|
+
class BioLibRuntimeNotRecognizedError(BioLibRuntimeError):
|
19
|
+
def __init__(self, message='The runtime is not recognized as a BioLib app'):
|
20
|
+
self.message = message
|
21
|
+
super().__init__(self.message)
|
22
|
+
|
23
|
+
|
24
|
+
class Runtime:
|
25
|
+
_job_data: Optional[RuntimeJobDataDict] = None
|
26
|
+
|
27
|
+
@staticmethod
|
28
|
+
def check_is_environment_biolib_app() -> bool:
|
29
|
+
return bool(Runtime._try_to_get_job_data())
|
30
|
+
|
31
|
+
@staticmethod
|
32
|
+
def get_job_id() -> str:
|
33
|
+
return Runtime._get_job_data()['job_uuid']
|
34
|
+
|
35
|
+
@staticmethod
|
36
|
+
def get_job_auth_token() -> str:
|
37
|
+
return Runtime._get_job_data()['job_auth_token']
|
38
|
+
|
39
|
+
@staticmethod
|
40
|
+
def get_job_requested_machine() -> str:
|
41
|
+
return Runtime._get_job_data()['job_requested_machine']
|
42
|
+
|
43
|
+
@staticmethod
|
44
|
+
def set_main_result_prefix(result_prefix: str) -> None:
|
45
|
+
job_data = Runtime._get_job_data()
|
46
|
+
api.client.patch(
|
47
|
+
data={'result_name_prefix': result_prefix},
|
48
|
+
headers={'Job-Auth-Token': job_data['job_auth_token']},
|
49
|
+
path=f"/jobs/{job_data['job_uuid']}/main_result/",
|
50
|
+
)
|
51
|
+
|
52
|
+
@staticmethod
|
53
|
+
def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
|
54
|
+
if not Runtime._job_data:
|
55
|
+
try:
|
56
|
+
with open('/biolib/secrets/biolib_system_secret') as file:
|
57
|
+
job_data: RuntimeJobDataDict = json.load(file)
|
58
|
+
except BaseException:
|
59
|
+
return None
|
60
|
+
|
61
|
+
if not job_data['version'].startswith('1.'):
|
62
|
+
raise BioLibRuntimeError(f"Unexpected system secret version {job_data['version']} expected 1.x.x")
|
63
|
+
|
64
|
+
Runtime._job_data = job_data
|
65
|
+
|
66
|
+
return cast(RuntimeJobDataDict, Runtime._job_data)
|
67
|
+
|
68
|
+
@staticmethod
|
69
|
+
def _get_job_data() -> RuntimeJobDataDict:
|
70
|
+
job_data = Runtime._try_to_get_job_data()
|
71
|
+
if not job_data:
|
72
|
+
raise BioLibRuntimeNotRecognizedError() from None
|
73
|
+
return job_data
|
@@ -0,0 +1,18 @@
|
|
1
|
+
import time
|
2
|
+
import uuid
|
3
|
+
|
4
|
+
|
5
|
+
def open_browser_window_from_notebook(url_to_open: str) -> None:
|
6
|
+
try:
|
7
|
+
from IPython.display import ( # type:ignore # pylint: disable=import-error, import-outside-toplevel
|
8
|
+
Javascript,
|
9
|
+
display,
|
10
|
+
update_display,
|
11
|
+
)
|
12
|
+
except ImportError as error:
|
13
|
+
raise Exception('Unexpected environment. This function can only be called from a notebook.') from error
|
14
|
+
|
15
|
+
display_id = str(uuid.uuid4())
|
16
|
+
display(Javascript(f'window.open("{url_to_open}");'), display_id=display_id)
|
17
|
+
time.sleep(1)
|
18
|
+
update_display(Javascript(''), display_id=display_id)
|
biolib/api/client.py
CHANGED
@@ -4,7 +4,10 @@ from biolib.typing_utils import Dict, Optional, Union
|
|
4
4
|
from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
|
5
5
|
from biolib._internal.http_client import HttpResponse, HttpClient
|
6
6
|
|
7
|
-
OptionalHeaders =
|
7
|
+
OptionalHeaders = Union[
|
8
|
+
Optional[Dict[str, str]],
|
9
|
+
Optional[Dict[str, Union[str, None]]],
|
10
|
+
]
|
8
11
|
|
9
12
|
|
10
13
|
class ApiClient(HttpClient):
|
@@ -14,11 +17,12 @@ class ApiClient(HttpClient):
|
|
14
17
|
params: Optional[Dict[str, Union[str, int]]] = None,
|
15
18
|
headers: OptionalHeaders = None,
|
16
19
|
authenticate: bool = True,
|
20
|
+
retries: int = 10,
|
17
21
|
) -> HttpResponse:
|
18
22
|
return self.request(
|
19
23
|
headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
|
20
24
|
method='GET',
|
21
|
-
retries=
|
25
|
+
retries=retries,
|
22
26
|
url=self._get_absolute_url(path=path, query_params=params),
|
23
27
|
)
|
24
28
|
|
@@ -27,21 +31,23 @@ class ApiClient(HttpClient):
|
|
27
31
|
path: str,
|
28
32
|
data: Optional[Union[Dict, bytes]] = None,
|
29
33
|
headers: OptionalHeaders = None,
|
34
|
+
authenticate: bool = True,
|
35
|
+
retries: int = 5,
|
30
36
|
) -> HttpResponse:
|
31
37
|
return self.request(
|
32
38
|
data=data,
|
33
|
-
headers=self._get_headers(opt_headers=headers, authenticate=
|
39
|
+
headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
|
34
40
|
method='POST',
|
35
|
-
retries=
|
41
|
+
retries=retries,
|
36
42
|
url=self._get_absolute_url(path=path, query_params=None),
|
37
43
|
)
|
38
44
|
|
39
|
-
def patch(self, path: str, data: Dict, headers: OptionalHeaders = None) -> HttpResponse:
|
45
|
+
def patch(self, path: str, data: Dict, headers: OptionalHeaders = None, retries: int = 5) -> HttpResponse:
|
40
46
|
return self.request(
|
41
47
|
data=data,
|
42
48
|
headers=self._get_headers(opt_headers=headers, authenticate=True),
|
43
49
|
method='PATCH',
|
44
|
-
retries=
|
50
|
+
retries=retries,
|
45
51
|
url=self._get_absolute_url(path=path, query_params=None),
|
46
52
|
)
|
47
53
|
|
biolib/app/app.py
CHANGED
@@ -263,7 +263,12 @@ Example: "app.cli('--help')"
|
|
263
263
|
if not key.startswith('--'):
|
264
264
|
key = f'--{key}'
|
265
265
|
|
266
|
-
args.
|
266
|
+
args.append(key)
|
267
|
+
if isinstance(value, list):
|
268
|
+
# TODO: only do this if argument key is of type file list
|
269
|
+
args.extend(value)
|
270
|
+
else:
|
271
|
+
args.append(value)
|
267
272
|
|
268
273
|
return self.cli(args, **biolib_kwargs)
|
269
274
|
|